-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature 320 update schema mpr orank (#322)
* Fix pwd value * Issue #320 updates for MPR and ORANK linetype changes * Fix indentation * Issue #320 updates to MPR and ORANK linetypes * Issue #320 schema updates to reflect changes to MPR and ORANK linetypes. Test, config, and test data. * Skip testing MPR linetype in reformatter. There were changes to the columns: names and additional columnsthat will break this test until the METreformat module is updated. * Issue #320 fixed syntax error * Add pip install of pytest * Issue #320 tests to verify that the renamed column names were created and the new columns are present. * Issue #320 config file used to test that the new schema was loaded. * Update update_for_6_0_beta6.sql fixed the ADD COLUMN lines for ORANK --------- Co-authored-by: Minna Win <minnawin@dakota.rap.ucar.edu>
- Loading branch information
Showing
10 changed files
with
193 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
DELIMITER | | ||
|
||
|
||
ALTER TABLE line_data_mpr | ||
RENAME COLUMN climo_mean TO obs_climo_mean | | ||
RENAME COLUMN climo_stdev TO obs_climo_stdev | | ||
RENAME COLUMN climo_cdf TO obs_climo_cdf | | ||
ADD COLUMN fcst_climo_mean DOUBLE | | ||
ADD COLUMN fcst_climo_stdev DOUBLE | | ||
|
||
ALTER TABLE line_data_orank | ||
RENAME COLUMN climo_mean TO obs_climo_mean | | ||
RENAME COLUMN climo_stdev TO obs_climo_stdev | | ||
ADD COLUMN fcst_climo_mean DOUBLE | | ||
ADD COLUMN fcst_climo_stdev DOUBLE | | ||
| | ||
|
||
DELIMITER ; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,7 +1,7 @@ | ||
hostname: 'localhost' | ||
port: 3306 | ||
username: 'mvadmin' | ||
password: '160GiltVa0D5M' | ||
password: 'passwordgoeshere' | ||
dbname: 'mv_load_test' | ||
output_dir: './output' | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Binary file not shown.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
hostname: 'localhost' | ||
port: 3306 | ||
username: 'dbusername' | ||
password: 'dbpassword' | ||
dbname: 'mv_mpr_orank' | ||
output_dir: './output' | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,147 @@ | ||
import pytest | ||
import pymysql | ||
import yaml | ||
from dataclasses import make_dataclass | ||
|
||
####################################################################### | ||
# These tests can only be run on the host where the database is running. | ||
# Pre-condition: | ||
# The data in the accompanying data directory ./Data, should | ||
# already be loaded in the database using the corresponding | ||
# schema: mv_mysql.sql and the appropriate xml specification file. | ||
# This is to avoid having the password visible in the test code. | ||
# | ||
|
||
|
||
# Make sure the database name matches with the one you created on the database host | ||
CONST_LOAD_DB_CMD = "use mv_mpr_orank" | ||
TEST_DB = "mv_mpr_orank" | ||
|
||
@pytest.fixture | ||
def setup_db(): | ||
""" | ||
Read in the config file to retrieve the database login information. | ||
""" | ||
config_file = 'test_loading.yaml' | ||
with open(config_file, 'r') as stream: | ||
try: | ||
parms: dict = yaml.load(stream, Loader=yaml.FullLoader) | ||
# pathlib.Path(parms['output_dir']).mkdir(parents=True, exist_ok=True) | ||
except yaml.YAMLError as exc: | ||
print(exc) | ||
|
||
# Create a dataclass of the database information | ||
DBS = make_dataclass("DBS", ["hostname", "port", "username", "password", "dbname"]) | ||
db_settings = DBS(parms['hostname'], parms['port'], parms['username'], parms['password'], parms['dbname']) | ||
|
||
# Return the db connection object | ||
conn = pymysql.connect( | ||
host=db_settings.hostname, | ||
port=db_settings.port, | ||
user=db_settings.username, | ||
password=db_settings.password, | ||
db=db_settings.dbname, | ||
charset='utf8mb4' | ||
) | ||
# settings (hostname, username, etc.) | ||
yield conn | ||
|
||
def test_db_created(setup_db): | ||
''' | ||
Verify that the mv_mpr_orank database was created | ||
Args: | ||
setup_db: db connection object | ||
Returns: None | ||
''' | ||
|
||
# connect to the database and verify the MPR and ORANK tables exist | ||
try: | ||
with setup_db.cursor() as cursor: | ||
# Check that the line_data_mpr and line_data_orank | ||
# tables were created | ||
cursor.execute(CONST_LOAD_DB_CMD) | ||
check_db_exists_query = "show databases;" | ||
cursor.execute(check_db_exists_query) | ||
|
||
|
||
# Get all rows | ||
rows = cursor.fetchall() | ||
list_of_rows = [r[0] for r in rows] | ||
assert TEST_DB in list_of_rows | ||
|
||
|
||
finally: | ||
setup_db.close() | ||
|
||
def test_tables_created(setup_db): | ||
|
||
# connect to the database and verify the MPR and ORANK tables exist | ||
try: | ||
with setup_db.cursor() as cursor: | ||
# Check that the line_data_mpr and | ||
# line_data_orank tables were created | ||
cursor.execute(CONST_LOAD_DB_CMD) | ||
check_db_exists_query = "show tables;" | ||
cursor.execute(check_db_exists_query) | ||
|
||
|
||
# Get all rows | ||
rows = cursor.fetchall() | ||
list_of_rows = [r[0] for r in rows] | ||
print(f"list of rows: {list_of_rows}") | ||
assert 'line_data_mpr' in list_of_rows | ||
assert 'line_data_orank' in list_of_rows | ||
|
||
finally: | ||
setup_db.close() | ||
|
||
def test_mpr_columns(setup_db): | ||
# log into the database and verify the renamed columns are in the | ||
# list_data_mpr database table, the previous/replaced columns do NOT | ||
# exist, and the new columns exist. | ||
|
||
try: | ||
with setup_db.cursor() as cursor: | ||
cursor.execute(CONST_LOAD_DB_CMD) | ||
check_columns_exist = "desc line_data_mpr;" | ||
cursor.execute(check_columns_exist) | ||
|
||
# Get all rows | ||
rows = cursor.fetchall() | ||
list_of_rows = [r[0] for r in rows] | ||
print(f"list of rows:\n{list_of_rows}") | ||
assert 'obs_climo_mean' in list_of_rows | ||
assert 'obs_climo_stdev' in list_of_rows | ||
assert 'obs_climo_cdf' in list_of_rows | ||
assert 'fcst_climo_mean' in list_of_rows | ||
assert 'fcst_climo_stdev' in list_of_rows | ||
|
||
|
||
finally: | ||
setup_db.close() | ||
|
||
def test_orank_columns(setup_db): | ||
# log into the database and verify the renamed and new columns are in the | ||
# list_data_orank database table, and the previous/replaced columns no longer | ||
# exist. | ||
|
||
try: | ||
with setup_db.cursor() as cursor: | ||
cursor.execute(CONST_LOAD_DB_CMD) | ||
check_columns_exist = "desc line_data_orank;" | ||
cursor.execute(check_columns_exist) | ||
|
||
# Get all rows | ||
rows = cursor.fetchall() | ||
list_of_rows = [r[0] for r in rows] | ||
assert 'obs_climo_mean' in list_of_rows | ||
assert 'obs_climo_stdev' in list_of_rows | ||
assert 'fcst_climo_mean' in list_of_rows | ||
assert 'fcst_climo_stdev' in list_of_rows | ||
|
||
finally: | ||
setup_db.close() | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters