From 6aa7fed2b3b0e25b5f6761791bc304aadddb61dc Mon Sep 17 00:00:00 2001 From: bikegeek Date: Sat, 21 Sep 2024 23:05:32 -0600 Subject: [PATCH 01/50] Internal issue #56 XML schema for validating to prevent DoS via large payload, recursive payloads, ill-formed XML specification file --- METdbLoad/ush/specification_schema.xsd | 118 +++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 METdbLoad/ush/specification_schema.xsd diff --git a/METdbLoad/ush/specification_schema.xsd b/METdbLoad/ush/specification_schema.xsd new file mode 100644 index 0000000..4abf554 --- /dev/null +++ b/METdbLoad/ush/specification_schema.xsd @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 4364b431756a14b2b218c3db5c4ea038905b31d5 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Sun, 22 Sep 2024 17:04:37 -0600 Subject: [PATCH 02/50] schema for validating the payload (recursive, excessively large) --- METdbLoad/ush/load_specification.xsd | 126 +++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 METdbLoad/ush/load_specification.xsd diff --git a/METdbLoad/ush/load_specification.xsd b/METdbLoad/ush/load_specification.xsd new file mode 100644 index 0000000..f0a4298 --- /dev/null +++ b/METdbLoad/ush/load_specification.xsd @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 1acd4964327020d97c8d79ec16cff8fe75f01d9f Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Sun, 22 Sep 2024 17:40:12 -0600 Subject: [PATCH 03/50] Delete METdbLoad/ush/specification_schema.xsd --- METdbLoad/ush/specification_schema.xsd | 118 ------------------------- 1 file changed, 118 deletions(-) delete mode 100644 METdbLoad/ush/specification_schema.xsd diff --git a/METdbLoad/ush/specification_schema.xsd b/METdbLoad/ush/specification_schema.xsd deleted file mode 100644 index 4abf554..0000000 --- a/METdbLoad/ush/specification_schema.xsd +++ /dev/null @@ -1,118 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From a892fc43dd891a9a05106ed3a8d5912547c21cc0 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 23 Sep 2024 17:19:24 -0600 Subject: [PATCH 04/50] internal issue #56 validating payload using XML schema --- METdbLoad/ush/read_load_xml.py | 48 ++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/METdbLoad/ush/read_load_xml.py b/METdbLoad/ush/read_load_xml.py index a14eab2..0471184 100644 --- a/METdbLoad/ush/read_load_xml.py +++ b/METdbLoad/ush/read_load_xml.py @@ -17,11 +17,14 @@ import sys import os +import datetime +import io from pathlib import Path import pandas as pd from lxml import etree import METreformat.util as util from METdbLoad.ush import constants as CN +from METdbLoad.test import utils as dbload_util class XmlLoadFile: @@ -68,7 +71,7 @@ def __init__(self, xmlfile, logger=None): if logger is None: log_filename = os.path.join(os.getcwd(), __name__ + "_log.txt") - self.logger = util.get_common_logger('DEBUG', log_filename) + self.logger = util.get_common_logger('INFO', log_filename) else: self.logger = logger @@ -85,7 +88,16 @@ def read_xml(self): if not Path(self.xmlfilename).is_file(): sys.exit("*** XML file " + self.xmlfilename + " can not be found!") - # parse the XML file + # Validate the XML file + self.logger.info(f"Validating the {self.xmlfilename} against the {dbload_util.LOAD_SPECIFICATION_SCHEMA}") + + if self.validate_xml() is False: + msg = ( + f"{self.xmlfilename} is not well-formed and may contain a recursive payload or an excessively large payload") + self.logger.error(msg) + print(f"{msg}") + raise ValueError(msg) + self.logger.info('Reading XML Load file') parser = etree.XMLParser(remove_comments=True, resolve_entities=False) tree = etree.parse(self.xmlfilename, parser=parser) @@ -162,6 +174,38 @@ def read_xml(self): self.logger.debug("[--- End read_xml ---]") + def validate_xml(self): + """ + Validate the XML specification file against the XML schema, which validates the payload, checking + for excessive payload size and recursive payloads. + + Args: None + + Returns: True if valid, False otherwise + """ + # Load the schema file, which resides in the same directory as this module + # self.logger.info(f"validating against schema: {os.path.dirname(os.path.abspath(__file__))}") + start = datetime.datetime.now() + self.logger.info(f"Validating against schema: {dbload_util.LOAD_SPECIFICATION_SCHEMA}") + xsd_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "load_specification.xsd") + with open(xsd_file, 'rb') as schema_file: + xmlschema_doc = etree.parse(schema_file) + xmlschema = etree.XMLSchema(xmlschema_doc) + + xml_document = etree.parse(self.xmlfilename) + # Validate the XML document against the schema + is_valid = xmlschema.validate(xml_document) + + total_time = datetime.datetime.now() - start + self.logger.info(f"Validation complete, took {total_time} seconds") + if is_valid: + self.logger.info(f"{self.xmlfilename} is valid") + + return True + else: + self.logger.info(f"{self.xmlfilename} NOT VALID") + return False + def read_file_info(self, root): """! Gather info on file template, fill-in values, and dates Returns: From eb41af1e48e8b9fad1b54e64c716393caa09b4cf Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 23 Sep 2024 17:23:26 -0600 Subject: [PATCH 05/50] internal issue #56 invalid XML spec files used to test XML validation --- METdbLoad/test/test_recursive_payload.xml | 47 +++++++++++++++++++++++ METdbLoad/test/test_size_payload.xml | 32 +++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 METdbLoad/test/test_recursive_payload.xml create mode 100644 METdbLoad/test/test_size_payload.xml diff --git a/METdbLoad/test/test_recursive_payload.xml b/METdbLoad/test/test_recursive_payload.xml new file mode 100644 index 0000000..7b64d2d --- /dev/null +++ b/METdbLoad/test/test_recursive_payload.xml @@ -0,0 +1,47 @@ + + + mariadb + localhost:3306 + mv_test + root + root_password + + + /METdataio/METreformat/test/data/point_stat + + true + 1 + True + false + false + false + false + true + true + true + true + true + + + + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + + + Testing + testing DB load + diff --git a/METdbLoad/test/test_size_payload.xml b/METdbLoad/test/test_size_payload.xml new file mode 100644 index 0000000..9b52887 --- /dev/null +++ b/METdbLoad/test/test_size_payload.xml @@ -0,0 +1,32 @@ + + + mariadb + + localhost-abcdefghijklmnopqrstuvwxyz-zyxwvutsrqponmlkjihgfedcba_999_123_rst_puytrewq:3306 + mv_test + root + root_password + + + /METdataio/METreformat/test/data/point_stat + + true + 1 + True + false + false + false + false + true + true + true + true + true + + + point_stat + + + Testing + testing DB load + From 5838ba9e33cce57c2e0e06571eff620250beddb1 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 23 Sep 2024 17:24:26 -0600 Subject: [PATCH 06/50] internal issue #56 add the location of the XML schema file to be used in validating the XML specification file --- METdbLoad/test/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/METdbLoad/test/utils.py b/METdbLoad/test/utils.py index d587021..f03e2e4 100644 --- a/METdbLoad/test/utils.py +++ b/METdbLoad/test/utils.py @@ -1,11 +1,14 @@ from pathlib import Path from argparse import Namespace +import os def abs_path(rel_path): """Turn a relative path into abs path""" return str(Path(str(Path(__file__).parents[2])) / rel_path) +# XML Schema for the load specification XML +LOAD_SPECIFICATION_SCHEMA = os.path.join(abs_path("METdbLoad/ush/"), "load_specification.xsd") # Use data from METreformat where available ENSEMBLE_STAT_DATA_DIR = abs_path("METreformat/test/data/ensemble_stat") From 40e53b48614d21001fdc2d289268c101285475cc Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 23 Sep 2024 17:25:11 -0600 Subject: [PATCH 07/50] internal issue #56 tests added to verify validation code is providing expected results --- METdbLoad/test/test_xml.py | 47 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index c303c89..f95b7ca 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -1,6 +1,13 @@ #!/usr/bin/env python3 + +import os +import pytest + """Test reading XML file.""" +# Location of the XML specification files that are used to test XML validation +TEST_XML_SPECIFICATION_FILEPATH = os.path.join(os.path.dirname(os.path.abspath(__file__))) + def test_loadflags(tmp_path, get_xml_loadfile): """Read various flags from XML file.""" XML_LOADFILE = get_xml_loadfile(tmp_path) @@ -37,3 +44,43 @@ def test_insertsize(tmp_path, get_xml_loadfile): """Read insert_size from XML file.""" XML_LOADFILE = get_xml_loadfile(tmp_path) assert XML_LOADFILE.insert_size == 1 + +def test_validation_recursive_payload(get_specified_xml_loadfile): + """ + Test validation against attempted recursive payload, ValueError should be raised for + the test_recursive_payload.xml XML specification file. + """ + # Get the XML specification file that has a recursive payload + xml_spec_filename = "test_recursive_payload.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + with pytest.raises(ValueError): + xml_load_file_obj.read_xml() + +def test_validation_large_payload(get_specified_xml_loadfile): + """ + Test validation against attempted "large" payload, ValueError should be raised for + the test_size_payload.xml XML specification file. + """ + # Get the XML specification file that has a recursive payload + + xml_spec_filename = "test_size_payload.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + with pytest.raises(ValueError): + xml_load_file_obj.read_xml() + +def test_validation_valid_xml(get_specified_xml_loadfile): + """ + Test validation against valid/well-formed XML specification file. + ValueError should be NOT be raised for + the test_size_payload.xml XML specification file. + """ + # Get the XML specification file that is well-formed/valid + + xml_spec_filename = "test_load_specification.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + try: + xml_load_file_obj.read_xml() + except ValueError: + msg = f"Unexpected ValueError when validating {os.path.join(TEST_XML_SPECIFICATION_FILEPATH,xml_spec_filename)}" + pytest.fail(msg) + From 5771ded470365bf25e29a9322ac8e843b1492db8 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 23 Sep 2024 17:28:52 -0600 Subject: [PATCH 08/50] internal issue#56 added new fixture used in testing XML validation code --- METdbLoad/conftest.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/METdbLoad/conftest.py b/METdbLoad/conftest.py index b1c1b38..638665f 100644 --- a/METdbLoad/conftest.py +++ b/METdbLoad/conftest.py @@ -10,7 +10,7 @@ get_xml_test_file, POINT_STAT_DATA_DIR, ) - +from read_load_xml import XmlLoadFile # add METdataio directory to path so packages can be found TOP_DIR = str(Path(__file__).parents[1]) @@ -129,4 +129,24 @@ def load_and_read_xml( @pytest.fixture def mock_logger(): - return MagicMock() \ No newline at end of file + return MagicMock() + +@pytest.fixture +def get_specified_xml_loadfile( ) -> XmlLoadFile: + """ + Retrieve the specified XML load specification filee. This is useful for using different XML + specification file for validating against recursive payloads, large payloads, etc. + + Args: + xml_filename: The name of the XML file of interest + Returns: + XML_LOAD_FILE: The XmlLoadFile instance corresponding to the XML specification file specified by path + and filename + + """ + def get_xml_spec_file(xml_path:str, xml_filename:str): + full_xml_filename = os.path.join(xml_path, xml_filename) + XML_LOAD_FILE = XmlLoadFile(full_xml_filename) + + return XML_LOAD_FILE + return get_xml_spec_file From 88b93674284ede313c22ac7be8027c248ad091d0 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 23 Sep 2024 17:33:02 -0600 Subject: [PATCH 09/50] fix import for read_load_xml module --- METdbLoad/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/conftest.py b/METdbLoad/conftest.py index 638665f..2b98d93 100644 --- a/METdbLoad/conftest.py +++ b/METdbLoad/conftest.py @@ -10,7 +10,7 @@ get_xml_test_file, POINT_STAT_DATA_DIR, ) -from read_load_xml import XmlLoadFile +from METdbLoad.ush.read_load_xml import XmlLoadFile # add METdataio directory to path so packages can be found TOP_DIR = str(Path(__file__).parents[1]) From 41de8b98467cb4542b51c47ba21d3be7b4ff4253 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Mon, 23 Sep 2024 17:47:04 -0600 Subject: [PATCH 10/50] removed extraneous ',' in import --- METdbLoad/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/conftest.py b/METdbLoad/conftest.py index 2b98d93..81a42fe 100644 --- a/METdbLoad/conftest.py +++ b/METdbLoad/conftest.py @@ -8,7 +8,7 @@ from METdataio.METdbLoad.ush.run_sql import RunSql from METdataio.METdbLoad.test.utils import ( get_xml_test_file, - POINT_STAT_DATA_DIR, + POINT_STAT_DATA_DIR ) from METdbLoad.ush.read_load_xml import XmlLoadFile From addb8f3a123ac0fcc6d5966afd207cef2904dfef Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 17:38:34 -0600 Subject: [PATCH 11/50] Working version but still needs to check for recursive payloads for some elements --- METdbLoad/ush/load_specification_schema.xsd | 149 ++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 METdbLoad/ush/load_specification_schema.xsd diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd new file mode 100644 index 0000000..76f9aee --- /dev/null +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -0,0 +1,149 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 23986aa1bc567df55688b28b2c23907d672a76c0 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 17:39:44 -0600 Subject: [PATCH 12/50] Valid XML that is used for real-world data and is valid XML --- METdbLoad/test/full_example.xml | 59 +++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 METdbLoad/test/full_example.xml diff --git a/METdbLoad/test/full_example.xml b/METdbLoad/test/full_example.xml new file mode 100644 index 0000000..b143404 --- /dev/null +++ b/METdbLoad/test/full_example.xml @@ -0,0 +1,59 @@ + + + mohawk.rap.ucar.edu:3306 + mv_rtps_href_spring_2022 + mvuser + mvuser + + + + 2022050100 + 2022051200 + 86400 + yyyyMMddHH + + + false + 1 + False + false + FALSE + true + True + false + false + Regional Ensemble + + /var/autofs/mnt/mandan_d2/projects/RRFS/prototype/met_out/{config}/{fcst_init}/{mem}/metprd/{met_out}/ + + + + + + + + HREF_lag_offset + RTPS + + + mem01 + mem02 + mem03 + mem04 + mem05 + mem06 + mem07 + mem08 + mem09 + mem10 + + + grid_stat_cmn + point_stat_cmn + + + + true + Load HREF and RTPS data for Spring 2022. + + From e419dc70eaaae02d3d118eb16622420e266d8ac2 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 17:41:17 -0600 Subject: [PATCH 13/50] Change the name of the XML schema file --- METdbLoad/test/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/METdbLoad/test/utils.py b/METdbLoad/test/utils.py index f03e2e4..40a1009 100644 --- a/METdbLoad/test/utils.py +++ b/METdbLoad/test/utils.py @@ -8,7 +8,7 @@ def abs_path(rel_path): return str(Path(str(Path(__file__).parents[2])) / rel_path) # XML Schema for the load specification XML -LOAD_SPECIFICATION_SCHEMA = os.path.join(abs_path("METdbLoad/ush/"), "load_specification.xsd") +LOAD_SPECIFICATION_SCHEMA = os.path.join(abs_path("METdbLoad/ush/"), "load_specification_schema.xsd") # Use data from METreformat where available ENSEMBLE_STAT_DATA_DIR = abs_path("METreformat/test/data/ensemble_stat") @@ -102,6 +102,7 @@ def get_xml_test_file(tmp_path, met_data_dir, met_tool, load_flags={}, local_inf xml_path = tmp_path / "test_load_specification.xml" with open(xml_path, "w") as text_file: text_file.write(populate_xml_load_spec(met_data_dir, met_tool, load_flags, local_infile)) + return xml_path From 2dbb111d4827cd1ecdd17863c173334b83201989 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 17:42:31 -0600 Subject: [PATCH 14/50] Use the full_example.xml file instead of the test_load_specification.xml for testing against a valid XML file --- METdbLoad/test/test_xml.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index f95b7ca..1455301 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -3,6 +3,9 @@ import os import pytest +import utils +from read_load_xml import XmlLoadFile + """Test reading XML file.""" # Location of the XML specification files that are used to test XML validation @@ -48,7 +51,7 @@ def test_insertsize(tmp_path, get_xml_loadfile): def test_validation_recursive_payload(get_specified_xml_loadfile): """ Test validation against attempted recursive payload, ValueError should be raised for - the test_recursive_payload.xml XML specification file. + the test_recursive_payload.xml XML-specification file. """ # Get the XML specification file that has a recursive payload xml_spec_filename = "test_recursive_payload.xml" @@ -62,21 +65,21 @@ def test_validation_large_payload(get_specified_xml_loadfile): the test_size_payload.xml XML specification file. """ # Get the XML specification file that has a recursive payload - xml_spec_filename = "test_size_payload.xml" xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) with pytest.raises(ValueError): xml_load_file_obj.read_xml() + def test_validation_valid_xml(get_specified_xml_loadfile): """ - Test validation against valid/well-formed XML specification file. + Test validation against a real-world, valid XML specification file. ValueError should be NOT be raised for - the test_size_payload.xml XML specification file. + the full_example.xml specification file which has been used on real data. """ - # Get the XML specification file that is well-formed/valid + # Get the XML specification file that has a recursive payload - xml_spec_filename = "test_load_specification.xml" + xml_spec_filename = "full_example.xml" xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) try: xml_load_file_obj.read_xml() From 422daaea506f6c80a729447195129e8942fb919b Mon Sep 17 00:00:00 2001 From: bikegeek <3753118+bikegeek@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:46:00 -0600 Subject: [PATCH 15/50] Delete METdbLoad/ush/load_specification.xsd --- METdbLoad/ush/load_specification.xsd | 126 --------------------------- 1 file changed, 126 deletions(-) delete mode 100644 METdbLoad/ush/load_specification.xsd diff --git a/METdbLoad/ush/load_specification.xsd b/METdbLoad/ush/load_specification.xsd deleted file mode 100644 index f0a4298..0000000 --- a/METdbLoad/ush/load_specification.xsd +++ /dev/null @@ -1,126 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 625b5ce32d6e627b907b5e98eff01b93c2a76ae0 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 18:03:59 -0600 Subject: [PATCH 16/50] Test for recursive payload in load_val fields --- .../test/test_recursive_load_val_fields.xml | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 METdbLoad/test/test_recursive_load_val_fields.xml diff --git a/METdbLoad/test/test_recursive_load_val_fields.xml b/METdbLoad/test/test_recursive_load_val_fields.xml new file mode 100644 index 0000000..9df37e3 --- /dev/null +++ b/METdbLoad/test/test_recursive_load_val_fields.xml @@ -0,0 +1,254 @@ + + + mariadb + localhost:3306 + mv_test + root + root_password + + + /METdataio/METreformat/test/data/point_stat + + true + 1 + True + false + false + false + false + true + true + true + true + true + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + point_stat + + + + + Testing + testing DB load + From 171d1c09ab1899cc51dcf4991e2f6f86dca8b9b2 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 18:06:50 -0600 Subject: [PATCH 17/50] Added test for recursion under the load_val complex type --- METdbLoad/ush/read_load_xml.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/METdbLoad/ush/read_load_xml.py b/METdbLoad/ush/read_load_xml.py index 0471184..fb8cd6d 100644 --- a/METdbLoad/ush/read_load_xml.py +++ b/METdbLoad/ush/read_load_xml.py @@ -91,12 +91,16 @@ def read_xml(self): # Validate the XML file self.logger.info(f"Validating the {self.xmlfilename} against the {dbload_util.LOAD_SPECIFICATION_SCHEMA}") - if self.validate_xml() is False: + if self.validate_xml() is False: msg = ( - f"{self.xmlfilename} is not well-formed and may contain a recursive payload or an excessively large payload") + f"{self.xmlfilename} is not valid and may contain a recursive payload or an excessively large payload") self.logger.error(msg) print(f"{msg}") raise ValueError(msg) + else: + msg = (f"{self.xmlfilename} is valid ") + self.logger.info(msg) + print(f"{msg}") self.logger.info('Reading XML Load file') parser = etree.XMLParser(remove_comments=True, resolve_entities=False) @@ -187,7 +191,7 @@ def validate_xml(self): # self.logger.info(f"validating against schema: {os.path.dirname(os.path.abspath(__file__))}") start = datetime.datetime.now() self.logger.info(f"Validating against schema: {dbload_util.LOAD_SPECIFICATION_SCHEMA}") - xsd_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), "load_specification.xsd") + xsd_file = os.path.join(os.path.dirname(os.path.abspath(__file__)), dbload_util.LOAD_SPECIFICATION_SCHEMA) with open(xsd_file, 'rb') as schema_file: xmlschema_doc = etree.parse(schema_file) xmlschema = etree.XMLSchema(xmlschema_doc) @@ -200,10 +204,12 @@ def validate_xml(self): self.logger.info(f"Validation complete, took {total_time} seconds") if is_valid: self.logger.info(f"{self.xmlfilename} is valid") - + print(f"xml file {self.xmlfilename} is valid against {schema_file}") return True else: self.logger.info(f"{self.xmlfilename} NOT VALID") + for error in xmlschema.error_log: + print(f"{error.message} at line: {error.line} and column: {error.column}") return False def read_file_info(self, root): @@ -410,3 +416,9 @@ def filenames_from_template(self, folder_template, template_fills): sys.exit("*** Error found while expanding XML folder templates!") return file_list + +if __name__ == "__main__": + # xml_file = "/Users/minnawin/feature_internal_56_METdataio_validate_payloads/METdataio/METdbLoad/test/valid_specification.xml" + xml_file = "/Users/minnawin/feature_internal_56_METdataio_validate_payloads/METdataio/METdbLoad/test/full_example.xml" + xlf = XmlLoadFile(xml_file) + xlf.read_xml() \ No newline at end of file From 5a85a9a3592a49dbe18908deff8438a69e5d335d Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 18:07:49 -0600 Subject: [PATCH 18/50] Change values to prevent recursive payloads and remove defunct regex --- METdbLoad/ush/load_specification_schema.xsd | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index 76f9aee..e2605a6 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -49,7 +49,6 @@ - @@ -133,7 +132,7 @@ - + From 1687ed847530d7d9486134db8710a75a620cffc9 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 18:14:21 -0600 Subject: [PATCH 19/50] Remove unused imports, add test for recursion under the load_val fields --- METdbLoad/test/test_xml.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 1455301..799d441 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -3,9 +3,6 @@ import os import pytest -import utils -from read_load_xml import XmlLoadFile - """Test reading XML file.""" # Location of the XML specification files that are used to test XML validation @@ -59,6 +56,19 @@ def test_validation_recursive_payload(get_specified_xml_loadfile): with pytest.raises(ValueError): xml_load_file_obj.read_xml() + +def test_validation_recursive_load_val(get_specified_xml_loadfile): + """ + Test validation against attempted recursive payload (multiple fields under load_val), + ValueError should be raised for + the test_recursive_payload.xml XML-specification file. + """ + # Get the XML specification file that has a recursive payload + xml_spec_filename = "test_recursive_load_val_fields.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + with pytest.raises(ValueError): + xml_load_file_obj.read_xml() + def test_validation_large_payload(get_specified_xml_loadfile): """ Test validation against attempted "large" payload, ValueError should be raised for From e0ab401de213ae975cd0123a9ce386573b5a41c3 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 18:15:17 -0600 Subject: [PATCH 20/50] Add some extra elements --- METdbLoad/test/test_load_specification.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/METdbLoad/test/test_load_specification.xml b/METdbLoad/test/test_load_specification.xml index 8556b16..326dcec 100644 --- a/METdbLoad/test/test_load_specification.xml +++ b/METdbLoad/test/test_load_specification.xml @@ -10,6 +10,7 @@ /METdataio/METreformat/test/data/point_stat true 1 + false true false false @@ -23,6 +24,11 @@ point_stat + grid_stat + ensemble_stat + wavelet_stat + mode + stat_analysis Testing From b1ad924c2c6b9e4d09e27c3671a1b0ff40f1028c Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 18:15:49 -0600 Subject: [PATCH 21/50] Add more recursive elements to trigger ValueError --- METdbLoad/test/test_recursive_payload.xml | 54 +++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/METdbLoad/test/test_recursive_payload.xml b/METdbLoad/test/test_recursive_payload.xml index 7b64d2d..80f9e88 100644 --- a/METdbLoad/test/test_recursive_payload.xml +++ b/METdbLoad/test/test_recursive_payload.xml @@ -40,6 +40,60 @@ point_stat point_stat point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + wavelet_stat + mode + mode + mode + mode + mode + mode + mode + mode + mode + mode + mode + mode + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat + point_stat Testing From 665b37df2a4613333a122ace08f6014cf67798c2 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 19:10:56 -0600 Subject: [PATCH 22/50] skip testing the recursion in load_val --- METdbLoad/test/test_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 799d441..81460f5 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -56,7 +56,7 @@ def test_validation_recursive_payload(get_specified_xml_loadfile): with pytest.raises(ValueError): xml_load_file_obj.read_xml() - +@pytest.skip() def test_validation_recursive_load_val(get_specified_xml_loadfile): """ Test validation against attempted recursive payload (multiple fields under load_val), From c0c28518bde516fb7c8f84a6547730fdffa830cb Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 19:11:17 -0600 Subject: [PATCH 23/50] Remove limit to number of load_val elements --- METdbLoad/ush/load_specification_schema.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index e2605a6..1797b86 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -120,7 +120,7 @@ - + From d85c1281d1ceeec8c3aae934ec59d5636c3284e2 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 19:12:23 -0600 Subject: [PATCH 24/50] Fixed incorrect skip syntax --- METdbLoad/test/test_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 81460f5..449322c 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -56,7 +56,7 @@ def test_validation_recursive_payload(get_specified_xml_loadfile): with pytest.raises(ValueError): xml_load_file_obj.read_xml() -@pytest.skip() +@pytest.mark.skip() def test_validation_recursive_load_val(get_specified_xml_loadfile): """ Test validation against attempted recursive payload (multiple fields under load_val), From 9fec829850687d03cd761f7579f1fae6359c25a9 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 25 Sep 2024 19:16:15 -0600 Subject: [PATCH 25/50] Reinstate the maxOccurs and minOccurs for the field --- METdbLoad/ush/load_specification_schema.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index 1797b86..95100db 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -120,7 +120,7 @@ - + @@ -132,7 +132,7 @@ - + From 71c559c458d1f66c477971c438bf6be2380c802a Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 27 Sep 2024 19:23:44 -0600 Subject: [PATCH 26/50] Update temporary XML spec file to match load_specification_schema.xsd --- METdbLoad/test/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/METdbLoad/test/utils.py b/METdbLoad/test/utils.py index 40a1009..8e5360b 100644 --- a/METdbLoad/test/utils.py +++ b/METdbLoad/test/utils.py @@ -73,17 +73,17 @@ def populate_xml_load_spec(met_data_dir, met_tool, load_flags=DEFAULT_LOAD_FLAGS {local_infile} - {met_data_dir} true 1 {flags} + Testing + testing with pytest + {met_data_dir} {met_tool} - Testing - testing with pytest """ From bfa1d770c0d6ee08088b6ef61a67219ceb1f7387 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 27 Sep 2024 19:26:15 -0600 Subject: [PATCH 27/50] Config file for testing recursive payload in the fields element --- .../test/test_recursive_payload_fields.xml | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 METdbLoad/test/test_recursive_payload_fields.xml diff --git a/METdbLoad/test/test_recursive_payload_fields.xml b/METdbLoad/test/test_recursive_payload_fields.xml new file mode 100644 index 0000000..35b7004 --- /dev/null +++ b/METdbLoad/test/test_recursive_payload_fields.xml @@ -0,0 +1,62 @@ + + + mariadb + localhost:3306 + mv_test + root + root_password + + + true + 1 + True + false + false + false + false + true + true + true + true + true + Testing + testing DB load + + /METdataio/METreformat/test/data/{config}/{mem}/{met_out}/{text}/{blah} + + + HREF_lag_offset + RTPS + + + mem01 + mem02 + mem03 + mem04 + mem05 + mem06 + mem07 + mem08 + mem09 + mem10 + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + + + From 1186e5be845a91c45f81d8e974d2d67941578203 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 27 Sep 2024 19:26:50 -0600 Subject: [PATCH 28/50] Work-in-progress. Recursive payloads checked for some elements but no checking for large payloads --- METdbLoad/test/test_load_specification.xml | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/METdbLoad/test/test_load_specification.xml b/METdbLoad/test/test_load_specification.xml index 326dcec..8b8f49f 100644 --- a/METdbLoad/test/test_load_specification.xml +++ b/METdbLoad/test/test_load_specification.xml @@ -7,10 +7,9 @@ root_password - /METdataio/METreformat/test/data/point_stat + true 1 - false true false false @@ -21,16 +20,13 @@ true true true + false + Testing + testing DB load + /METdataio/METreformat/test/data/{met_tool} - point_stat - grid_stat - ensemble_stat - wavelet_stat - mode - stat_analysis + mode - Testing - testing DB load - + From 7368a9debb0c715438426875947c3611b336a2d1 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 27 Sep 2024 19:27:31 -0600 Subject: [PATCH 29/50] Added test for recursive payload for fields (in addition to test for recursive val elements) --- METdbLoad/test/test_xml.py | 62 +++++++++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 449322c..a3d1fd7 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -3,6 +3,8 @@ import os import pytest +import utils + """Test reading XML file.""" # Location of the XML specification files that are used to test XML validation @@ -45,18 +47,32 @@ def test_insertsize(tmp_path, get_xml_loadfile): XML_LOADFILE = get_xml_loadfile(tmp_path) assert XML_LOADFILE.insert_size == 1 -def test_validation_recursive_payload(get_specified_xml_loadfile): +# @pytest.mark.skip +def test_validation_recursive_payload_fields(get_specified_xml_loadfile): """ Test validation against attempted recursive payload, ValueError should be raised for the test_recursive_payload.xml XML-specification file. """ # Get the XML specification file that has a recursive payload - xml_spec_filename = "test_recursive_payload.xml" + xml_spec_filename = "test_recursive_payload_fields.xml" xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) with pytest.raises(ValueError): xml_load_file_obj.read_xml() -@pytest.mark.skip() + +def test_validation_recursive_payload_vals(get_specified_xml_loadfile): + """ + Test validation against attempted recursive payload, ValueError should be raised for + the test_recursive_payload.xml XML-specification file. + """ + # Get the XML specification file that has a recursive payload + xml_spec_filename = "test_recursive_payload_vals.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + with pytest.raises(ValueError): + xml_load_file_obj.read_xml() + + +# @pytest.mark.skip() def test_validation_recursive_load_val(get_specified_xml_loadfile): """ Test validation against attempted recursive payload (multiple fields under load_val), @@ -81,9 +97,27 @@ def test_validation_large_payload(get_specified_xml_loadfile): xml_load_file_obj.read_xml() -def test_validation_valid_xml(get_specified_xml_loadfile): +def test_validation_simple_xml(get_specified_xml_loadfile): """ - Test validation against a real-world, valid XML specification file. + Test validation against a simple, valid XML specification file. + ValueError should be NOT be raised for + the full_example.xml specification file which has been used on real data. + """ + # Get the XML specification file that has a recursive payload + + # xml_spec_filename = "full_example.xml" + # xml_spec_filename = "modified_example.xml" + xml_spec_filename = "test_load_specification.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + try: + xml_load_file_obj.read_xml() + except ValueError: + msg = f"Unexpected ValueError when validating {os.path.join(TEST_XML_SPECIFICATION_FILEPATH,xml_spec_filename)}" + pytest.fail(msg) + +def test_validation_real_xml(get_specified_xml_loadfile): + """ + Test validation against an XML specification file that is in use by a project. ValueError should be NOT be raised for the full_example.xml specification file which has been used on real data. """ @@ -97,3 +131,21 @@ def test_validation_valid_xml(get_specified_xml_loadfile): msg = f"Unexpected ValueError when validating {os.path.join(TEST_XML_SPECIFICATION_FILEPATH,xml_spec_filename)}" pytest.fail(msg) +def test_tmp_xml(get_specified_xml_loadfile): + """ + Test validation against an XML specification file that was created as a fixture. + ValueError should be NOT be raised for + the full_example.xml specification file which has been used on real data. + """ + # Get the XML specification file that has a recursive payload + + # xml_spec_filename = "full_example.xml" + xml_spec_filename = "tmp.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + try: + xml_load_file_obj.read_xml() + except ValueError: + msg = (f"Unexpected ValueError when validating {os.path.join(TEST_XML_SPECIFICATION_FILEPATH,xml_spec_filename)} against " + f"schema {utils.LOAD_SPECIFICATION_SCHEMA}") + pytest.fail(msg) + From 3b0366a9e51e7dbfc9e57120e51502fec14f9ec7 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 1 Oct 2024 16:48:33 -0600 Subject: [PATCH 30/50] Update tests and test config files --- .../test/test_recursive_payload_fields.xml | 76 +++++++++- .../test/test_recursive_payload_vals.xml | 141 ++++++++++++++++++ METdbLoad/test/test_xml.py | 20 +-- 3 files changed, 217 insertions(+), 20 deletions(-) create mode 100644 METdbLoad/test/test_recursive_payload_vals.xml diff --git a/METdbLoad/test/test_recursive_payload_fields.xml b/METdbLoad/test/test_recursive_payload_fields.xml index 35b7004..4febe60 100644 --- a/METdbLoad/test/test_recursive_payload_fields.xml +++ b/METdbLoad/test/test_recursive_payload_fields.xml @@ -22,13 +22,17 @@ Testing testing DB load - /METdataio/METreformat/test/data/{config}/{mem}/{met_out}/{text}/{blah} + /METdataio/METreformat/test/data/{config}/{mem}/{met_out}/{text}/{blah}/{foo} - + HREF_lag_offset RTPS - + + HREF_lag_offset + RTPS + + mem01 mem02 mem03 @@ -44,7 +48,55 @@ grid_stat_cmn point_stat_cmn - + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + HREF_lag_offset + RTPS + + + mem01 + mem02 + mem03 + mem04 + mem05 + mem06 + mem07 + mem08 + mem09 + mem10 + + + grid_stat_cmn + point_stat_cmn + + grid_stat_cmn point_stat_cmn @@ -56,6 +108,22 @@ grid_stat_cmn point_stat_cmn + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + + + grid_stat_cmn + point_stat_cmn + diff --git a/METdbLoad/test/test_recursive_payload_vals.xml b/METdbLoad/test/test_recursive_payload_vals.xml new file mode 100644 index 0000000..a42334f --- /dev/null +++ b/METdbLoad/test/test_recursive_payload_vals.xml @@ -0,0 +1,141 @@ + + + mariadb + localhost:3306 + mv_test + root + root_password + + + true + 1 + True + false + false + false + false + true + true + true + true + true + Testing + testing DB load + + /METdataio/METreformat/test/data/{config}/{mem}/{met_out} + + + HREF_lag_offset + RTPS + + + mem01 + mem02 + mem03 + mem04 + mem05 + mem06 + mem07 + mem08 + mem09 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + mem10 + + + grid_stat_cmn + point_stat_cmn + + + + + diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index a3d1fd7..18a65e4 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -3,7 +3,7 @@ import os import pytest -import utils +import METdbLoad.test.utils as utils """Test reading XML file.""" @@ -51,7 +51,8 @@ def test_insertsize(tmp_path, get_xml_loadfile): def test_validation_recursive_payload_fields(get_specified_xml_loadfile): """ Test validation against attempted recursive payload, ValueError should be raised for - the test_recursive_payload.xml XML-specification file. + the test_recursive_payload_fields.xml XML-specification file because the max allowed number + of field elements is 5 and the test config file has 6. """ # Get the XML specification file that has a recursive payload xml_spec_filename = "test_recursive_payload_fields.xml" @@ -72,19 +73,6 @@ def test_validation_recursive_payload_vals(get_specified_xml_loadfile): xml_load_file_obj.read_xml() -# @pytest.mark.skip() -def test_validation_recursive_load_val(get_specified_xml_loadfile): - """ - Test validation against attempted recursive payload (multiple fields under load_val), - ValueError should be raised for - the test_recursive_payload.xml XML-specification file. - """ - # Get the XML specification file that has a recursive payload - xml_spec_filename = "test_recursive_load_val_fields.xml" - xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) - with pytest.raises(ValueError): - xml_load_file_obj.read_xml() - def test_validation_large_payload(get_specified_xml_loadfile): """ Test validation against attempted "large" payload, ValueError should be raised for @@ -123,7 +111,7 @@ def test_validation_real_xml(get_specified_xml_loadfile): """ # Get the XML specification file that has a recursive payload - xml_spec_filename = "full_example.xml" + xml_spec_filename = "modified_example.xml" xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) try: xml_load_file_obj.read_xml() From 2a329b0d527fcaae61d2db499764f66f0f59bf7e Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 1 Oct 2024 16:48:57 -0600 Subject: [PATCH 31/50] Additional test configuration files --- METdbLoad/test/modified_example.xml | 58 +++++++++++++++++++++++++++++ METdbLoad/test/tmp.xml | 33 ++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 METdbLoad/test/modified_example.xml create mode 100644 METdbLoad/test/tmp.xml diff --git a/METdbLoad/test/modified_example.xml b/METdbLoad/test/modified_example.xml new file mode 100644 index 0000000..4f0fa8d --- /dev/null +++ b/METdbLoad/test/modified_example.xml @@ -0,0 +1,58 @@ + + + mohawk.rap.ucar.edu:3306 + mv_rtps_href_spring_2022 + mvuser + mvuser + + + + 2022050100 + 2022051200 + 86400 + yyyyMMddHH + + + false + 1 + False + false + FALSE + true + True + false + false + Regional Ensemble + + /var/autofs/mnt/mandan_d2/projects/RRFS/prototype/met_out/{config}/{mem}/{fcst_init}/{met_out} + + + + HREF_lag_offset + RTPS + + + mem01 + mem02 + mem03 + mem04 + mem05 + mem06 + mem07 + mem08 + mem09 + mem10 + + + grid_stat_cmn + point_stat_cmn + + + + + + + true + Load HREF and RTPS data for Spring 2022. + + diff --git a/METdbLoad/test/tmp.xml b/METdbLoad/test/tmp.xml new file mode 100644 index 0000000..e1e3f4a --- /dev/null +++ b/METdbLoad/test/tmp.xml @@ -0,0 +1,33 @@ + + + mysql + localhost:3306 + mv_test + root + root_password + True + + + + true + 1 + true + false + false + false + false + true + true + true + true + true + Testing + testing with pytest + /Users/minnawin/feature_internal_56_METdataio_validate_payloads/METdataio/METreformat/test/data/{met_tool} + + + point_stat + grod_stat + + + \ No newline at end of file From a289b6ed1822fdd04d6e5fe6d589eff4a4e7b49d Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 1 Oct 2024 16:49:47 -0600 Subject: [PATCH 32/50] updated schema, now working --- METdbLoad/ush/load_specification_schema.xsd | 38 ++++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index 95100db..b7af40b 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -44,8 +44,8 @@ - - + + @@ -74,17 +74,17 @@ - + - + - + @@ -100,10 +100,9 @@ - - - - + + + @@ -113,17 +112,21 @@ - - + + + - + + + - + - + + @@ -132,9 +135,10 @@ - + + - + @@ -142,7 +146,7 @@ - + \ No newline at end of file From a77daa1a12334a8d1a40c53d9fb1f221f59600e0 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Tue, 1 Oct 2024 16:50:09 -0600 Subject: [PATCH 33/50] modified test configuration file --- METdbLoad/test/full_example.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/METdbLoad/test/full_example.xml b/METdbLoad/test/full_example.xml index b143404..9310127 100644 --- a/METdbLoad/test/full_example.xml +++ b/METdbLoad/test/full_example.xml @@ -29,7 +29,6 @@ - HREF_lag_offset From d4e7f53727f4630f2e09395caa74eef73c647488 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 2 Oct 2024 10:09:02 -0600 Subject: [PATCH 34/50] Allow '-' in regex for limited string type --- METdbLoad/ush/load_specification_schema.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index b7af40b..f95520c 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -66,7 +66,7 @@ - + From 48a31f7fcbd4301e0a38372ee27dd19b3abdf069 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 2 Oct 2024 10:09:37 -0600 Subject: [PATCH 35/50] Updated file so it is valid with respect to the schema --- METdbLoad/test/load_met_gha_prod.xml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/METdbLoad/test/load_met_gha_prod.xml b/METdbLoad/test/load_met_gha_prod.xml index 0298553..817473d 100644 --- a/METdbLoad/test/load_met_gha_prod.xml +++ b/METdbLoad/test/load_met_gha_prod.xml @@ -7,7 +7,7 @@ root - /home/runner/work/METdataio/METdataio/metdata/met_out/{met_tool} + @@ -24,7 +24,10 @@ true false true + METplus-Training + MET output generated by make test. + /home/runner/work/METdataio/METdataio/metdata/met_out/{met_tool} ensemble_stat @@ -36,7 +39,7 @@ - METplus-Training - MET output generated by make test. + + From 2a8ffed3b53d9a45be571ce8de112fd1e485896c Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 2 Oct 2024 10:10:08 -0600 Subject: [PATCH 36/50] Include testing one of the xml specification files used in testing two databases --- METdbLoad/test/test_xml.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 18a65e4..882288f 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -137,3 +137,20 @@ def test_tmp_xml(get_specified_xml_loadfile): f"schema {utils.LOAD_SPECIFICATION_SCHEMA}") pytest.fail(msg) +def test_db_xml(get_specified_xml_loadfile): + """ + Test validation against an XML specification file that is used in another test that compares two + databases. + ValueError should be NOT be raised for + the load_met_gha_prod specification file which has been used on real data. + """ + + xml_spec_filename = "load_met_gha_prod.xml" + xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) + try: + xml_load_file_obj.read_xml() + except ValueError: + msg = (f"Unexpected ValueError when validating {os.path.join(TEST_XML_SPECIFICATION_FILEPATH,xml_spec_filename)} against " + f"schema {utils.LOAD_SPECIFICATION_SCHEMA}") + pytest.fail(msg) + From cf5633560568ebbfa601d79753709d711f6ad9ed Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 2 Oct 2024 13:27:47 -0600 Subject: [PATCH 37/50] Reformat code for easier reading, update the load_met_gha_new.xml file to be valid --- .../test/test_recursive_load_val_fields.xml | 254 ------------------ METdbLoad/test/test_recursive_payload.xml | 101 ------- 2 files changed, 355 deletions(-) delete mode 100644 METdbLoad/test/test_recursive_load_val_fields.xml delete mode 100644 METdbLoad/test/test_recursive_payload.xml diff --git a/METdbLoad/test/test_recursive_load_val_fields.xml b/METdbLoad/test/test_recursive_load_val_fields.xml deleted file mode 100644 index 9df37e3..0000000 --- a/METdbLoad/test/test_recursive_load_val_fields.xml +++ /dev/null @@ -1,254 +0,0 @@ - - - mariadb - localhost:3306 - mv_test - root - root_password - - - /METdataio/METreformat/test/data/point_stat - - true - 1 - True - false - false - false - false - true - true - true - true - true - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - point_stat - - - - - Testing - testing DB load - diff --git a/METdbLoad/test/test_recursive_payload.xml b/METdbLoad/test/test_recursive_payload.xml deleted file mode 100644 index 80f9e88..0000000 --- a/METdbLoad/test/test_recursive_payload.xml +++ /dev/null @@ -1,101 +0,0 @@ - - - mariadb - localhost:3306 - mv_test - root - root_password - - - /METdataio/METreformat/test/data/point_stat - - true - 1 - True - false - false - false - false - true - true - true - true - true - - - - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - wavelet_stat - mode - mode - mode - mode - mode - mode - mode - mode - mode - mode - mode - mode - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - point_stat - - - Testing - testing DB load - From 94b6c50b908834f4ba884dc615d6ac9d11c12c0a Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 2 Oct 2024 13:28:32 -0600 Subject: [PATCH 38/50] include testing the load_met_gha_new.xml file --- METdbLoad/test/test_xml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 882288f..463b038 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -137,7 +137,8 @@ def test_tmp_xml(get_specified_xml_loadfile): f"schema {utils.LOAD_SPECIFICATION_SCHEMA}") pytest.fail(msg) -def test_db_xml(get_specified_xml_loadfile): +@pytest.mark.parametrize("xml_config", ["load_met_gha_prod.xml", "load_met_gha_new.xml"]) +def test_db_xml(get_specified_xml_loadfile, xml_config): """ Test validation against an XML specification file that is used in another test that compares two databases. @@ -145,7 +146,7 @@ def test_db_xml(get_specified_xml_loadfile): the load_met_gha_prod specification file which has been used on real data. """ - xml_spec_filename = "load_met_gha_prod.xml" + xml_spec_filename = xml_config xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) try: xml_load_file_obj.read_xml() From d3ee5b1be211ee0739c7ced42b6620377b775d40 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 2 Oct 2024 13:29:22 -0600 Subject: [PATCH 39/50] Updated: reformatted and updated to conform to schema --- METdbLoad/test/load_met_gha_new.xml | 79 ++++++++++++++-------------- METdbLoad/test/load_met_gha_prod.xml | 79 +++++++++++++--------------- 2 files changed, 78 insertions(+), 80 deletions(-) diff --git a/METdbLoad/test/load_met_gha_new.xml b/METdbLoad/test/load_met_gha_new.xml index 4700811..a1a5fd4 100644 --- a/METdbLoad/test/load_met_gha_new.xml +++ b/METdbLoad/test/load_met_gha_new.xml @@ -1,42 +1,43 @@ - - mysql - localhost:3306 - mv_ci_new - root - root - - - /home/runner/work/METdataio/METdataio/metdata/met_out/{met_tool} - - - - true - 1 - true - true - true - false - false - - true - true - true - false - true - - - - ensemble_stat - grid_stat - mode - point_stat - stat_analysis - wavelet_stat - - - - METplus-Training - MET output generated by make test. + + mysql + localhost:3306 + mv_ci_new + root + root + + + + + + true + 1 + true + true + true + false + false + + + true + true + true + false + true + METplus-Training + MET output generated by make test. + + /home/runner/work/METdataio/METdataio/metdata/met_out/{met_tool} + + + ensemble_stat + grid_stat + mode + point_stat + stat_analysis + wavelet_stat + + + diff --git a/METdbLoad/test/load_met_gha_prod.xml b/METdbLoad/test/load_met_gha_prod.xml index 817473d..1bcfb21 100644 --- a/METdbLoad/test/load_met_gha_prod.xml +++ b/METdbLoad/test/load_met_gha_prod.xml @@ -1,45 +1,42 @@ - - mysql - localhost:3306 - mv_ci_prod - root - root - - - - - - - true - 1 - true - true - true - false - false - - true - true - true - false - true - METplus-Training - MET output generated by make test. - - /home/runner/work/METdataio/METdataio/metdata/met_out/{met_tool} - - - ensemble_stat - grid_stat - mode - point_stat - stat_analysis - wavelet_stat - - - - + + mysql + localhost:3306 + mv_ci_prod + root + root + + + + + + true + 1 + true + true + true + false + false + + true + true + true + false + true + METplus-Training + MET output generated by make test. + + /home/runner/work/METdataio/METdataio/metdata/met_out/{met_tool} + + + ensemble_stat + grid_stat + mode + point_stat + stat_analysis + wavelet_stat + + From b05221789976ddb8c1bb0c82b5925f6210f066c0 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 3 Oct 2024 10:42:54 -0600 Subject: [PATCH 40/50] Added an extra date_list element --- METdbLoad/test/full_example.xml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/METdbLoad/test/full_example.xml b/METdbLoad/test/full_example.xml index 9310127..6151ee9 100644 --- a/METdbLoad/test/full_example.xml +++ b/METdbLoad/test/full_example.xml @@ -12,6 +12,12 @@ 86400 yyyyMMddHH + + 2022050100 + 2022051200 + 0600 + yyyyMMddHH + false 1 @@ -30,6 +36,9 @@ + + + HREF_lag_offset RTPS From 0194da71137233dbd48ce92bde7dd90395a28424 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 3 Oct 2024 10:43:21 -0600 Subject: [PATCH 41/50] added testing xml specification file with more than one date_list --- METdbLoad/test/test_xml.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 463b038..5e4b013 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -103,15 +103,15 @@ def test_validation_simple_xml(get_specified_xml_loadfile): msg = f"Unexpected ValueError when validating {os.path.join(TEST_XML_SPECIFICATION_FILEPATH,xml_spec_filename)}" pytest.fail(msg) -def test_validation_real_xml(get_specified_xml_loadfile): +@pytest.mark.parametrize("xmlconfig", ["modified_example.xml", "full_example.xml"]) +def test_validation_real_xml(get_specified_xml_loadfile, xmlconfig): """ Test validation against an XML specification file that is in use by a project. ValueError should be NOT be raised for - the full_example.xml specification file which has been used on real data. + the specification file that has been used on real data. """ - # Get the XML specification file that has a recursive payload - xml_spec_filename = "modified_example.xml" + xml_spec_filename = xmlconfig xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) try: xml_load_file_obj.read_xml() @@ -127,7 +127,6 @@ def test_tmp_xml(get_specified_xml_loadfile): """ # Get the XML specification file that has a recursive payload - # xml_spec_filename = "full_example.xml" xml_spec_filename = "tmp.xml" xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) try: From 0d19e5fbe7e6253363e6977bee3ff5cd6ecc71b5 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 3 Oct 2024 10:53:27 -0600 Subject: [PATCH 42/50] Clean up unnecessary comments --- METdbLoad/test/test_xml.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 5e4b013..9130311 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -91,10 +91,7 @@ def test_validation_simple_xml(get_specified_xml_loadfile): ValueError should be NOT be raised for the full_example.xml specification file which has been used on real data. """ - # Get the XML specification file that has a recursive payload - # xml_spec_filename = "full_example.xml" - # xml_spec_filename = "modified_example.xml" xml_spec_filename = "test_load_specification.xml" xml_load_file_obj = get_specified_xml_loadfile(TEST_XML_SPECIFICATION_FILEPATH, xml_spec_filename) try: From 14f5ce4049293df632d16568109056997a8a1b00 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Thu, 3 Oct 2024 12:43:28 -0600 Subject: [PATCH 43/50] Update number of date_list items --- METdbLoad/ush/load_specification_schema.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index f95520c..732a651 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -88,7 +88,7 @@ - + From 21b4af767fa720d18ca7f587ff97e8deb84aa781 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 4 Oct 2024 13:50:14 -0600 Subject: [PATCH 44/50] Explicitly set minLength and maxLength for hostname, db name, password, etc. --- METdbLoad/ush/load_specification_schema.xsd | 25 ++++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index 732a651..9eb65df 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -20,27 +20,36 @@ - + + + - + + + - + + + - + + + + @@ -49,7 +58,9 @@ - + + + @@ -66,7 +77,9 @@ - + + + From d3d604c6971d4e9a33584da591f1b28f5cda6099 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Sat, 5 Oct 2024 18:44:32 -0600 Subject: [PATCH 45/50] allow password to be string type and limit length of password --- METdbLoad/ush/load_specification_schema.xsd | 2 -- 1 file changed, 2 deletions(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index 9eb65df..009254e 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -46,10 +46,8 @@ - - From eb95a1535748659b7faa3d71dac06c93bfda29ec Mon Sep 17 00:00:00 2001 From: bikegeek Date: Sat, 5 Oct 2024 18:48:41 -0600 Subject: [PATCH 46/50] comment out mysql commands. ci-run-all-cases --- .github/workflows/compare_db.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compare_db.yaml b/.github/workflows/compare_db.yaml index d0a3e65..bae14b7 100644 --- a/.github/workflows/compare_db.yaml +++ b/.github/workflows/compare_db.yaml @@ -85,14 +85,14 @@ jobs: export PYTHONPATH=${GITHUB_WORKSPACE}/baseold cd ${GITHUB_WORKSPACE}/baseold/METdbLoad/ush python met_db_load.py ${GITHUB_WORKSPACE}/headnew/METdbLoad/test/load_met_gha_prod.xml - mysql -e 'SHOW TABLE STATUS WHERE `rows` > 0;' -uroot -proot mv_ci_prod + #mysql -e 'SHOW TABLE STATUS WHERE `rows` > 0;' -uroot -proot mv_ci_prod - name: run METdbload new shell: bash run: | export PYTHONPATH=${GITHUB_WORKSPACE}/headnew cd ${GITHUB_WORKSPACE}/headnew/METdbLoad/ush python met_db_load.py ${GITHUB_WORKSPACE}/headnew/METdbLoad/test/load_met_gha_new.xml - mysql -e 'SHOW TABLE STATUS WHERE `rows` > 0;' -uroot -proot mv_ci_new + #mysql -e 'SHOW TABLE STATUS WHERE `rows` > 0;' -uroot -proot mv_ci_new - name: run test_tables to compare tables in 2 databases shell: bash run: python ${GITHUB_WORKSPACE}/headnew/METdbLoad/test/test_tables.py From ac1dde83880c820bc3bb488586f9eb62a4ba64df Mon Sep 17 00:00:00 2001 From: bikegeek Date: Sat, 5 Oct 2024 18:56:47 -0600 Subject: [PATCH 47/50] Fix comment --- METdbLoad/test/test_xml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/METdbLoad/test/test_xml.py b/METdbLoad/test/test_xml.py index 9130311..4857def 100644 --- a/METdbLoad/test/test_xml.py +++ b/METdbLoad/test/test_xml.py @@ -139,7 +139,7 @@ def test_db_xml(get_specified_xml_loadfile, xml_config): Test validation against an XML specification file that is used in another test that compares two databases. ValueError should be NOT be raised for - the load_met_gha_prod specification file which has been used on real data. + the load_met_gha_prod/new specification files which are used on real data. """ xml_spec_filename = xml_config From d78459b54ed08a732794c0e9fb173926281b4ee6 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Fri, 11 Oct 2024 17:16:19 -0600 Subject: [PATCH 48/50] Remove main function with hard-coded paths. Only useful during development. --- METdbLoad/ush/read_load_xml.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/METdbLoad/ush/read_load_xml.py b/METdbLoad/ush/read_load_xml.py index 02c1d9c..ea716bf 100644 --- a/METdbLoad/ush/read_load_xml.py +++ b/METdbLoad/ush/read_load_xml.py @@ -460,9 +460,3 @@ def filenames_from_template(self, folder_template, template_fills): sys.exit("*** Error found while expanding XML folder templates!") return file_list - -if __name__ == "__main__": - # xml_file = "/Users/minnawin/feature_internal_56_METdataio_validate_payloads/METdataio/METdbLoad/test/valid_specification.xml" - xml_file = "/Users/minnawin/feature_internal_56_METdataio_validate_payloads/METdataio/METdbLoad/test/full_example.xml" - xlf = XmlLoadFile(xml_file) - xlf.read_xml() \ No newline at end of file From 14f6023e70c34124e484ae8df0c109ab6af8d81a Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 23 Oct 2024 14:49:34 -0600 Subject: [PATCH 49/50] Add support for line_type element --- METdbLoad/ush/load_specification_schema.xsd | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/METdbLoad/ush/load_specification_schema.xsd b/METdbLoad/ush/load_specification_schema.xsd index 009254e..fa8270c 100644 --- a/METdbLoad/ush/load_specification_schema.xsd +++ b/METdbLoad/ush/load_specification_schema.xsd @@ -81,6 +81,14 @@ + + + + + + + + @@ -136,7 +144,6 @@ - @@ -147,11 +154,17 @@ - + + + + + + + From 0d868e08e8472176392036e529d741cdd1b99af0 Mon Sep 17 00:00:00 2001 From: bikegeek Date: Wed, 23 Oct 2024 14:51:27 -0600 Subject: [PATCH 50/50] Added line_type element and some values for example xml specification --- METdbLoad/test/full_example.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/METdbLoad/test/full_example.xml b/METdbLoad/test/full_example.xml index 6151ee9..b393bfe 100644 --- a/METdbLoad/test/full_example.xml +++ b/METdbLoad/test/full_example.xml @@ -59,6 +59,12 @@ grid_stat_cmn point_stat_cmn + + CNT + ORANK + SEEPS_MPR + ME + true