From a464e1ac481be527d4f299e705d29d0d73a40a30 Mon Sep 17 00:00:00 2001 From: John-Sharples Date: Mon, 16 Sep 2024 16:12:45 +0000 Subject: [PATCH 1/6] 318: update tests/data for MTD 3d cluster --- .../mtd_CONV_TIME_20100517_010000V_3d_pair_cluster.txt | 5 +++++ METdbLoad/test/test_met_db_load.py | 9 ++++++++- METdbLoad/test/test_read_data_files.py | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 METdbLoad/test/data/mtd/mtd_CONV_TIME_20100517_010000V_3d_pair_cluster.txt diff --git a/METdbLoad/test/data/mtd/mtd_CONV_TIME_20100517_010000V_3d_pair_cluster.txt b/METdbLoad/test/data/mtd/mtd_CONV_TIME_20100517_010000V_3d_pair_cluster.txt new file mode 100644 index 0000000..ce9d856 --- /dev/null +++ b/METdbLoad/test/data/mtd/mtd_CONV_TIME_20100517_010000V_3d_pair_cluster.txt @@ -0,0 +1,5 @@ +VERSION MODEL DESC FCST_LEAD FCST_VALID OBS_LEAD OBS_VALID T_DELTA FCST_T_BEG FCST_T_END FCST_RAD FCST_THR OBS_T_BEG OBS_T_END OBS_RAD OBS_THR FCST_VAR FCST_UNITS FCST_LEV OBS_VAR OBS_UNITS OBS_LEV OBJECT_ID OBJECT_CAT SPACE_CENTROID_DIST TIME_CENTROID_DELTA AXIS_DIFF SPEED_DELTA DIRECTION_DIFF VOLUME_RATIO START_TIME_DELTA END_TIME_DELTA INTERSECTION_VOLUME DURATION_DIFF INTEREST +V12.0.0 FCST NA NA 20100517_010000 NA 20100517_010000 010000 -2 3 2 >=0.5 -1 1 2 >=0.5 APCP_01 kg/m^2 A01 APCP_01 kg/m^2 A01 CF001_CO001 CF001_CO001 6.89 -0.01 3.894 0.37 31.96 1.19 0 0 47061 0 -1.000 +V12.0.0 FCST NA NA 20100517_010000 NA 20100517_010000 010000 -2 3 2 >=0.5 -1 1 2 >=0.5 APCP_01 kg/m^2 A01 APCP_01 kg/m^2 A01 CF002_CO002 CF002_CO002 60.32 -1.06 5.214 1.07 3.46 0.75 0 0 0 0 -1.000 +V12.0.0 FCST NA NA 20100517_010000 NA 20100517_010000 010000 -2 3 2 >=0.5 -1 1 2 >=0.5 APCP_01 kg/m^2 A01 APCP_01 kg/m^2 A01 CF003_CO003 CF003_CO003 65.19 -0.67 62.974 -3.31 177.03 0.98 0 -1 7 -1 -1.000 +V12.0.0 FCST NA NA 20100517_010000 NA 20100517_010000 010000 -2 3 2 >=0.5 -1 1 2 >=0.5 APCP_01 kg/m^2 A01 APCP_01 kg/m^2 A01 CF004_CO004 CF004_CO004 4.72 0.84 2.870 -1.85 1.95 0.90 -2 0 1151 2 -1.000 diff --git a/METdbLoad/test/test_met_db_load.py b/METdbLoad/test/test_met_db_load.py index deb92a8..a4c3727 100644 --- a/METdbLoad/test/test_met_db_load.py +++ b/METdbLoad/test/test_met_db_load.py @@ -85,6 +85,8 @@ def assert_count_rows(cur, table, expected_count): { "mtd_2d_obj": 278, "mtd_3d_obj_single": 8, + "mtd_3d_obj_pair": 4, + "mtd_header": 24, }, ), ( @@ -128,9 +130,14 @@ def test_met_db_table_counts( expected_counts, ): + load_flags = { + "mode_header_db_check": "true", + "mtd_header_db_check": "true", + } + test_args = dict_to_args( { - "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool)), + "xmlfile": str(get_xml_test_file(tmp_path, met_data_dir, met_tool, load_flags=load_flags)), "index": "true", "tmpdir": [str(tmp_path)], "loglevel": None, diff --git a/METdbLoad/test/test_read_data_files.py b/METdbLoad/test/test_read_data_files.py index 4daa278..ab9825b 100644 --- a/METdbLoad/test/test_read_data_files.py +++ b/METdbLoad/test/test_read_data_files.py @@ -42,7 +42,7 @@ def test_mtd_loads(tmp_path, get_xml_loadfile): ) # number of files - assert len(XML_LOADFILE.load_files) == 2 + assert len(XML_LOADFILE.load_files) == 3 # number of lines of data assert FILE_DATA.mtd_2d_data.shape == (278, 43) assert FILE_DATA.mtd_3d_single_data.shape == (8, 48) From 76e1072ef8e4d3be391956a46c416ec01d59c6f6 Mon Sep 17 00:00:00 2001 From: John-Sharples Date: Tue, 17 Sep 2024 18:56:28 +0000 Subject: [PATCH 2/6] Add pyproject.toml and test config --- .coveragerc | 2 -- .github/workflows/sonarqube.yaml | 14 +++++++++++--- .github/workflows/unit_tests.yaml | 4 +--- pyproject.toml | 20 ++++++++++++++++++++ 4 files changed, 32 insertions(+), 8 deletions(-) delete mode 100644 .coveragerc create mode 100644 pyproject.toml diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 4edd7b1..0000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -relative_files = True diff --git a/.github/workflows/sonarqube.yaml b/.github/workflows/sonarqube.yaml index 759cce9..96cbae4 100644 --- a/.github/workflows/sonarqube.yaml +++ b/.github/workflows/sonarqube.yaml @@ -40,7 +40,16 @@ jobs: sonarqube: name: SonarQube Scan runs-on: ubuntu-latest - + + services: + mariadb: + image: mariadb:latest + env: + MARIADB_ROOT_PASSWORD: root_password + ports: + - 3306:3306 + options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: - name: Check if PR is from a fork @@ -75,8 +84,7 @@ jobs: - name: Run Pytests run: | - coverage run --append -m pytest METreformat - coverage run --append -m pytest METreadnc + coverage run --append -m pytest - name: Output coverage report run: coverage report -m diff --git a/.github/workflows/unit_tests.yaml b/.github/workflows/unit_tests.yaml index b651b3f..94a560d 100644 --- a/.github/workflows/unit_tests.yaml +++ b/.github/workflows/unit_tests.yaml @@ -76,9 +76,7 @@ jobs: - name: Test with pytest run: | - coverage run --append -m pytest METdbLoad/test - coverage run --append -m pytest METreformat - coverage run --append -m pytest METreadnc + coverage run --append -m pytest coverage report -m echo "Finished unit tests and coverage" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d5521a6 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,20 @@ + +[tool.pytest.ini_options] +addopts = "-q" +testpaths = [ + "METdbLoad/test", + "METreadnc/test", + "METreformat/test", +] + +[tool.coverage.run] +relative_files = true +source = [ + "METdbLoad/ush", + "METreadnc/util", + "METreformat/", + ] +omit = [ + "*__init__.py", + "METreformat/test/*" +] \ No newline at end of file From 4b4f75dcd09527bc1e685bd407f8ae35f50ea7f3 Mon Sep 17 00:00:00 2001 From: John-Sharples Date: Tue, 17 Sep 2024 19:19:02 +0000 Subject: [PATCH 3/6] Remove sonarQube coverage config --- internal/scripts/sonarqube/sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 763997d..cdaea59 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -4,7 +4,7 @@ sonar.projectName=METdataio sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=METdbLoad,METreadnc,METreformat -sonar.coverage.exclusions=**/test/*,docs/**,setup.py +#sonar.coverage.exclusions=**/test/*,docs/**,setup.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 From 5c13132255f4ff8c420f0be759085f8577895633 Mon Sep 17 00:00:00 2001 From: John-Sharples Date: Tue, 17 Sep 2024 19:37:43 +0000 Subject: [PATCH 4/6] Reinstate sonarQube converage config --- internal/scripts/sonarqube/sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index cdaea59..763997d 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -4,7 +4,7 @@ sonar.projectName=METdataio sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=METdbLoad,METreadnc,METreformat -#sonar.coverage.exclusions=**/test/*,docs/**,setup.py +sonar.coverage.exclusions=**/test/*,docs/**,setup.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 From 22599c21cc6f7bb30c6b7efe836a3a0b99c30698 Mon Sep 17 00:00:00 2001 From: John-Sharples Date: Tue, 17 Sep 2024 19:53:03 +0000 Subject: [PATCH 5/6] Exclude tests/ from sonarQube coverage --- internal/scripts/sonarqube/sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 763997d..4952b02 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -4,7 +4,7 @@ sonar.projectName=METdataio sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME sonar.sources=METdbLoad,METreadnc,METreformat -sonar.coverage.exclusions=**/test/*,docs/**,setup.py +sonar.coverage.exclusions=**/test/*,**/tests/*,docs/**,setup.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 From acf1735806696a8d573bb2becfe8e4b7605285aa Mon Sep 17 00:00:00 2001 From: John-Sharples Date: Tue, 17 Sep 2024 19:56:26 +0000 Subject: [PATCH 6/6] Get consistency with sonarQube and coverage --- internal/scripts/sonarqube/sonar-project.properties | 2 +- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/scripts/sonarqube/sonar-project.properties b/internal/scripts/sonarqube/sonar-project.properties index 4952b02..e795fd2 100644 --- a/internal/scripts/sonarqube/sonar-project.properties +++ b/internal/scripts/sonarqube/sonar-project.properties @@ -3,7 +3,7 @@ sonar.projectKey=METdataio sonar.projectName=METdataio sonar.projectVersion=SONAR_PROJECT_VERSION sonar.branch.name=SONAR_BRANCH_NAME -sonar.sources=METdbLoad,METreadnc,METreformat +sonar.sources=METdbLoad/ush,METreadnc/util,METreformat sonar.coverage.exclusions=**/test/*,**/tests/*,docs/**,setup.py sonar.python.coverage.reportPaths=coverage.xml sonar.sourceEncoding=UTF-8 diff --git a/pyproject.toml b/pyproject.toml index d5521a6..4b41f06 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,5 @@ source = [ "METreformat/", ] omit = [ - "*__init__.py", "METreformat/test/*" ] \ No newline at end of file