From 07f05d6da40d10f520bacb347c54483f65265474 Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 20:46:31 +0200 Subject: [PATCH 01/12] fixing test for catalog clas --- CHANGELOG.md | 4 + tests/test_catalog.py | 205 +++++++++++++++++++++++------------------- up42/catalog.py | 16 ++-- 3 files changed, 126 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f28c4b3..af396551 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,10 @@ You can check your current version with the following command: ``` For more information, see [UP42 Python package description](https://pypi.org/project/up42-py/). +## 2.1.0a5 + +**Sep 12, 2024** +- Fix test coverage for `Catalog` class. ## 2.1.0a4 diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 7946f190..52bb0aa5 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,4 +1,3 @@ -import json import pathlib import geopandas as gpd # type: ignore @@ -8,22 +7,21 @@ import requests_mock as req_mock import shapely # type: ignore -from up42 import catalog, glossary, order +from up42 import catalog, glossary, order, utils from . import helpers from .fixtures import fixtures_globals as constants PHR = "phr" SIMPLE_BOX = shapely.box(0, 0, 1, 1).__geo_interface__ +START_DATE = "2014-01-01" +END_DATE = "2022-12-31" +DATE_RANGE = f"{utils.format_time(START_DATE)}/{utils.format_time(END_DATE, set_end_of_day=True)}" SEARCH_PARAMETERS = { - "datetime": "2014-01-01T00:00:00Z/2022-12-31T23:59:59Z", + "datetime": DATE_RANGE, "intersects": SIMPLE_BOX, "collections": [PHR], - "limit": 4, - "query": { - "cloudCoverage": {"lte": 20}, - "up42:usageType": {"in": ["DATA", "ANALYTICS"]}, - }, + "limit": 10, } @@ -49,6 +47,56 @@ def test_get_data_product_schema(catalog_mock): assert data_product_schema["properties"] +class TestCatalogBase: + def test_should_get_data_product_schema(self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker): + data_product_schema = {"schema": "some-schema"} + url = f"{constants.API_HOST}/orders/schema/{constants.DATA_PRODUCT_ID}" + requests_mock.get(url=url, json=data_product_schema) + catalog_mock = catalog.CatalogBase(auth_mock, constants.WORKSPACE_ID) + assert catalog_mock.get_data_product_schema(constants.DATA_PRODUCT_ID) == data_product_schema + + def test_should_place_order_from_catalog_base( + self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker, order_parameters: order.OrderParams + ): + info = {"status": "SOME STATUS"} + requests_mock.get( + url=f"{constants.API_HOST}/v2/orders/{constants.ORDER_ID}", + json=info, + ) + requests_mock.post( + url=f"{constants.API_HOST}/v2/orders?workspaceId={constants.WORKSPACE_ID}", + json={"results": [{"id": constants.ORDER_ID}], "errors": []}, + ) + order_obj = catalog.CatalogBase(auth_mock, constants.WORKSPACE_ID).place_order( + order_parameters=order_parameters + ) + assert isinstance(order_obj, order.Order) + assert order_obj.order_id == constants.ORDER_ID + + @pytest.mark.parametrize( + "info", + [{"type": "ARCHIVE"}, {"type": "TASKING", "orderDetails": {"subStatus": "substatus"}}], + ids=["ARCHIVE", "TASKING"], + ) + def test_should_track_order_status_from_catalog_base( + self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker, info: dict, order_parameters: order.OrderParams + ): + requests_mock.post( + url=f"{constants.API_HOST}/v2/orders?workspaceId={constants.WORKSPACE_ID}", + json={"results": [{"id": constants.ORDER_ID}], "errors": []}, + ) + statuses = ["INITIAL STATUS", "PLACED", "BEING_FULFILLED", "FULFILLED"] + responses = [{"json": {"status": status, **info}} for status in statuses] + requests_mock.get(f"{constants.API_HOST}/v2/orders/{constants.ORDER_ID}", responses) + order_obj = catalog.CatalogBase(auth_mock, constants.WORKSPACE_ID).place_order( + order_parameters=order_parameters, + track_status=True, + report_time=0.1, + ) + assert isinstance(order_obj, order.Order) + assert order_obj.track_status(report_time=0.1) == "FULFILLED" + + class TestCatalog: host = "oneatlas" catalog = catalog.Catalog(auth=mock.MagicMock(), workspace_id=constants.WORKSPACE_ID) @@ -180,50 +228,63 @@ def test_should_download_available_quicklooks(self, requests_mock: req_mock.Mock ) assert out_paths == [str(tmp_path / f"quicklook_{image_id}.jpg")] - -def test_construct_search_parameters(catalog_mock): - assert ( - catalog_mock.construct_search_parameters( - geometry=SIMPLE_BOX, - collections=[PHR], - start_date="2014-01-01", - end_date="2022-12-31", - usage_type=["DATA", "ANALYTICS"], - limit=4, - max_cloudcover=20, - ) - == SEARCH_PARAMETERS + @pytest.mark.parametrize( + "usage_type", + [ + {"usage_type": ["DATA"]}, + {"usage_type": ["ANALYTICS"]}, + {"usage_type": ["DATA", "ANALYTICS"]}, + {}, + ], ) - - -def test_construct_search_parameters_fc_multiple_features_raises(catalog_mock): - with open( - pathlib.Path(__file__).resolve().parent / "mock_data/search_footprints.geojson", - encoding="utf-8", - ) as file: - fc = json.load(file) - - with pytest.raises(ValueError) as e: - catalog_mock.construct_search_parameters( - geometry=fc, - start_date="2020-01-01", - end_date="2020-08-10", - collections=[PHR], - limit=10, - max_cloudcover=15, - ) - assert str(e.value) == "UP42 only accepts single geometries, the provided geometry contains multiple geometries." - - -def test_construct_order_parameters(catalog_mock): - order_parameters = catalog_mock.construct_order_parameters( - data_product_id=constants.DATA_PRODUCT_ID, - image_id="123", - aoi=SIMPLE_BOX, + @pytest.mark.parametrize( + "max_cloudcover", + [ + {"max_cloudcover": 100}, + {}, + ], ) - assert isinstance(order_parameters, dict) - assert list(order_parameters.keys()) == ["dataProduct", "params"] - assert order_parameters["params"]["acquisitionMode"] is None + def test_should_construct_search_parameters(self, usage_type: dict, max_cloudcover: dict): + params = { + "geometry": SIMPLE_BOX, + "collections": [PHR], + "start_date": START_DATE, + "end_date": END_DATE, + **usage_type, + **max_cloudcover, + } + response = {**SEARCH_PARAMETERS} + optional_response: dict = {"query": {}} + if "max_cloudcover" in max_cloudcover: + optional_response["query"]["cloudCoverage"] = {"lte": max_cloudcover["max_cloudcover"]} + if "usage_type" in usage_type: + optional_response["query"]["up42:usageType"] = {"in": usage_type["usage_type"]} + response = {**response, **optional_response} + assert self.catalog.construct_search_parameters(**params) == response + + def test_should_fail_construct_search_parameters_with_wrong_data_usage(self): + usage_type = ["WRONG_TYPE"] + error_msg = r"""usage_type only allows \["DATA"\], \["ANALYTICS"\] or \["DATA", "ANALYTICS"\]""" + with pytest.raises(catalog.UsageTypeError, match=error_msg): + self.catalog.construct_search_parameters( + geometry=SIMPLE_BOX, collections=[PHR], start_date=START_DATE, end_date=END_DATE, usage_type=usage_type + ) + + def test_should_construct_order_parameters(self, requests_mock: req_mock.Mocker): + url_schema = f"{constants.API_HOST}/orders/schema/{constants.DATA_PRODUCT_ID}" + requests_mock.get( + url_schema, + json={ + "required": ["additional_requirement"], + }, + ) + order_parameters: order.OrderParams = self.catalog.construct_order_parameters( + data_product_id=constants.DATA_PRODUCT_ID, + image_id="123", + aoi=SIMPLE_BOX, + tags=None, + ) + assert "additional_requirement" in order_parameters def test_search_usagetype(catalog_mock): @@ -277,47 +338,3 @@ def test_estimate_order_from_catalog(catalog_order_parameters, requests_mock, au estimation = catalog_instance.estimate_order(catalog_order_parameters) assert isinstance(estimation, int) assert estimation == 100 - - -def test_order_from_catalog( - order_parameters, - order_mock, # pylint: disable=unused-argument - catalog_mock, - requests_mock, -): - requests_mock.post( - url=f"{constants.API_HOST}/v2/orders?workspaceId={constants.WORKSPACE_ID}", - json={ - "results": [{"index": 0, "id": constants.ORDER_ID}], - "errors": [], - }, - ) - placed_order = catalog_mock.place_order(order_parameters=order_parameters) - assert isinstance(placed_order, order.Order) - assert placed_order.order_id == constants.ORDER_ID - - -def test_order_from_catalog_track_status(catalog_order_parameters, order_mock, catalog_mock, requests_mock): - requests_mock.post( - url=f"{constants.API_HOST}/v2/orders?workspaceId={constants.WORKSPACE_ID}", - json={ - "results": [{"index": 0, "id": constants.ORDER_ID}], - "errors": [], - }, - ) - url_order_info = f"{constants.API_HOST}/v2/orders/{order_mock.order_id}" - requests_mock.get( - url_order_info, - [ - {"json": {"status": "PLACED", "type": "ARCHIVE"}}, - {"json": {"status": "BEING_FULFILLED", "type": "ARCHIVE"}}, - {"json": {"status": "FULFILLED", "type": "ARCHIVE"}}, - ], - ) - placed_order = catalog_mock.place_order( - order_parameters=catalog_order_parameters, - track_status=True, - report_time=0.1, - ) - assert isinstance(placed_order, order.Order) - assert placed_order.order_id == constants.ORDER_ID diff --git a/up42/catalog.py b/up42/catalog.py index 28af1332..8d964329 100644 --- a/up42/catalog.py +++ b/up42/catalog.py @@ -3,7 +3,7 @@ """ import pathlib -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union, cast import geojson # type: ignore import geopandas # type: ignore @@ -16,6 +16,10 @@ logger = utils.get_logger(__name__) +class UsageTypeError(ValueError): + pass + + class CatalogBase: """ The base for Catalog and Tasking class, shared functionality. @@ -45,7 +49,7 @@ def place_order( self, order_parameters: order.OrderParams, track_status: bool = False, - report_time: int = 120, + report_time: float = 120, ) -> order.Order: """ Place an order. @@ -91,6 +95,8 @@ class Catalog(CatalogBase): [CatalogBase](catalogbase-reference.md) class. """ + session = base.Session() + def __init__(self, auth: up42_auth.Auth, workspace_id: str): super().__init__(auth, workspace_id) self.type: glossary.CollectionType = glossary.CollectionType.ARCHIVE @@ -177,7 +183,7 @@ def construct_search_parameters( elif usage_type == ["DATA", "ANALYTICS"]: query_filters["up42:usageType"] = {"in": ["DATA", "ANALYTICS"]} else: - raise ValueError("Select correct `usage_type`") + raise UsageTypeError("""usage_type only allows ["DATA"], ["ANALYTICS"] or ["DATA", "ANALYTICS"]""") return { "datetime": time_period, @@ -273,7 +279,7 @@ def construct_order_parameters( geom.Polygon, ] = None, tags: Optional[List[str]] = None, - ): + ) -> order.OrderParams: """ Helps constructing the parameters dictionary required for the catalog order. Some collections have @@ -324,7 +330,7 @@ def construct_order_parameters( aoi = utils.fc_to_query_geometry(fc=aoi, geometry_operation="intersects") order_parameters["params"]["aoi"] = aoi # type: ignore - return order_parameters + return cast(order.OrderParams, order_parameters) def download_quicklooks( self, From d4f7927653b026483ee5aff2169e7b282dc8a104 Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 21:04:39 +0200 Subject: [PATCH 02/12] fixing test for catalog class --- tests/test_catalog.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 52bb0aa5..122309a5 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -41,19 +41,13 @@ def set_status_raising_session(): catalog.CatalogBase.session = session # type: ignore -def test_get_data_product_schema(catalog_mock): - data_product_schema = catalog_mock.get_data_product_schema(constants.DATA_PRODUCT_ID) - assert isinstance(data_product_schema, dict) - assert data_product_schema["properties"] - - class TestCatalogBase: def test_should_get_data_product_schema(self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker): data_product_schema = {"schema": "some-schema"} url = f"{constants.API_HOST}/orders/schema/{constants.DATA_PRODUCT_ID}" requests_mock.get(url=url, json=data_product_schema) - catalog_mock = catalog.CatalogBase(auth_mock, constants.WORKSPACE_ID) - assert catalog_mock.get_data_product_schema(constants.DATA_PRODUCT_ID) == data_product_schema + catalog_obj = catalog.CatalogBase(auth_mock, constants.WORKSPACE_ID) + assert catalog_obj.get_data_product_schema(constants.DATA_PRODUCT_ID) == data_product_schema def test_should_place_order_from_catalog_base( self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker, order_parameters: order.OrderParams @@ -270,21 +264,25 @@ def test_should_fail_construct_search_parameters_with_wrong_data_usage(self): geometry=SIMPLE_BOX, collections=[PHR], start_date=START_DATE, end_date=END_DATE, usage_type=usage_type ) - def test_should_construct_order_parameters(self, requests_mock: req_mock.Mocker): + def test_should_construct_order_parameters(self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker): url_schema = f"{constants.API_HOST}/orders/schema/{constants.DATA_PRODUCT_ID}" requests_mock.get( url_schema, json={ - "required": ["additional_requirement"], + "required": ["additional"], + "properties": {"additional": {"type": "string", "title": "string", "format": "string"}}, + "definitions": {}, }, ) - order_parameters: order.OrderParams = self.catalog.construct_order_parameters( + order_parameters: order.OrderParams = catalog.Catalog( + auth_mock, constants.WORKSPACE_ID + ).construct_order_parameters( data_product_id=constants.DATA_PRODUCT_ID, image_id="123", - aoi=SIMPLE_BOX, + aoi=None, tags=None, ) - assert "additional_requirement" in order_parameters + assert "additional" in order_parameters["params"] def test_search_usagetype(catalog_mock): From c584ee32748979aed0d62a9e6bf62314122c874c Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 21:30:38 +0200 Subject: [PATCH 03/12] fixing test for catalog class --- tests/test_catalog.py | 64 +++++++++++++++++++++++++++++++++++++------ up42/catalog.py | 27 ++++++++---------- 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 122309a5..2053218f 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,4 +1,6 @@ import pathlib +import uuid +from typing import List, Optional import geopandas as gpd # type: ignore import mock @@ -23,6 +25,7 @@ "collections": [PHR], "limit": 10, } +Geometries = catalog.Geometries @pytest.fixture(autouse=True) @@ -230,6 +233,12 @@ def test_should_download_available_quicklooks(self, requests_mock: req_mock.Mock {"usage_type": ["DATA", "ANALYTICS"]}, {}, ], + ids=[ + "usage_type:DATA", + "usage_type:ANALYTICS", + "usage_type:DATA, ANALYTICS", + "usage_type:None", + ], ) @pytest.mark.parametrize( "max_cloudcover", @@ -237,6 +246,10 @@ def test_should_download_available_quicklooks(self, requests_mock: req_mock.Mock {"max_cloudcover": 100}, {}, ], + ids=[ + "max_cloudcover:100", + "max_cloudcover:None", + ], ) def test_should_construct_search_parameters(self, usage_type: dict, max_cloudcover: dict): params = { @@ -264,25 +277,60 @@ def test_should_fail_construct_search_parameters_with_wrong_data_usage(self): geometry=SIMPLE_BOX, collections=[PHR], start_date=START_DATE, end_date=END_DATE, usage_type=usage_type ) - def test_should_construct_order_parameters(self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker): + @pytest.mark.parametrize( + "aoi", + [ + SIMPLE_BOX, + None, + ], + ids=[ + "aoi:SIMPLE_BOX", + "aoi:None", + ], + ) + @pytest.mark.parametrize( + "tags", + [ + ["tag"], + None, + ], + ids=[ + "tags:Value", + "tags:None", + ], + ) + def test_should_construct_order_parameters( + self, + auth_mock: mock.MagicMock, + requests_mock: req_mock.Mocker, + aoi: Optional[Geometries], + tags: Optional[List[str]], + ): + schema_property = "any-property" + image_id = str(uuid.uuid4()) url_schema = f"{constants.API_HOST}/orders/schema/{constants.DATA_PRODUCT_ID}" requests_mock.get( url_schema, json={ - "required": ["additional"], - "properties": {"additional": {"type": "string", "title": "string", "format": "string"}}, - "definitions": {}, + "required": [schema_property], + "properties": {schema_property: {"type": "string", "title": "string", "format": "string"}}, }, ) order_parameters: order.OrderParams = catalog.Catalog( auth_mock, constants.WORKSPACE_ID ).construct_order_parameters( data_product_id=constants.DATA_PRODUCT_ID, - image_id="123", - aoi=None, - tags=None, + image_id=image_id, + aoi=aoi, + tags=tags, ) - assert "additional" in order_parameters["params"] + assert schema_property in order_parameters["params"] + assert order_parameters["params"]["id"] == image_id + assert order_parameters["dataProduct"] == constants.DATA_PRODUCT_ID + if tags is not None: + assert order_parameters["tags"] == tags + if aoi is not None: + assert order_parameters["params"]["aoi"] == aoi def test_search_usagetype(catalog_mock): diff --git a/up42/catalog.py b/up42/catalog.py index 8d964329..78b17b9d 100644 --- a/up42/catalog.py +++ b/up42/catalog.py @@ -15,6 +15,15 @@ logger = utils.get_logger(__name__) +Geometries = Union[ + dict, + geojson.Feature, + geojson.FeatureCollection, + list, + geopandas.GeoDataFrame, + geom.Polygon, +] + class UsageTypeError(ValueError): pass @@ -117,14 +126,7 @@ def estimate_order(self, order_parameters: order.OrderParams) -> int: @staticmethod def construct_search_parameters( - geometry: Union[ - geojson.FeatureCollection, - geojson.Feature, - dict, - list, - geopandas.GeoDataFrame, - geom.Polygon, - ], + geometry: Geometries, collections: List[str], start_date: str = "2020-01-01", end_date: str = "2020-01-30", @@ -270,14 +272,7 @@ def construct_order_parameters( self, data_product_id: str, image_id: str, - aoi: Union[ - dict, - geojson.Feature, - geojson.FeatureCollection, - list, - geopandas.GeoDataFrame, - geom.Polygon, - ] = None, + aoi: Optional[Geometries] = None, tags: Optional[List[str]] = None, ) -> order.OrderParams: """ From 1d90bc2b901863324b9ec0e4de77bdda247bd4d3 Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 21:51:08 +0200 Subject: [PATCH 04/12] fixing test for catalog class --- tests/test_catalog.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 2053218f..e66503e5 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -332,6 +332,24 @@ def test_should_construct_order_parameters( if aoi is not None: assert order_parameters["params"]["aoi"] == aoi + def test_estimate_order_from_catalog( + self, + auth_mock: mock.MagicMock, + requests_mock: req_mock.Mocker, + catalog_order_parameters: order.OrderParams, + ): + order_estimate_url = f"{constants.API_HOST}/v2/orders/estimate" + expected_credits = 100 + requests_mock.post( + url=order_estimate_url, + json={ + "summary": {"totalCredits": expected_credits}, + "errors": [], + }, + ) + catalog_obj = catalog.Catalog(auth=auth_mock, workspace_id=constants.WORKSPACE_ID) + assert catalog_obj.estimate_order(catalog_order_parameters) == expected_credits + def test_search_usagetype(catalog_mock): """ @@ -370,17 +388,3 @@ def test_search_usagetype(catalog_mock): "collections": [PHR], "query": {"up42:usageType": {"in": params["usage_type"]}}, } - - -def test_estimate_order_from_catalog(catalog_order_parameters, requests_mock, auth_mock): - catalog_instance = catalog.Catalog(auth=auth_mock, workspace_id=constants.WORKSPACE_ID) - expected_payload = { - "summary": {"totalCredits": 100, "totalSize": 0.1, "unit": "SQ_KM"}, - "results": [{"index": 0, "credits": 100, "unit": "SQ_KM", "size": 0.1}], - "errors": [], - } - url_order_estimation = f"{constants.API_HOST}/v2/orders/estimate" - requests_mock.post(url=url_order_estimation, json=expected_payload) - estimation = catalog_instance.estimate_order(catalog_order_parameters) - assert isinstance(estimation, int) - assert estimation == 100 From 3900e3d401f1b44b662b3d654e88cd1783dada2f Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 21:53:59 +0200 Subject: [PATCH 05/12] fixing test for catalog class --- tests/test_catalog.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index e66503e5..6995dd56 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -349,42 +349,3 @@ def test_estimate_order_from_catalog( ) catalog_obj = catalog.Catalog(auth=auth_mock, workspace_id=constants.WORKSPACE_ID) assert catalog_obj.estimate_order(catalog_order_parameters) == expected_credits - - -def test_search_usagetype(catalog_mock): - """ - Result & Result2 are one of the combinations of - "DATA" and "ANALYTICS". Result2 can be None. - - The result assertion needs to allow multiple combinations, - e.g. when searching for ["DATA", "ANALYTICS"], - the result can be ["DATA"], ["ANALYTICS"] - or ["DATA", "ANALYTICS"]. - """ - params1 = {"usage_type": ["DATA"], "result1": "DATA", "result2": ""} - params2 = { - "usage_type": ["ANALYTICS"], - "result1": "ANALYTICS", - "result2": "", - } - params3 = { - "usage_type": ["DATA", "ANALYTICS"], - "result1": "DATA", - "result2": "ANALYTICS", - } - - for params in [params1, params2, params3]: - assert catalog_mock.construct_search_parameters( - start_date="2014-01-01T00:00:00", - end_date="2020-12-31T23:59:59", - collections=[PHR], - limit=1, - usage_type=params["usage_type"], - geometry=SIMPLE_BOX, - ) == { - "datetime": "2014-01-01T00:00:00Z/2020-12-31T23:59:59Z", - "intersects": SIMPLE_BOX, - "limit": 1, - "collections": [PHR], - "query": {"up42:usageType": {"in": params["usage_type"]}}, - } From 99c70bc3118baf98c07ed46dfbbfb07b267bff6b Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 22:49:07 +0200 Subject: [PATCH 06/12] fixing test for catalog class --- tests/conftest.py | 1 - tests/fixtures/fixtures_catalog.py | 21 --------------------- 2 files changed, 22 deletions(-) delete mode 100644 tests/fixtures/fixtures_catalog.py diff --git a/tests/conftest.py b/tests/conftest.py index 1c0eb0f6..7f65d548 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,6 @@ pytest_plugins = [ "tests.fixtures.fixtures_auth", - "tests.fixtures.fixtures_catalog", "tests.fixtures.fixtures_order", "tests.fixtures.fixtures_storage", "tests.fixtures.fixtures_tasking", diff --git a/tests/fixtures/fixtures_catalog.py b/tests/fixtures/fixtures_catalog.py deleted file mode 100644 index 974bf7ed..00000000 --- a/tests/fixtures/fixtures_catalog.py +++ /dev/null @@ -1,21 +0,0 @@ -import json -import pathlib - -import pytest - -from up42 import catalog - -from . import fixtures_globals as constants - - -@pytest.fixture -def catalog_mock(auth_mock, requests_mock): - url_data_product_schema = f"{constants.API_HOST}/orders/schema/{constants.DATA_PRODUCT_ID}" - with open( - pathlib.Path(__file__).resolve().parents[1] / "mock_data/data_product_spot_schema.json", - encoding="utf-8", - ) as json_file: - json_data_product_schema = json.load(json_file) - requests_mock.get(url=url_data_product_schema, json=json_data_product_schema) - - return catalog.Catalog(auth=auth_mock, workspace_id=constants.WORKSPACE_ID) From f0494dc9b2ebad333837ddb01e2623875b6ede62 Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 22:57:28 +0200 Subject: [PATCH 07/12] fixing error in case of missing quickview --- tests/test_catalog.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 6995dd56..9187ac91 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -212,7 +212,7 @@ def test_should_download_available_quicklooks(self, requests_mock: req_mock.Mock missing_image_id = "missing-image-id" image_id = "image-id" missing_quicklook_url = f"{constants.API_HOST}/catalog/{self.host}/image/{missing_image_id}/quicklook" - requests_mock.get(missing_quicklook_url, status_code=404) + requests_mock.get(missing_quicklook_url, exc=requests.exceptions.RequestException) quicklook_url = f"{constants.API_HOST}/catalog/{self.host}/image/{image_id}/quicklook" quicklook_file = pathlib.Path(__file__).resolve().parent / "mock_data/a_quicklook.png" From e3b1833860dd8a4b442e4c98c3e0af888892b999 Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 12 Sep 2024 23:00:31 +0200 Subject: [PATCH 08/12] fixing error in case of missing quickview --- tests/test_catalog.py | 2 +- up42/catalog.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 9187ac91..6995dd56 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -212,7 +212,7 @@ def test_should_download_available_quicklooks(self, requests_mock: req_mock.Mock missing_image_id = "missing-image-id" image_id = "image-id" missing_quicklook_url = f"{constants.API_HOST}/catalog/{self.host}/image/{missing_image_id}/quicklook" - requests_mock.get(missing_quicklook_url, exc=requests.exceptions.RequestException) + requests_mock.get(missing_quicklook_url, status_code=404) quicklook_url = f"{constants.API_HOST}/catalog/{self.host}/image/{image_id}/quicklook" quicklook_file = pathlib.Path(__file__).resolve().parent / "mock_data/a_quicklook.png" diff --git a/up42/catalog.py b/up42/catalog.py index 78b17b9d..b992aedd 100644 --- a/up42/catalog.py +++ b/up42/catalog.py @@ -363,6 +363,7 @@ def download_quicklooks( try: url = host.endpoint(f"/catalog/{product_host}/image/{image_id}/quicklook") response = self.session.get(url) + response.raise_for_status() # TODO: should detect extensions based on response content type # TODO: to be simplified using utils.download_file out_path = str(output_directory / f"quicklook_{image_id}.jpg") From dbdf35ab9f78f6a440990395a6e75c5831b9b8fa Mon Sep 17 00:00:00 2001 From: andher1802 Date: Fri, 13 Sep 2024 11:25:55 +0200 Subject: [PATCH 09/12] covering uncovered lines --- tests/test_catalog.py | 100 +++++++++++++++++++++++++++++++----------- 1 file changed, 74 insertions(+), 26 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 6995dd56..2269639b 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -1,9 +1,12 @@ import pathlib import uuid -from typing import List, Optional +from typing import List, Optional, cast +import geojson # type: ignore import geopandas as gpd # type: ignore import mock +import numpy as np +import pandas as pd import pytest import requests import requests_mock as req_mock @@ -25,6 +28,12 @@ "collections": [PHR], "limit": 10, } +FEATURE = { + "type": "Feature", + "properties": {"some": "data"}, + "geometry": {"type": "Point", "coordinates": (1.0, 2.0)}, +} +POINT_BBOX = (1.0, 2.0, 1.0, 2.0) Geometries = catalog.Geometries @@ -98,6 +107,10 @@ class TestCatalog: host = "oneatlas" catalog = catalog.Catalog(auth=mock.MagicMock(), workspace_id=constants.WORKSPACE_ID) + @pytest.fixture(params=["output_dir", "no_output_dir"]) + def output_directory(self, request, tmp_path) -> Optional[str]: + return tmp_path if request.param == "output_dir" else None + @pytest.fixture def product_glossary(self, requests_mock: req_mock.Mocker): collections_url = f"{constants.API_HOST}/v2/collections" @@ -161,21 +174,42 @@ def test_search_fails_if_collections_hosted_by_different_hosts(self, requests_mo with pytest.raises(ValueError): self.catalog.search({"collections": ["collection1", "collection2"]}) + @pytest.mark.parametrize( + "feature, expected_geom", + [ + ( + FEATURE, + gpd.GeoDataFrame.from_features( + geojson.FeatureCollection( + features=[ + {**FEATURE, "id": "0", "bbox": POINT_BBOX}, + {**FEATURE, "id": "1", "bbox": POINT_BBOX}, + ] + ), + crs="EPSG:4326", + ), + ), + (None, gpd.GeoDataFrame(columns=["geometry"], geometry="geometry")), + ], + ids=["with_features", "without_features"], + ) + @pytest.mark.parametrize( + "as_dataframe", + [True, False], + ids=["as_dataframe", "as_dict"], + ) @pytest.mark.usefixtures("product_glossary") - def test_should_search(self, requests_mock: req_mock.Mocker): + def test_should_search( + self, requests_mock: req_mock.Mocker, feature: dict, expected_geom: gpd.GeoDataFrame, as_dataframe: bool + ): search_url = f"{constants.API_HOST}/catalog/hosts/{self.host}/stac/search" next_page_url = f"{search_url}/next" - bbox = (1.0, 2.0, 1.0, 2.0) - feature = { - "type": "Feature", - "properties": { - "some": "data", - }, - "geometry": {"type": "Point", "coordinates": (1.0, 2.0)}, - } + features = [] + if feature: + features.append(feature) first_page = { "type": "FeatureCollection", - "features": [feature], + "features": features, "links": [ { "rel": "next", @@ -194,21 +228,32 @@ def test_should_search(self, requests_mock: req_mock.Mocker): json=second_page, additional_matcher=helpers.match_request_body(SEARCH_PARAMETERS), ) - - results = self.catalog.search(SEARCH_PARAMETERS) - - assert isinstance(results, gpd.GeoDataFrame) - assert results.__geo_interface__ == { - "type": "FeatureCollection", - "features": [ - {**feature, "id": "0", "bbox": bbox}, - {**feature, "id": "1", "bbox": bbox}, - ], - "bbox": bbox, - } + results = self.catalog.search(SEARCH_PARAMETERS, as_dataframe=as_dataframe) + if as_dataframe: + results = cast(gpd.GeoDataFrame, results) + pd.testing.assert_frame_equal(results.drop(columns="geometry"), expected_geom.drop(columns="geometry")) + assert results.geometry.equals(expected_geom.geometry) + assert results.crs == expected_geom.crs + else: + bbox = results.pop("bbox") + assert results == { + "type": "FeatureCollection", + "features": [ + {**FEATURE, "id": "0", "bbox": POINT_BBOX}, + {**FEATURE, "id": "1", "bbox": POINT_BBOX}, + ] + if feature + else [], + } + if feature: + assert bbox == POINT_BBOX + else: + assert all(np.isnan(coor) for coor in bbox) @pytest.mark.usefixtures("product_glossary") - def test_should_download_available_quicklooks(self, requests_mock: req_mock.Mocker, tmp_path): + def test_should_download_available_quicklooks( + self, requests_mock: req_mock.Mocker, output_directory: Optional[str] + ): missing_image_id = "missing-image-id" image_id = "image-id" missing_quicklook_url = f"{constants.API_HOST}/catalog/{self.host}/image/{missing_image_id}/quicklook" @@ -221,9 +266,12 @@ def test_should_download_available_quicklooks(self, requests_mock: req_mock.Mock out_paths = self.catalog.download_quicklooks( image_ids=[image_id, missing_image_id], collection=PHR, - output_directory=tmp_path, + output_directory=output_directory, + ) + download_folder: pathlib.Path = ( + pathlib.Path(output_directory) if output_directory else pathlib.Path.cwd() / "catalog" ) - assert out_paths == [str(tmp_path / f"quicklook_{image_id}.jpg")] + assert out_paths == [str(download_folder / f"quicklook_{image_id}.jpg")] @pytest.mark.parametrize( "usage_type", From f782d35a84f220fd971b31db5b768b08ed7adbb5 Mon Sep 17 00:00:00 2001 From: andher1802 Date: Fri, 13 Sep 2024 13:55:00 +0200 Subject: [PATCH 10/12] adding specific errors --- tests/test_catalog.py | 4 ++-- up42/catalog.py | 12 ++++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 2269639b..3ef39176 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -141,7 +141,7 @@ def product_glossary(self, requests_mock: req_mock.Mocker): @pytest.mark.usefixtures("product_glossary") def test_search_fails_if_host_is_not_found(self): - with pytest.raises(ValueError, match=r"Selected collections \['unknown'\] are not valid.*"): + with pytest.raises(catalog.InvalidCollections, match=r"Selected collections \['unknown'\] are not valid.*"): self.catalog.search({"collections": ["unknown"]}) def test_search_fails_if_collections_hosted_by_different_hosts(self, requests_mock: req_mock.Mocker): @@ -171,7 +171,7 @@ def test_search_fails_if_collections_hosted_by_different_hosts(self, requests_mo "totalPages": 1, }, ) - with pytest.raises(ValueError): + with pytest.raises(catalog.MultipleHosts): self.catalog.search({"collections": ["collection1", "collection2"]}) @pytest.mark.parametrize( diff --git a/up42/catalog.py b/up42/catalog.py index b992aedd..3e52e66f 100644 --- a/up42/catalog.py +++ b/up42/catalog.py @@ -29,6 +29,14 @@ class UsageTypeError(ValueError): pass +class InvalidCollections(ValueError): + pass + + +class MultipleHosts(ValueError): + pass + + class CatalogBase: """ The base for Catalog and Tasking class, shared functionality. @@ -206,11 +214,11 @@ def _get_host(self, collection_names: list[str]) -> str: } if not hosts: - raise ValueError( + raise InvalidCollections( f"Selected collections {collection_names} are not valid. See ProductGlossary.get_collections." ) if len(hosts) > 1: - raise ValueError("Only collections with the same host can be searched at the same time.") + raise MultipleHosts("Only collections with the same host can be searched at the same time.") return hosts.pop() def search(self, search_parameters: dict, as_dataframe: bool = True) -> Union[geopandas.GeoDataFrame, dict]: From ae0724e9a7306b6c90eccea5165335b8e86cb2fe Mon Sep 17 00:00:00 2001 From: andher1802 Date: Thu, 19 Sep 2024 18:16:55 +0200 Subject: [PATCH 11/12] first round of feedback --- tests/test_catalog.py | 35 ++++++++++++++++++----------------- up42/catalog.py | 17 +++++++---------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/tests/test_catalog.py b/tests/test_catalog.py index 3ef39176..6a77cb40 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -12,16 +12,18 @@ import requests_mock as req_mock import shapely # type: ignore -from up42 import catalog, glossary, order, utils +from up42 import catalog, glossary, order from . import helpers from .fixtures import fixtures_globals as constants +Geometry = catalog.Geometry + PHR = "phr" SIMPLE_BOX = shapely.box(0, 0, 1, 1).__geo_interface__ START_DATE = "2014-01-01" END_DATE = "2022-12-31" -DATE_RANGE = f"{utils.format_time(START_DATE)}/{utils.format_time(END_DATE, set_end_of_day=True)}" +DATE_RANGE = "2014-01-01T00:00:00Z/2022-12-31T23:59:59Z" SEARCH_PARAMETERS = { "datetime": DATE_RANGE, "intersects": SIMPLE_BOX, @@ -34,7 +36,6 @@ "geometry": {"type": "Point", "coordinates": (1.0, 2.0)}, } POINT_BBOX = (1.0, 2.0, 1.0, 2.0) -Geometries = catalog.Geometries @pytest.fixture(autouse=True) @@ -61,7 +62,7 @@ def test_should_get_data_product_schema(self, auth_mock: mock.MagicMock, request catalog_obj = catalog.CatalogBase(auth_mock, constants.WORKSPACE_ID) assert catalog_obj.get_data_product_schema(constants.DATA_PRODUCT_ID) == data_product_schema - def test_should_place_order_from_catalog_base( + def test_should_place_order( self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker, order_parameters: order.OrderParams ): info = {"status": "SOME STATUS"} @@ -84,7 +85,7 @@ def test_should_place_order_from_catalog_base( [{"type": "ARCHIVE"}, {"type": "TASKING", "orderDetails": {"subStatus": "substatus"}}], ids=["ARCHIVE", "TASKING"], ) - def test_should_track_order_status_from_catalog_base( + def test_should_track_order_status( self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker, info: dict, order_parameters: order.OrderParams ): requests_mock.post( @@ -99,8 +100,8 @@ def test_should_track_order_status_from_catalog_base( track_status=True, report_time=0.1, ) - assert isinstance(order_obj, order.Order) - assert order_obj.track_status(report_time=0.1) == "FULFILLED" + assert order_obj.order_id == constants.ORDER_ID + assert order_obj.status == "FULFILLED" class TestCatalog: @@ -108,7 +109,7 @@ class TestCatalog: catalog = catalog.Catalog(auth=mock.MagicMock(), workspace_id=constants.WORKSPACE_ID) @pytest.fixture(params=["output_dir", "no_output_dir"]) - def output_directory(self, request, tmp_path) -> Optional[str]: + def output_directory(self, request, tmp_path) -> Optional[pathlib.Path]: return tmp_path if request.param == "output_dir" else None @pytest.fixture @@ -252,7 +253,7 @@ def test_should_search( @pytest.mark.usefixtures("product_glossary") def test_should_download_available_quicklooks( - self, requests_mock: req_mock.Mocker, output_directory: Optional[str] + self, requests_mock: req_mock.Mocker, output_directory: Optional[pathlib.Path] ): missing_image_id = "missing-image-id" image_id = "image-id" @@ -268,9 +269,7 @@ def test_should_download_available_quicklooks( collection=PHR, output_directory=output_directory, ) - download_folder: pathlib.Path = ( - pathlib.Path(output_directory) if output_directory else pathlib.Path.cwd() / "catalog" - ) + download_folder: pathlib.Path = output_directory if output_directory else pathlib.Path.cwd() / "catalog" assert out_paths == [str(download_folder / f"quicklook_{image_id}.jpg")] @pytest.mark.parametrize( @@ -318,11 +317,13 @@ def test_should_construct_search_parameters(self, usage_type: dict, max_cloudcov assert self.catalog.construct_search_parameters(**params) == response def test_should_fail_construct_search_parameters_with_wrong_data_usage(self): - usage_type = ["WRONG_TYPE"] - error_msg = r"""usage_type only allows \["DATA"\], \["ANALYTICS"\] or \["DATA", "ANALYTICS"\]""" - with pytest.raises(catalog.UsageTypeError, match=error_msg): + with pytest.raises(catalog.InvalidUsageType, match="usage_type is invalid"): self.catalog.construct_search_parameters( - geometry=SIMPLE_BOX, collections=[PHR], start_date=START_DATE, end_date=END_DATE, usage_type=usage_type + geometry=SIMPLE_BOX, + collections=[PHR], + start_date=START_DATE, + end_date=END_DATE, + usage_type=["WRONG_TYPE"], # type: ignore ) @pytest.mark.parametrize( @@ -351,7 +352,7 @@ def test_should_construct_order_parameters( self, auth_mock: mock.MagicMock, requests_mock: req_mock.Mocker, - aoi: Optional[Geometries], + aoi: Optional[Geometry], tags: Optional[List[str]], ): schema_property = "any-property" diff --git a/up42/catalog.py b/up42/catalog.py index 3e52e66f..5749a305 100644 --- a/up42/catalog.py +++ b/up42/catalog.py @@ -3,7 +3,7 @@ """ import pathlib -from typing import Any, Dict, List, Optional, Union, cast +from typing import Any, Dict, List, Literal, Optional, Union, cast import geojson # type: ignore import geopandas # type: ignore @@ -15,7 +15,7 @@ logger = utils.get_logger(__name__) -Geometries = Union[ +Geometry = Union[ dict, geojson.Feature, geojson.FeatureCollection, @@ -25,7 +25,7 @@ ] -class UsageTypeError(ValueError): +class InvalidUsageType(ValueError): pass @@ -112,8 +112,6 @@ class Catalog(CatalogBase): [CatalogBase](catalogbase-reference.md) class. """ - session = base.Session() - def __init__(self, auth: up42_auth.Auth, workspace_id: str): super().__init__(auth, workspace_id) self.type: glossary.CollectionType = glossary.CollectionType.ARCHIVE @@ -134,11 +132,11 @@ def estimate_order(self, order_parameters: order.OrderParams) -> int: @staticmethod def construct_search_parameters( - geometry: Geometries, + geometry: Geometry, collections: List[str], start_date: str = "2020-01-01", end_date: str = "2020-01-30", - usage_type: Optional[List[str]] = None, + usage_type: Optional[List[Literal["DATA", "ANALYTICS"]]] = None, limit: int = 10, max_cloudcover: Optional[int] = None, ) -> dict: @@ -193,7 +191,7 @@ def construct_search_parameters( elif usage_type == ["DATA", "ANALYTICS"]: query_filters["up42:usageType"] = {"in": ["DATA", "ANALYTICS"]} else: - raise UsageTypeError("""usage_type only allows ["DATA"], ["ANALYTICS"] or ["DATA", "ANALYTICS"]""") + raise InvalidUsageType("usage_type is invalid") return { "datetime": time_period, @@ -280,7 +278,7 @@ def construct_order_parameters( self, data_product_id: str, image_id: str, - aoi: Optional[Geometries] = None, + aoi: Optional[Geometry] = None, tags: Optional[List[str]] = None, ) -> order.OrderParams: """ @@ -371,7 +369,6 @@ def download_quicklooks( try: url = host.endpoint(f"/catalog/{product_host}/image/{image_id}/quicklook") response = self.session.get(url) - response.raise_for_status() # TODO: should detect extensions based on response content type # TODO: to be simplified using utils.download_file out_path = str(output_directory / f"quicklook_{image_id}.jpg") From 40931580ba60dfbdc55d903d3c748b589d5b474e Mon Sep 17 00:00:00 2001 From: andher1802 Date: Fri, 20 Sep 2024 15:28:15 +0200 Subject: [PATCH 12/12] first round of comments --- CHANGELOG.md | 4 ++-- pyproject.toml | 2 +- up42/catalog.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a36dd790..8def3e50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,9 +29,9 @@ You can check your current version with the following command: ``` For more information, see [UP42 Python package description](https://pypi.org/project/up42-py/). -## 2.1.0a5 +## 2.1.0a6 -**Sep 12, 2024** +**Sep TBD, 2024** - Fix test coverage for `Catalog` class. ## ## 2.1.0a5 diff --git a/pyproject.toml b/pyproject.toml index 27ff2d38..fa25134d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "up42-py" -version = "2.1.0a5" +version = "2.1.0a6" description = "Python SDK for UP42, the geospatial marketplace and developer platform." authors = ["UP42 GmbH "] license = "https://github.com/up42/up42-py/blob/master/LICENSE" diff --git a/up42/catalog.py b/up42/catalog.py index ad108eca..10d798d4 100644 --- a/up42/catalog.py +++ b/up42/catalog.py @@ -3,7 +3,7 @@ """ import pathlib -from typing import Any, Dict, List, Literal, Optional, Union, cast +from typing import Any, Dict, List, Literal, Optional, Union import geojson # type: ignore import geopandas # type: ignore @@ -332,7 +332,7 @@ def construct_order_parameters( aoi = utils.fc_to_query_geometry(fc=aoi, geometry_operation="intersects") order_parameters["params"]["aoi"] = aoi # type: ignore - return cast(order.OrderParams, order_parameters) + return order_parameters def download_quicklooks( self,