From 21212bce6a97777ee951a97e5296cc74e858dd44 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 1 Feb 2024 10:02:02 -0800 Subject: [PATCH 01/22] update to 3.10+ --- .github/workflows/test.yml | 2 +- CHANGELOG.md | 1 + environment.yml | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d926a32c..7cdf82e6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v3 diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b00f57c..4f93d1df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * Records parameters in the product including the CLI command to regenerate said product * If parameters are not standard uses prefix `S1-GUNW_CUSTOM-...` * Pydantic dependency for parameter accounting +* Support for python 3.10+ ### Changed * The CLI now *requires* `frame_id` (use `frame_id = -1` for old API and what is now considered a "non"-standard product) diff --git a/environment.yml b/environment.yml index d2431c01..3ba45cc6 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: topsapp_env channels: - conda-forge dependencies: - - python>=3.9,<3.10 + - python>=3.9 - pip - affine - asf_search>=5.0.0 @@ -17,7 +17,7 @@ dependencies: - geopandas - hyp3lib>=3,<4 - ipykernel - - isce2==2.6.1 + - isce2=>2.6.3 - jinja2 - joblib - jsonschema==3.2.0 From c44a5e813917847e7443479a0eefe6ae916c1ae4 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 1 Feb 2024 10:03:55 -0800 Subject: [PATCH 02/22] fail slow --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7cdf82e6..1c5902d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,7 @@ jobs: pytest: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12"] From 8762306f9d6ee9553c12653a095f9345007579c0 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 1 Feb 2024 10:42:58 -0800 Subject: [PATCH 03/22] issue with 3.12 --- .github/workflows/test.yml | 2 +- environment.yml | 2 +- setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c5902d0..611eac6a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 diff --git a/environment.yml b/environment.yml index 3ba45cc6..55f562db 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: topsapp_env channels: - conda-forge dependencies: - - python>=3.9 + - python>=3.9,<3.12 - pip - affine - asf_search>=5.0.0 diff --git a/setup.py b/setup.py index 3175b473..faefaf5e 100644 --- a/setup.py +++ b/setup.py @@ -23,9 +23,9 @@ 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], python_requires='>=3.8', From 1b2130c582619d029102306439b58981bd34a360 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Fri, 2 Feb 2024 08:56:55 -0800 Subject: [PATCH 04/22] remove zlib from string and iterable writes --- isce2_topsapp/packaging_utils/nc_packaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/packaging_utils/nc_packaging.py b/isce2_topsapp/packaging_utils/nc_packaging.py index 1fa2a35f..4648ac58 100755 --- a/isce2_topsapp/packaging_utils/nc_packaging.py +++ b/isce2_topsapp/packaging_utils/nc_packaging.py @@ -133,7 +133,7 @@ def write_dataset(fid,data,properties_data): else: if isinstance(data,str): - dset = fid.createVariable(properties_data.name, str, ('matchup',), zlib=True) + dset = fid.createVariable(properties_data.name, str, ('matchup',)) dset[0]=data elif isinstance(data,np.ndarray): # make sure the _fillvalue is formatted the same as the data_type @@ -156,7 +156,7 @@ def write_dataset(fid,data,properties_data): dset[:] = data elif isinstance(data, collections.abc.Iterable): if isinstance(data[0],str): - dset = fid.createVariable(properties_data.name, str, ('matchup',), zlib=True) + dset = fid.createVariable(properties_data.name, str, ('matchup',)) count = 0 for data_line in data: dset[count]=data_line From 7e860dbd56e7e787ef6cadd7ff333f75820ee201 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Fri, 2 Feb 2024 13:40:00 -0800 Subject: [PATCH 05/22] fix order of geq --- README.md | 20 +++++++++++++++++--- environment.yml | 2 +- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index bf35575c..fe93d5c4 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,21 @@ We note all the input datasets are publicly available using a NASA Earthdata acc 3. Create a new environment and install requirements using `conda env update --file environment.yml` (or use [`mamba`](https://github.com/mamba-org/mamba) to speed install up) 4. Install the package from cloned repo using `python -m pip install -e .` +### For Mac M1 Silicon Users + +`ISCE2` requires Intel `x86_64` complied, conda-forge packages. Please follow the directions [here](https://conda-forge.org/docs/user/tipsandtricks.html#installing-apple-intel-packages-on-apple-silicon) i.e. + +``` +CONDA_SUBDIR=osx-64 conda create -n topsapp_env python +conda activate topsapp_env +conda config --env --set subdir osx-64 +``` +Then check +``` +python -c "import platform;print(platform.machine())" # Should print "x86_64" +echo "CONDA_SUBDIR: $CONDA_SUBDIR" # Should print "CONDA_SUBDIR: osx-64" +``` + ## Additional setup 1. Ensure that your `~/.netrc` file has: @@ -158,7 +173,6 @@ or as a json: ## Expedient Docker Test for GUNW Generation Create a new directory (for all the intermediate files) and navigate to it. - ``` docker run -ti -v $PWD:/home/ops/topsapp_data topsapp_img \ --reference-scenes S1A_IW_SLC__1SDV_20220212T222803_20220212T222830_041886_04FCA3_2B3E \ @@ -209,8 +223,8 @@ EARTHDATA_PASSWORD=... The main entrypoint is in the `__main__.py` file [here](https://github.com/ACCESS-Cloud-Based-InSAR/DockerizedTopsApp/blob/dev/isce2_topsapp/__main__.py). The whole standard `++slc` workflow that generates the `ARIA-S1-GUNW` product is summarized in this file. The workflow takes 1.5 - 3 hours to complete. -For developing a new feature, we need to modify a very small parts of this long running workflow e.g. the packaging of ISCE2 outputs into a netcdf file, staging/preparation of auxiliary data, etc. -Thus, for development it is not necessary to rerun the workflow each time to test a new feature, but utilize the intermediate ISCE data files and fix relevant parts of the code. +Likely, when developing a small new feature, we need to modify only of this long running workflow e.g. the packaging of ISCE2 outputs into a netcdf file, staging/preparation of auxiliary data, etc. +Thus, for development, it is recommended to not rerun the workflow each time, but utilize the intermediate ISCE data files and the metadata stored as json by this plugin. We have some sample [notebooks](https://github.com/ACCESS-Cloud-Based-InSAR/DockerizedTopsapp-Debugging-NBs) to load the relevant metadata to make this "jumping" into the code slightly easier. We note that `ISCE2` generates *a lot* of interemdiate files. For our workflow, this can be between 100-150 GBs of disk required. diff --git a/environment.yml b/environment.yml index 55f562db..b9dccd64 100644 --- a/environment.yml +++ b/environment.yml @@ -17,7 +17,7 @@ dependencies: - geopandas - hyp3lib>=3,<4 - ipykernel - - isce2=>2.6.3 + - isce2>=2.6.3 - jinja2 - joblib - jsonschema==3.2.0 From 5b5130832b998dccfc9435f1fad3b9cc5fc02756 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Fri, 2 Feb 2024 13:43:20 -0800 Subject: [PATCH 06/22] link to GUNW comparison NB --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fe93d5c4..e96f67d6 100644 --- a/README.md +++ b/README.md @@ -239,6 +239,7 @@ However, the plugin takes about 1.5 to 3 hours (depending on the number of corre Therefore, these integration are *not* sufficient to permit a new release (see more instructions below). Until we have a complete end-to-end test of the workflow (via Hyp3), any new feature cannot be integrated into official production (i.e. the `main` branch). As a first step, it is imperative to share the output of a new feature (i.e. the GUNW file and the command to generate it). +Here is a notebook that demonstrates how to compare GUNWs that will be very helpful: https://github.com/ACCESS-Cloud-Based-InSAR/DockerizedTopsapp-Debugging-NBs/blob/dev/2_Compare_GUNWs.ipynb ## Hyp3 and Cloud Submission @@ -275,9 +276,10 @@ Here are more detailed [notes](https://github.com/ACCESS-Cloud-Based-InSAR/CICD- ### Additional (manual) checks -Even though there are some integration tests and unit tests in our test suites, this CPU intensive workflow cannot be tested end-to-end using github actions (there is simply not enough memory and CPU on a github runner). +Even though there are some integration tests and unit tests in our test suites, this CPU intensive workflow cannot be tested end-to-end using github actions (there is simply not enough memory and CPU for these workflows). Therefore, even if all the tests pass, there is still a nontrivial chance a GUNW is not successfully generated (e.g. the `topo` step of topsapp fizzles out because `numpy` API was not successfully tracked in the latest ISCE2 release). We request a sample GUNW be shared in a PR. +Ideally, a comparison of the GUNW created with a new branch and an existing one (as done in this [notebook](https://github.com/ACCESS-Cloud-Based-InSAR/DockerizedTopsapp-Debugging-NBs/blob/dev/2_Compare_GUNWs.ipynb)) is ideal. Even if we go through careful accounting, once a PR is merged into `dev`, we will use `hyp3` to further inspect the new plugin e.g. through this [notebook](https://github.com/ACCESS-Cloud-Based-InSAR/s1_frame_enumerator/blob/dev/notebooks/Submitting_to_Hyp3.ipynb) using a few sites. It is important to use `INSAR_ISCE_TEST` job to ensure the features from the `dev` branch are used. Only after these **manual** checks, will we continue with a release of the plugin. From 5482e3b26c54b5a9b1c842cad606afb1dced8538 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Fri, 2 Feb 2024 14:50:23 -0800 Subject: [PATCH 07/22] fix python_requires --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index faefaf5e..d75385fa 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ 'Programming Language :: Python :: 3.11', ], - python_requires='>=3.8', + python_requires='>=3.9', install_requires=[ 'asf_search>=3.0.4', From 4fbf8c1964293aec0f445fe4e870ed8e66459b86 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 13:29:58 -0800 Subject: [PATCH 08/22] use 30 meter water mask --- isce2_topsapp/__main__.py | 2 +- isce2_topsapp/localize_mask.py | 18 +++++++++++++++++- isce2_topsapp/water_mask.py | 6 +++--- tests/test_packaging.py | 2 +- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/isce2_topsapp/__main__.py b/isce2_topsapp/__main__.py index 94551ac2..ca42ed43 100644 --- a/isce2_topsapp/__main__.py +++ b/isce2_topsapp/__main__.py @@ -66,7 +66,7 @@ def localize_data( # For ionospheric correction computation if water_mask_flag: out_water_mask = download_water_mask( - out_slc["extent"], water_mask_name="esa_world_cover_2021_10m") + out_slc["extent"], water_mask_name="pekel_water_occurrence_2021") out_aux_cal = download_aux_cal() diff --git a/isce2_topsapp/localize_mask.py b/isce2_topsapp/localize_mask.py index 0c99a4fe..71ab95fd 100644 --- a/isce2_topsapp/localize_mask.py +++ b/isce2_topsapp/localize_mask.py @@ -10,7 +10,7 @@ def download_water_mask( - extent: list, water_mask_name: str = "esa_world_cover_2021_10m", buffer: float = 0.1 + extent: list, water_mask_name: str = "pekel_water_occurrence_2021", buffer: float = 0.1 ) -> dict: output_dir = Path(".").absolute() @@ -39,6 +39,22 @@ def download_water_mask( mask_filename = fix_image_xml(mask_filename) + elif water_mask_name == 'pekel_water_occurrence_2021': + X, p = get_raster_from_tiles(extent_buffered, tile_shortname='pekel_water_occurrence_2021') + mask = (X >= 95) + mask[mask.astype(bool)] = 255 + mask_filename = 'water_mask_derived_from_pekel_water_occurrence_2021_with_at_least_95_perc_water.geo' + + # Remove esa nodata, change gdal driver to ISCE, and generate VRT as in localize DEM + p_isce = p.copy() + p_isce['nodata'] = None + p_isce['driver'] = 'ISCE' + + with rasterio.open(mask_filename, 'w', **p_isce) as ds: + ds.write(mask) + + mask_filename = fix_image_xml(mask_filename) + elif water_mask_name == "SWBD": # Download SRTM-SWDB water mask # Water mask dataset extent diff --git a/isce2_topsapp/water_mask.py b/isce2_topsapp/water_mask.py index 2cec0bbf..32847ca3 100644 --- a/isce2_topsapp/water_mask.py +++ b/isce2_topsapp/water_mask.py @@ -21,8 +21,8 @@ def get_water_mask_raster_for_browse_image(profile: dict) -> np.ndarray: """ extent = array_bounds(profile["height"], profile["width"], profile["transform"]) - X_esa, p_esa = get_raster_from_tiles(extent, tile_shortname="esa_world_cover_2021") - X_esa_r, _ = reproject_arr_to_match_profile(X_esa, p_esa, profile, resampling='nearest') - mask = (X_esa_r == 80).astype(bool) + X_occ, p_occ = get_raster_from_tiles(extent, tile_shortname="peckel_water_occ_2021") + X_occ_r, _ = reproject_arr_to_match_profile(X_occ, p_occ, profile, resampling='bilinear') + mask = (X_occ_r >= 95).astype(bool) mask = mask[0, ...] return mask diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 4a5c2af5..845d2e24 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -66,7 +66,7 @@ def test_mean_of_geocoded_isce_outputs(): "mean_filtered_coherence_without_water_mask": 0.4395995, "mean_filtered_coherence_with_water_mask": 0.4649538, "mean_unfiltered_coherence_without_water_mask": 0.33125195, - "mean_unfiltered_coherence_with_water_mask": 0.33531728, + "mean_unfiltered_coherence_with_water_mask": 0.3347433, "mean_incidence_angle": 38.845047, "mean_azimuth_angle": -169.84756469726562, } From f920ad99c06168f3a778a48adcea5093be01bc5e Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 13:37:43 -0800 Subject: [PATCH 09/22] revert to 3.9 --- environment.yml | 4 ++-- setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/environment.yml b/environment.yml index b9dccd64..d2431c01 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: topsapp_env channels: - conda-forge dependencies: - - python>=3.9,<3.12 + - python>=3.9,<3.10 - pip - affine - asf_search>=5.0.0 @@ -17,7 +17,7 @@ dependencies: - geopandas - hyp3lib>=3,<4 - ipykernel - - isce2>=2.6.3 + - isce2==2.6.1 - jinja2 - joblib - jsonschema==3.2.0 diff --git a/setup.py b/setup.py index d75385fa..3175b473 100644 --- a/setup.py +++ b/setup.py @@ -23,12 +23,12 @@ 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', - 'Programming Language :: Python :: 3.11', ], - python_requires='>=3.9', + python_requires='>=3.8', install_requires=[ 'asf_search>=3.0.4', From d98534390485a91ab4fb848f0c808cfb2b33754b Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 13:39:08 -0800 Subject: [PATCH 10/22] update changelog --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f93d1df..c75ddd36 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,12 +23,11 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * Records parameters in the product including the CLI command to regenerate said product * If parameters are not standard uses prefix `S1-GUNW_CUSTOM-...` * Pydantic dependency for parameter accounting -* Support for python 3.10+ ### Changed * The CLI now *requires* `frame_id` (use `frame_id = -1` for old API and what is now considered a "non"-standard product) * Water mask now uses `tile-mate` to download and merge ESA World cover tiles on -* All water masks applied to processing/packaging use ESA 10 meter world cover: ionosphere processing, browse imagery, and global attributes associate with mean coherence +* All water masks applied to processing/packaging use Pekel Occurence (>= 95 percent occurence): ionosphere processing, browse imagery, and global attributes associate with mean coherence * Some function names associated to writing global attributes in the netcdf file were renamed to be more descriptive e.g. `record_stats` became `record_stats_as_global_attrs` ## [0.3.0] From 51b67f9121d5bd540e3fccff98072fd09fe5b995 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 13:39:33 -0800 Subject: [PATCH 11/22] revert tests --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 611eac6a..d926a32c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,8 @@ jobs: pytest: runs-on: ubuntu-latest strategy: - fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] + python-version: ["3.9"] steps: - uses: actions/checkout@v3 From 9dc474cbbce59df841b6ae2ca45568289d02a388 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 13:40:48 -0800 Subject: [PATCH 12/22] revert packaging --- isce2_topsapp/packaging_utils/nc_packaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/packaging_utils/nc_packaging.py b/isce2_topsapp/packaging_utils/nc_packaging.py index 4648ac58..1fa2a35f 100755 --- a/isce2_topsapp/packaging_utils/nc_packaging.py +++ b/isce2_topsapp/packaging_utils/nc_packaging.py @@ -133,7 +133,7 @@ def write_dataset(fid,data,properties_data): else: if isinstance(data,str): - dset = fid.createVariable(properties_data.name, str, ('matchup',)) + dset = fid.createVariable(properties_data.name, str, ('matchup',), zlib=True) dset[0]=data elif isinstance(data,np.ndarray): # make sure the _fillvalue is formatted the same as the data_type @@ -156,7 +156,7 @@ def write_dataset(fid,data,properties_data): dset[:] = data elif isinstance(data, collections.abc.Iterable): if isinstance(data[0],str): - dset = fid.createVariable(properties_data.name, str, ('matchup',)) + dset = fid.createVariable(properties_data.name, str, ('matchup',), zlib=True) count = 0 for data_line in data: dset[count]=data_line From 4f1f86f1a39932174dbaa7ec33709fbe669f3643 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 13:42:54 -0800 Subject: [PATCH 13/22] fix types --- isce2_topsapp/localize_mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isce2_topsapp/localize_mask.py b/isce2_topsapp/localize_mask.py index 71ab95fd..2d33d1ef 100644 --- a/isce2_topsapp/localize_mask.py +++ b/isce2_topsapp/localize_mask.py @@ -41,7 +41,7 @@ def download_water_mask( elif water_mask_name == 'pekel_water_occurrence_2021': X, p = get_raster_from_tiles(extent_buffered, tile_shortname='pekel_water_occurrence_2021') - mask = (X >= 95) + mask = (X >= 95).astype(np.uint8) mask[mask.astype(bool)] = 255 mask_filename = 'water_mask_derived_from_pekel_water_occurrence_2021_with_at_least_95_perc_water.geo' From fe778ddcf31a67347e0e6fd206522af7590d8561 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 18:16:30 -0800 Subject: [PATCH 14/22] fix tile shortname --- isce2_topsapp/localize_mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isce2_topsapp/localize_mask.py b/isce2_topsapp/localize_mask.py index 2d33d1ef..3147a07d 100644 --- a/isce2_topsapp/localize_mask.py +++ b/isce2_topsapp/localize_mask.py @@ -40,7 +40,7 @@ def download_water_mask( mask_filename = fix_image_xml(mask_filename) elif water_mask_name == 'pekel_water_occurrence_2021': - X, p = get_raster_from_tiles(extent_buffered, tile_shortname='pekel_water_occurrence_2021') + X, p = get_raster_from_tiles(extent_buffered, tile_shortname='pekel_water_occ_2021') mask = (X >= 95).astype(np.uint8) mask[mask.astype(bool)] = 255 mask_filename = 'water_mask_derived_from_pekel_water_occurrence_2021_with_at_least_95_perc_water.geo' From 42038c0bce2b03fab531a9f5de10400d1766432c Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 18:17:08 -0800 Subject: [PATCH 15/22] fix tile-mate version --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index d2431c01..3d96de90 100644 --- a/environment.yml +++ b/environment.yml @@ -44,4 +44,4 @@ dependencies: - tqdm - dem_stitcher>=2.5.5 - aiohttp # only needed for manifest and swath download - - tile_mate>=0.0.7 + - tile_mate>=0.0.8 From 99129753a49787b672c643aa77304198f2d0cfbb Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 18:21:29 -0800 Subject: [PATCH 16/22] fix misspelling again --- isce2_topsapp/water_mask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isce2_topsapp/water_mask.py b/isce2_topsapp/water_mask.py index 32847ca3..6ea8564a 100644 --- a/isce2_topsapp/water_mask.py +++ b/isce2_topsapp/water_mask.py @@ -21,7 +21,7 @@ def get_water_mask_raster_for_browse_image(profile: dict) -> np.ndarray: """ extent = array_bounds(profile["height"], profile["width"], profile["transform"]) - X_occ, p_occ = get_raster_from_tiles(extent, tile_shortname="peckel_water_occ_2021") + X_occ, p_occ = get_raster_from_tiles(extent, tile_shortname="pekel_water_occ_2021") X_occ_r, _ = reproject_arr_to_match_profile(X_occ, p_occ, profile, resampling='bilinear') mask = (X_occ_r >= 95).astype(bool) mask = mask[0, ...] From fd852afdce496f789639b27080eb912a0724aa67 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 18:28:12 -0800 Subject: [PATCH 17/22] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c75ddd36..d7362332 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed * The CLI now *requires* `frame_id` (use `frame_id = -1` for old API and what is now considered a "non"-standard product) -* Water mask now uses `tile-mate` to download and merge ESA World cover tiles on +* Water mask now uses `tile-mate>=0.0.8` to download and merge water mask tiles (Pekel Occurence data >= 95 is the default) * All water masks applied to processing/packaging use Pekel Occurence (>= 95 percent occurence): ionosphere processing, browse imagery, and global attributes associate with mean coherence * Some function names associated to writing global attributes in the netcdf file were renamed to be more descriptive e.g. `record_stats` became `record_stats_as_global_attrs` From 956f08e891dbe4afbc601273c41e5d5a4512a23d Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 18:47:27 -0800 Subject: [PATCH 18/22] update env again --- CHANGELOG.md | 1 + environment.yml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7362332..cac7019b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * Records parameters in the product including the CLI command to regenerate said product * If parameters are not standard uses prefix `S1-GUNW_CUSTOM-...` * Pydantic dependency for parameter accounting +* Support for python 3.10 and 3.11 ### Changed * The CLI now *requires* `frame_id` (use `frame_id = -1` for old API and what is now considered a "non"-standard product) diff --git a/environment.yml b/environment.yml index 3d96de90..5ab4d1ab 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: topsapp_env channels: - conda-forge dependencies: - - python>=3.9,<3.10 + - python>=3.9,<3.12 - pip - affine - asf_search>=5.0.0 @@ -17,7 +17,7 @@ dependencies: - geopandas - hyp3lib>=3,<4 - ipykernel - - isce2==2.6.1 + - isce2==2.6.3 - jinja2 - joblib - jsonschema==3.2.0 From ca160bda05f2827743a6969aeedcd72c66757f0e Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 18:52:39 -0800 Subject: [PATCH 19/22] remove zlib again --- isce2_topsapp/packaging_utils/nc_packaging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/isce2_topsapp/packaging_utils/nc_packaging.py b/isce2_topsapp/packaging_utils/nc_packaging.py index 1fa2a35f..4648ac58 100755 --- a/isce2_topsapp/packaging_utils/nc_packaging.py +++ b/isce2_topsapp/packaging_utils/nc_packaging.py @@ -133,7 +133,7 @@ def write_dataset(fid,data,properties_data): else: if isinstance(data,str): - dset = fid.createVariable(properties_data.name, str, ('matchup',), zlib=True) + dset = fid.createVariable(properties_data.name, str, ('matchup',)) dset[0]=data elif isinstance(data,np.ndarray): # make sure the _fillvalue is formatted the same as the data_type @@ -156,7 +156,7 @@ def write_dataset(fid,data,properties_data): dset[:] = data elif isinstance(data, collections.abc.Iterable): if isinstance(data[0],str): - dset = fid.createVariable(properties_data.name, str, ('matchup',), zlib=True) + dset = fid.createVariable(properties_data.name, str, ('matchup',)) count = 0 for data_line in data: dset[count]=data_line From 30a96b4065f55734a5e1067cfe615a23f1a12fb4 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 19:07:49 -0800 Subject: [PATCH 20/22] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cac7019b..9aaa84d4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). * Records parameters in the product including the CLI command to regenerate said product * If parameters are not standard uses prefix `S1-GUNW_CUSTOM-...` * Pydantic dependency for parameter accounting -* Support for python 3.10 and 3.11 +* Support for python 3.10 and 3.11 (Note: 3.12 cannot be supported due to pysolid issue: https://github.com/insarlab/PySolid/issues/78) ### Changed * The CLI now *requires* `frame_id` (use `frame_id = -1` for old API and what is now considered a "non"-standard product) From c630f0a852d7e73f9b1c262d3f2b9ac11a46cbd3 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 19:09:03 -0800 Subject: [PATCH 21/22] add back python versions --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d926a32c..04c788fb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From d80c708743a60494673a7e9a5b5291f22c2297f0 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Thu, 8 Feb 2024 19:09:14 -0800 Subject: [PATCH 22/22] fix supported python --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3175b473..faefaf5e 100644 --- a/setup.py +++ b/setup.py @@ -23,9 +23,9 @@ 'License :: OSI Approved :: Apache Software License', 'Natural Language :: English', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', ], python_requires='>=3.8',