From cac34f1e04d62301d703df27d26308a8b52620e8 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 27 Feb 2024 15:42:24 -0900 Subject: [PATCH 1/8] Add ISCE2 handling to init so we only have to do it once --- isce2_topsapp/__init__.py | 44 +++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/isce2_topsapp/__init__.py b/isce2_topsapp/__init__.py index 661f1e5..43fede9 100644 --- a/isce2_topsapp/__init__.py +++ b/isce2_topsapp/__init__.py @@ -1,17 +1,39 @@ +import logging +import os import warnings from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + +# ---------------------------------------------------------------------------------------------------------------------- +# Handling ISCE2 idiosyncrasies; At least the logging fix should be done BEFORE most other imports +# ---------------------------------------------------------------------------------------------------------------------- +import isce # noqa: F401 +# This ensures all ISCE2 paths and environment variables are set when using this module, see: +# https://github.com/isce-framework/isce2/blob/main/__init__.py#L41-L50 + +# ISCE2 sets the root logger to DEBUG resulting in excessively verbose logging, see: +# https://github.com/isce-framework/isce2/issues/258 +root_logger = logging.getLogger() +root_logger.setLevel('WARNING') + +# ISCE2 also needs its applications to be on the system path, even though they say it's only "for convenience", see: +# https://github.com/isce-framework/isce2#setup-your-environment +ISCE_APPLICATIONS = str(Path(os.environ['ISCE_HOME']) / 'applications') +if ISCE_APPLICATIONS not in (PATH := os.environ['PATH'].split(os.pathsep)): + os.environ['PATH'] = os.pathsep.join([ISCE_APPLICATIONS] + PATH) +# ---------------------------------------------------------------------------------------------------------------------- + +from isce2_topsapp.delivery_prep import prepare_for_delivery # noqa: E402 +from isce2_topsapp.localize_aux_cal import download_aux_cal # noqa: E402 +from isce2_topsapp.localize_burst import BurstParams, download_bursts, get_region_of_interest # noqa: E402 +from isce2_topsapp.localize_dem import download_dem_for_isce2 # noqa: E402 +from isce2_topsapp.localize_mask import download_water_mask # noqa: E402 +from isce2_topsapp.localize_orbits import download_orbits # noqa: E402 +from isce2_topsapp.localize_slc import download_slcs, get_asf_slc_objects # noqa: E402 +from isce2_topsapp.packaging import package_gunw_product # noqa: E402 +from isce2_topsapp.topsapp_params import topsappParams # noqa: E402 +from isce2_topsapp.topsapp_proc import topsapp_processing # noqa: E402 -from isce2_topsapp.delivery_prep import prepare_for_delivery -from isce2_topsapp.localize_aux_cal import download_aux_cal -from isce2_topsapp.localize_burst import (BurstParams, download_bursts, - get_region_of_interest) -from isce2_topsapp.localize_dem import download_dem_for_isce2 -from isce2_topsapp.localize_mask import download_water_mask -from isce2_topsapp.localize_orbits import download_orbits -from isce2_topsapp.localize_slc import download_slcs, get_asf_slc_objects -from isce2_topsapp.packaging import package_gunw_product -from isce2_topsapp.topsapp_params import topsappParams -from isce2_topsapp.topsapp_proc import topsapp_processing try: __version__ = version(__name__) From 08d51dfce57bd9488a63ba4c1aebf62815e1ff7c Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 27 Feb 2024 15:43:17 -0900 Subject: [PATCH 2/8] drop ISCE2 application PATH handling everywhere besides init --- isce2_topsapp/iono_proc.py | 6 ------ isce2_topsapp/localize_dem.py | 3 +-- isce2_topsapp/topsapp_proc.py | 5 ----- 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/isce2_topsapp/iono_proc.py b/isce2_topsapp/iono_proc.py index b226593..aaece72 100644 --- a/isce2_topsapp/iono_proc.py +++ b/isce2_topsapp/iono_proc.py @@ -47,12 +47,6 @@ def iono_processing( Outlier removal and masking using connected component 0 will be skipped ''' - - # Update PATH with ISCE2 applications - # Need for isce2/bin/imageMath.py in runIon.unwrap function - isce_app_path = Path(f"{site.getsitepackages()[0]}" "/isce/applications/") - os.environ["PATH"] += ":" + str(isce_app_path) - # Use Connected component 0 to mask unwrapped interferogram # to improve masking after using water mask, e.g. remove noisy # pixels along the coastline diff --git a/isce2_topsapp/localize_dem.py b/isce2_topsapp/localize_dem.py index 617b6b2..f513f2f 100644 --- a/isce2_topsapp/localize_dem.py +++ b/isce2_topsapp/localize_dem.py @@ -28,8 +28,7 @@ def tag_dem_xml_as_ellipsoidal(dem_path: Path) -> str: def fix_image_xml(isce_raster_path: str) -> str: - isce_apps_path = site.getsitepackages()[0] + '/isce/applications' - fix_cmd = [f'{isce_apps_path}/fixImageXml.py', + fix_cmd = [f'fixImageXml.py', '-i', str(isce_raster_path), '--full'] diff --git a/isce2_topsapp/topsapp_proc.py b/isce2_topsapp/topsapp_proc.py index d378f88..eea1b45 100644 --- a/isce2_topsapp/topsapp_proc.py +++ b/isce2_topsapp/topsapp_proc.py @@ -64,11 +64,6 @@ def topsapp_processing(*, else: raise ValueError('Output resolution must be "30" or "90"') - # Update PATH with ISCE2 applications - isce_application_path = Path(f'{site.getsitepackages()[0]}' - '/isce/applications/') - os.environ['PATH'] += (':' + str(isce_application_path)) - with open(TEMPLATE_DIR/'topsapp_template.xml', 'r') as file: template = Template(file.read()) From d48d8254b4341ac4bc24ce3b12381308e62db085 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 27 Feb 2024 15:53:09 -0900 Subject: [PATCH 3/8] drop no longer necessary isce imports; remove some unused imports --- isce2_topsapp/iono_proc.py | 1 - isce2_topsapp/localize_dem.py | 3 +-- isce2_topsapp/packaging_utils/isce_functions.py | 13 +++---------- isce2_topsapp/packaging_utils/makeGeocube.py | 6 +----- isce2_topsapp/packaging_utils/nc_packaging.py | 4 ---- isce2_topsapp/topsapp_proc.py | 5 +---- 6 files changed, 6 insertions(+), 26 deletions(-) diff --git a/isce2_topsapp/iono_proc.py b/isce2_topsapp/iono_proc.py index aaece72..13d082a 100644 --- a/isce2_topsapp/iono_proc.py +++ b/isce2_topsapp/iono_proc.py @@ -5,7 +5,6 @@ import datetime import os import shutil -import site from pathlib import Path from typing import Type, Union diff --git a/isce2_topsapp/localize_dem.py b/isce2_topsapp/localize_dem.py index f513f2f..25e0ddc 100644 --- a/isce2_topsapp/localize_dem.py +++ b/isce2_topsapp/localize_dem.py @@ -1,4 +1,3 @@ -import site import subprocess from pathlib import Path @@ -28,7 +27,7 @@ def tag_dem_xml_as_ellipsoidal(dem_path: Path) -> str: def fix_image_xml(isce_raster_path: str) -> str: - fix_cmd = [f'fixImageXml.py', + fix_cmd = ['fixImageXml.py', '-i', str(isce_raster_path), '--full'] diff --git a/isce2_topsapp/packaging_utils/isce_functions.py b/isce2_topsapp/packaging_utils/isce_functions.py index caec438..f3e4df7 100755 --- a/isce2_topsapp/packaging_utils/isce_functions.py +++ b/isce2_topsapp/packaging_utils/isce_functions.py @@ -5,7 +5,9 @@ from __future__ import division from builtins import str from builtins import range -from osgeo import gdal, ogr, osr +from osgeo import gdal, ogr + +import isce def data_loading(filename,out_data_type=None,data_band=None): @@ -209,11 +211,8 @@ def get_topsApp_data(topsapp_xml='topsApp'): ''' loading the topsapp xml file ''' - - import isce from topsApp import TopsInSAR import os - import pdb # prvide the full path and strip off any .xml if pressent topsapp_xml = os.path.splitext(os.path.abspath(topsapp_xml))[0] curdir = os.getcwd() @@ -228,7 +227,6 @@ def get_topsApp_data(topsapp_xml='topsApp'): return insar def get_isce_version_info(args): - import isce isce_version = isce.release_version if isce.release_svn_revision: isce_version = "ISCE version = " + isce_version + ", " + "SVN revision = " + isce.release_svn_revision @@ -442,8 +440,6 @@ def check_file_exist(infile): raise Exception(infile + " does not exist") def read_isce_product(xmlfile): - import os - import isce from iscesys.Component.ProductManager import ProductManager as PM # check if the file does exist @@ -622,6 +618,3 @@ def get_bbox(args): # return the polygon as a list of strings, which each poly a list argument geom_union_str = ["%s"%geom_union] return geom_union_str - - - diff --git a/isce2_topsapp/packaging_utils/makeGeocube.py b/isce2_topsapp/packaging_utils/makeGeocube.py index d76e9a7..356cbd5 100644 --- a/isce2_topsapp/packaging_utils/makeGeocube.py +++ b/isce2_topsapp/packaging_utils/makeGeocube.py @@ -2,20 +2,16 @@ from builtins import str from builtins import range from builtins import object -import math import numpy as np import os -import isce import argparse import h5py -import datetime import pyproj -import pdb import logging import shutil from time import time from functools import wraps -from joblib import Parallel, delayed, dump, load +from joblib import Parallel, delayed from pathlib import Path from pyproj import CRS diff --git a/isce2_topsapp/packaging_utils/nc_packaging.py b/isce2_topsapp/packaging_utils/nc_packaging.py index 1fa2a35..c990052 100755 --- a/isce2_topsapp/packaging_utils/nc_packaging.py +++ b/isce2_topsapp/packaging_utils/nc_packaging.py @@ -5,7 +5,6 @@ from builtins import range from builtins import object import argparse -import sys import json import logging import traceback @@ -13,12 +12,9 @@ import os from netCDF4 import Dataset import numpy as np -import isce -import osgeo from osgeo import gdal from osgeo import osr import collections.abc -import pdb log_format = "[%(asctime)s: %(levelname)s/%(funcName)s] %(message)s" logging.basicConfig(format=log_format, level=logging.INFO) diff --git a/isce2_topsapp/topsapp_proc.py b/isce2_topsapp/topsapp_proc.py index eea1b45..33735bf 100644 --- a/isce2_topsapp/topsapp_proc.py +++ b/isce2_topsapp/topsapp_proc.py @@ -1,5 +1,3 @@ -import os -import site import subprocess from pathlib import Path @@ -99,9 +97,8 @@ def topsapp_processing(*, with open('topsApp.xml', "w") as file: file.write(topsApp_xml) - tops_app_cmd = f'{isce_application_path}/topsApp.py' for step in tqdm(TOPSAPP_STEPS, desc='TopsApp Steps'): - step_cmd = f'{tops_app_cmd} --dostep={step}' + step_cmd = f'topsApp.py --dostep={step}' result = subprocess.run(step_cmd, shell=True) if result.returncode != 0: From 4d332f6343e09d8f555fc54f0c3d59fa60c123e7 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 27 Feb 2024 16:13:43 -0900 Subject: [PATCH 4/8] Add CHANGELOG --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1972b04..8234a23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.4] + +### Changed +* ISCE2 shenanigans (PYTHON_PATH, PATH, and logging changes) are now handled in the `isce2_topsapp.__init__` ensuring we only have to do it once, no matter where we are in the package. + +### Fixed +* The root logger is no longer set to DEBUG by ISCE2 preventing excessive logging from all packages in the environment + + ## [0.3.3] ### Fixed From 8f9c53fcbf8cd05061c1508476af5c56de9317d6 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Tue, 27 Feb 2024 16:16:16 -0900 Subject: [PATCH 5/8] comment consistency --- isce2_topsapp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isce2_topsapp/__init__.py b/isce2_topsapp/__init__.py index 43fede9..9b82468 100644 --- a/isce2_topsapp/__init__.py +++ b/isce2_topsapp/__init__.py @@ -7,9 +7,9 @@ # ---------------------------------------------------------------------------------------------------------------------- # Handling ISCE2 idiosyncrasies; At least the logging fix should be done BEFORE most other imports # ---------------------------------------------------------------------------------------------------------------------- -import isce # noqa: F401 # This ensures all ISCE2 paths and environment variables are set when using this module, see: # https://github.com/isce-framework/isce2/blob/main/__init__.py#L41-L50 +import isce # noqa: F401 # ISCE2 sets the root logger to DEBUG resulting in excessively verbose logging, see: # https://github.com/isce-framework/isce2/issues/258 From d052cd0a5cada50ce2f87430ee0b3a43db1b98e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Mar 2024 06:51:02 +0000 Subject: [PATCH 6/8] Bump extractions/netrc from 1 to 2 Bumps [extractions/netrc](https://github.com/extractions/netrc) from 1 to 2. - [Release notes](https://github.com/extractions/netrc/releases) - [Commits](https://github.com/extractions/netrc/compare/v1...v2) --- updated-dependencies: - dependency-name: extractions/netrc dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 90b73c2..fd6257b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: extractions/netrc@v1 + - uses: extractions/netrc@v2 with: machine: urs.earthdata.nasa.gov username: ${{ secrets.EARTHDATA_USERNAME }} From 9950be4d5cd7d8fa6aa3d6427c7af6c283c278fb Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 11 Mar 2024 15:40:59 -0700 Subject: [PATCH 7/8] Update dataset version --- isce2_topsapp/packaging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/isce2_topsapp/packaging.py b/isce2_topsapp/packaging.py index 622cb04..e1806e2 100644 --- a/isce2_topsapp/packaging.py +++ b/isce2_topsapp/packaging.py @@ -21,7 +21,7 @@ from isce2_topsapp.templates import read_netcdf_packaging_template from isce2_topsapp.water_mask import get_water_mask_raster_for_browse_image -DATASET_VERSION = "3.0.0" +DATASET_VERSION = "3.0.1" STANDARD_PROD_PREFIX = "S1-GUNW" CUSTOM_PROD_PREFIX = "S1-GUNW_CUSTOM" From d776a90ec4c3b62f4470b132785983eb3defaa42 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Mon, 11 Mar 2024 15:45:07 -0700 Subject: [PATCH 8/8] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8234a23..5f98145 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed * ISCE2 shenanigans (PYTHON_PATH, PATH, and logging changes) are now handled in the `isce2_topsapp.__init__` ensuring we only have to do it once, no matter where we are in the package. +* Update dataset version from `3.0.0` -> `3.0.1` to track products over areas processed during development of v3 product. In other words, large scale processing was done during v3 development but not all v3 products will be delivered to DAAC - we have standardized numerous attributes and layers. Utlimately, 3.0.0 and 3.0.1 at the DAAC will represent the v3 products created by this repository. ### Fixed * The root logger is no longer set to DEBUG by ISCE2 preventing excessive logging from all packages in the environment