Merge pull request #178 from ACCESS-Cloud-Based-InSAR/dev

v0.3.4

.github/workflows/test.yml

@@ -20,7 +20,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - uses: extractions/netrc@v1
+      - uses: extractions/netrc@v2
         with:
           machine: urs.earthdata.nasa.gov
           username: ${{ secrets.EARTHDATA_USERNAME }}

CHANGELOG.md

@@ -6,6 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.4]
+
+### Changed
+* ISCE2 shenanigans (PYTHON_PATH, PATH, and logging changes) are now handled in the `isce2_topsapp.__init__` ensuring we only have to do it once, no matter where we are in the package.
+* Update dataset version from `3.0.0` -> `3.0.1` to track products over areas processed during development of v3 product. In other words, large scale processing was done during v3 development but not all v3 products will be delivered to DAAC - we have standardized numerous attributes and layers. Utlimately, 3.0.0 and 3.0.1 at the DAAC will represent the v3 products created by this repository.
+
+### Fixed
+* The root logger is no longer set to DEBUG by ISCE2 preventing excessive logging from all packages in the environment
+
+
 ## [0.3.3]
 
 ### Fixed

isce2_topsapp/__init__.py

@@ -1,17 +1,39 @@
+import logging
+import os
 import warnings
 from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+
+# ----------------------------------------------------------------------------------------------------------------------
+# Handling ISCE2 idiosyncrasies; At least the logging fix should be done BEFORE most other imports
+# ----------------------------------------------------------------------------------------------------------------------
+# This ensures all ISCE2 paths and environment variables are set when using this module, see:
+# https://github.com/isce-framework/isce2/blob/main/__init__.py#L41-L50
+import isce   # noqa: F401
+
+# ISCE2 sets the root logger to DEBUG resulting in excessively verbose logging, see:
+# https://github.com/isce-framework/isce2/issues/258
+root_logger = logging.getLogger()
+root_logger.setLevel('WARNING')
+
+# ISCE2 also needs its applications to be on the system path, even though they say it's only "for convenience", see:
+# https://github.com/isce-framework/isce2#setup-your-environment
+ISCE_APPLICATIONS = str(Path(os.environ['ISCE_HOME']) / 'applications')
+if ISCE_APPLICATIONS not in (PATH := os.environ['PATH'].split(os.pathsep)):
+    os.environ['PATH'] = os.pathsep.join([ISCE_APPLICATIONS] + PATH)
+# ----------------------------------------------------------------------------------------------------------------------
+
+from isce2_topsapp.delivery_prep import prepare_for_delivery  # noqa: E402
+from isce2_topsapp.localize_aux_cal import download_aux_cal  # noqa: E402
+from isce2_topsapp.localize_burst import BurstParams, download_bursts, get_region_of_interest  # noqa: E402
+from isce2_topsapp.localize_dem import download_dem_for_isce2  # noqa: E402
+from isce2_topsapp.localize_mask import download_water_mask  # noqa: E402
+from isce2_topsapp.localize_orbits import download_orbits  # noqa: E402
+from isce2_topsapp.localize_slc import download_slcs, get_asf_slc_objects  # noqa: E402
+from isce2_topsapp.packaging import package_gunw_product  # noqa: E402
+from isce2_topsapp.topsapp_params import topsappParams  # noqa: E402
+from isce2_topsapp.topsapp_proc import topsapp_processing  # noqa: E402
 
-from isce2_topsapp.delivery_prep import prepare_for_delivery
-from isce2_topsapp.localize_aux_cal import download_aux_cal
-from isce2_topsapp.localize_burst import (BurstParams, download_bursts,
-                                          get_region_of_interest)
-from isce2_topsapp.localize_dem import download_dem_for_isce2
-from isce2_topsapp.localize_mask import download_water_mask
-from isce2_topsapp.localize_orbits import download_orbits
-from isce2_topsapp.localize_slc import download_slcs, get_asf_slc_objects
-from isce2_topsapp.packaging import package_gunw_product
-from isce2_topsapp.topsapp_params import topsappParams
-from isce2_topsapp.topsapp_proc import topsapp_processing
 
 try:
     __version__ = version(__name__)

isce2_topsapp/iono_proc.py

@@ -5,7 +5,6 @@
 import datetime
 import os
 import shutil
-import site
 from pathlib import Path
 from typing import Type, Union
 
@@ -47,12 +46,6 @@ def iono_processing(
         Outlier removal and masking using connected component 0 will
         be skipped
     '''
-
-    # Update PATH with ISCE2 applications
-    # Need for isce2/bin/imageMath.py in runIon.unwrap function
-    isce_app_path = Path(f"{site.getsitepackages()[0]}" "/isce/applications/")
-    os.environ["PATH"] += ":" + str(isce_app_path)
-
     # Use Connected component 0 to mask unwrapped interferogram
     # to improve masking after using water mask, e.g. remove noisy
     # pixels along the coastline

isce2_topsapp/localize_dem.py

@@ -1,4 +1,3 @@
-import site
 import subprocess
 from pathlib import Path
 
@@ -28,8 +27,7 @@ def tag_dem_xml_as_ellipsoidal(dem_path: Path) -> str:
 
 
 def fix_image_xml(isce_raster_path: str) -> str:
-    isce_apps_path = site.getsitepackages()[0] + '/isce/applications'
-    fix_cmd = [f'{isce_apps_path}/fixImageXml.py',
+    fix_cmd = ['fixImageXml.py',
                '-i',
                str(isce_raster_path),
                '--full']

isce2_topsapp/packaging.py

@@ -21,7 +21,7 @@
 from isce2_topsapp.templates import read_netcdf_packaging_template
 from isce2_topsapp.water_mask import get_water_mask_raster_for_browse_image
 
-DATASET_VERSION = "3.0.0"
+DATASET_VERSION = "3.0.1"
 STANDARD_PROD_PREFIX = "S1-GUNW"
 CUSTOM_PROD_PREFIX = "S1-GUNW_CUSTOM"
 

isce2_topsapp/packaging_utils/isce_functions.py

@@ -5,7 +5,9 @@
 from __future__ import division
 from builtins import str
 from builtins import range
-from osgeo import gdal, ogr, osr
+from osgeo import gdal, ogr
+
+import isce
 
 
 def data_loading(filename,out_data_type=None,data_band=None):
@@ -209,11 +211,8 @@ def get_topsApp_data(topsapp_xml='topsApp'):
     '''
         loading the topsapp xml file
     '''
-
-    import isce
     from topsApp import TopsInSAR
     import os
-    import pdb
     # prvide the full path and strip off any .xml if pressent
     topsapp_xml = os.path.splitext(os.path.abspath(topsapp_xml))[0]
     curdir = os.getcwd()
@@ -228,7 +227,6 @@ def get_topsApp_data(topsapp_xml='topsApp'):
     return insar
 
 def get_isce_version_info(args):
-    import isce
     isce_version = isce.release_version
     if isce.release_svn_revision:
         isce_version = "ISCE version = " + isce_version + ", " + "SVN revision = " + isce.release_svn_revision
@@ -442,8 +440,6 @@ def check_file_exist(infile):
         raise Exception(infile + " does not exist")
 
 def read_isce_product(xmlfile):
-    import os
-    import isce
     from iscesys.Component.ProductManager import ProductManager as PM
 
     # check if the file does exist
@@ -622,6 +618,3 @@ def get_bbox(args):
     # return the polygon as a list of strings, which each poly a list argument
     geom_union_str = ["%s"%geom_union]
     return geom_union_str
-
-
-

isce2_topsapp/packaging_utils/makeGeocube.py

@@ -2,20 +2,16 @@
 from builtins import str
 from builtins import range
 from builtins import object
-import math
 import numpy as np
 import os
-import isce
 import argparse
 import h5py
-import datetime
 import pyproj
-import pdb
 import logging
 import shutil
 from time import time
 from functools import wraps
-from joblib import Parallel, delayed, dump, load
+from joblib import Parallel, delayed
 from pathlib import Path
 from pyproj import CRS
 

isce2_topsapp/packaging_utils/nc_packaging.py

@@ -5,20 +5,16 @@
 from builtins import range
 from builtins import object
 import argparse
-import sys
 import json
 import logging
 import traceback
 from collections import OrderedDict
 import os
 from netCDF4 import Dataset
 import numpy as np
-import isce
-import osgeo
 from osgeo import gdal
 from osgeo import osr
 import collections.abc
-import pdb
 
 log_format = "[%(asctime)s: %(levelname)s/%(funcName)s] %(message)s"
 logging.basicConfig(format=log_format, level=logging.INFO)

isce2_topsapp/topsapp_proc.py

@@ -1,5 +1,3 @@
-import os
-import site
 import subprocess
 from pathlib import Path
 
@@ -64,11 +62,6 @@ def topsapp_processing(*,
     else:
         raise ValueError('Output resolution must be "30" or "90"')
 
-    # Update PATH with ISCE2 applications
-    isce_application_path = Path(f'{site.getsitepackages()[0]}'
-                                 '/isce/applications/')
-    os.environ['PATH'] += (':' + str(isce_application_path))
-
     with open(TEMPLATE_DIR/'topsapp_template.xml', 'r') as file:
         template = Template(file.read())
 
@@ -104,9 +97,8 @@ def topsapp_processing(*,
     with open('topsApp.xml', "w") as file:
         file.write(topsApp_xml)
 
-    tops_app_cmd = f'{isce_application_path}/topsApp.py'
     for step in tqdm(TOPSAPP_STEPS, desc='TopsApp Steps'):
-        step_cmd = f'{tops_app_cmd} --dostep={step}'
+        step_cmd = f'topsApp.py --dostep={step}'
         result = subprocess.run(step_cmd,
                                 shell=True)
         if result.returncode != 0: