Merge remote-tracking branch 'noaa/develop' into david-develop

.github/workflows/ci.yml

@@ -45,14 +45,15 @@ jobs:
           create-args: >-
             python=${{ matrix.python-version }}
             attrs=22.2.0
+            aioitertools=0.11.0
 
       - name: Test with pytest
-        run: pytest -n auto -v -W "ignore:Downloading test file:UserWarning::"
+        run: pytest -n auto -v -rsx -W "ignore:Downloading test file:UserWarning::"
 
       - name: Test with pytspack installed
         run: |
           pip install https://github.com/noaa-oar-arl/pytspack/archive/master.zip
-          pytest -n auto -v -k with_pytspack
+          pytest -n auto -v -rsx -k with_pytspack
 
   docs:
     name: Check docs build

monetio/sat/_omps_nadir_mm.py

@@ -13,30 +13,35 @@ def read_OMPS_nm(files):
     import xarray as xr
 
     count = 0
-    print(files)
     # Check if files are url
     if "https" in files[0]:
         filelist = sorted(files)
         for filename in filelist:
             data = extract_OMPS_nm_opendap(filename)
-            # print(data)
             if count == 0:
                 data_array = data
                 count += 1
             else:
                 data_array = xr.concat([data_array, data], "x")
-    else:
-        filelist = sorted(glob(files, recursive=False))
-
-        for filename in filelist:
-            data = extract_OMPS_nm(filename)
-
-            if count == 0:
-                data_array = data
-                count += 1
-            else:
-                data_array = xr.concat([data_array, data], "x")
-
+    else:  # using local files
+        if isinstance(files, str):  # expansion of filestring to list
+            filelist = sorted(glob(files, recursive=False))
+        else:  # ensure provided filelist is sorted
+            filelist = sorted(files)  # assume list
+        for filename in filelist:  # extract data
+            try:
+                data = extract_OMPS_nm(filename)
+                if count == 0:
+                    data_array = data
+                    count += 1
+                else:
+                    data_array = xr.concat([data_array, data], "x")
+            except (KeyError, ValueError) as e:
+                # KeyError occurs in load when file exists but contains no data
+                # ValueError occurs in concat when file cross-track dimensions are different than other files loaded
+                print(f"warning: skipping {filename}. {type(e).__name__} occurred: {e}")
+    if count == 0:
+        raise RuntimeError(f"no files loaded from files={files}")
     return data_array
 
 

tests/test_ish.py

@@ -6,6 +6,13 @@
 
 from monetio import ish
 
+try:
+    import requests
+
+    requests.head("https://www1.ncdc.noaa.gov/pub/data/noaa/")
+except Exception:
+    pytest.skip("NCEI server issues", allow_module_level=True)
+
 
 def test_ish_read_history():
     dates = pd.date_range("2020-09-01", "2020-09-02")

tests/test_ish_lite.py

@@ -3,6 +3,13 @@
 
 from monetio import ish_lite
 
+try:
+    import requests
+
+    r = requests.head("https://www1.ncdc.noaa.gov/pub/data/noaa/isd-lite/")
+except Exception:
+    pytest.skip("NCEI server issues", allow_module_level=True)
+
 
 def test_ish_read_history():
     dates = pd.date_range("2020-09-01", "2020-09-02")