Fix multiple generate patches bugs (lanha#6)

* Fix import and data_path * Fix path to xml file * Add data_name to class * Handle unever array case in downsizing * Add data_name instance var * Format * Format * Add generate patches script * Add logger * Add sys * Add md file for patch gen * Add test imgs ids for L1C * Add test images L2A * Add test commands * Remove xtra - * Remove duplicated test_data * remove / * Fix linter issues * Fix duplicated code * Disable tuple unbalance error * Disable type for var input_shape
CarlosFora · Aug 14, 2020 · 574550a · 574550a
1 parent 8a27f35
commit 574550a
Show file tree

Hide file tree

Showing 11 changed files with 178 additions and 82 deletions.
diff --git a/S2L1C_test_img_ids.txt b/S2L1C_test_img_ids.txt
@@ -0,0 +1,15 @@
+S2B_MSIL1C_20200131T050029_N0208_R119_T45RTK_20200131T074329
+S2B_MSIL1C_20200101T082239_N0208_R121_T34HCJ_20200101T102510
+S2B_MSIL1C_20200103T024319_N0208_R003_T49JFM_20200103T054632
+S2B_MSIL1C_20200112T143449_N0208_R139_T23VMG_20200112T162018
+S2B_MSIL1C_20200116T155609_N0208_R054_T18TVK_20200116T192635
+S2B_MSIL1C_20200112T144659_N0208_R139_T19LED_20200112T181024
+S2B_MSIL1C_20200210T064019_N0209_R120_T43WEP_20200210T102521
+S2B_MSIL1C_20200105T132229_N0208_R038_T22JEP_20200105T140318
+S2A_MSIL1C_20200102T223701_N0208_R072_T59GLM_20200102T233015
+S2B_MSIL1C_20200117T102249_N0208_R065_T33UVA_20200117T113858
+S2B_MSIL1C_20200112T111329_N0208_R137_T30UWB_20200112T114220
+S2B_MSIL1C_20200131T050029_N0208_R119_T45RTK_20200131T074329
+S2A_MSIL1C_20200104T160641_N0208_R097_T16PET_20200104T193742
+S2A_MSIL1C_20200105T021051_N0208_R103_T52SEC_20200105T040308
+S2A_MSIL1C_20200104T074311_N0208_R092_T37NDF_20200104T092554
diff --git a/S2L2A_test_img_ids.txt b/S2L2A_test_img_ids.txt
@@ -0,0 +1,15 @@
+S2B_MSIL2A_20200805T044709_N0214_R076_T45RTK_20200805T090405
+S2B_MSIL2A_20200808T081609_N0214_R121_T34HDK_20200808T122511
+S2A_MSIL2A_20200805T024331_N0214_R003_T49JEN_20200805T042709
+S2B_MSIL2A_20200806T141739_N0214_R096_T23VNH_20200806T164315
+S2A_MSIL2A_20200805T153911_N0214_R011_T18TWK_20200805T200607
+S2B_MSIL2A_20200809T144729_N0214_R139_T19LFE_20200809T190255
+S2A_MSIL2A_20200806T064631_N0214_R020_T43WEP_20200806T083125
+S2A_MSIL2A_20200807T132241_N0214_R038_T22JFP_20200807T154317
+S2A_MSIL2A_20200809T223721_N0214_R072_T59GMN_20200810T002854
+S2A_MSIL2A_20200809T102031_N0214_R065_T33UWA_20200809T130506
+S2B_MSIL2A_20200809T110629_N0214_R137_T30UXC_20200809T140506
+S2B_MSIL2A_20200805T044709_N0214_R076_T45RTK_20200805T090405
+S2A_MSIL2A_20200804T160911_N0214_R140_T16PET_20200804T215526
+S2B_MSIL2A_20200804T015659_N0214_R060_T52SED_20200804T044656
+S2A_MSIL2A_20200801T073621_N0214_R092_T37NCE_20200801T102406
diff --git a/testing/demoDSen2.py b/testing/demoDSen2.py
@@ -13,7 +13,7 @@
 
 DATA_PATH = "../data/"
 
-
+# pylint: disable=unbalanced-tuple-unpacking
 def readh5(fname, im60=False, imGT=False):
     with h5py.File(DATA_PATH + fname, "r") as f:
         d10 = f["im10"][()].transpose()

diff --git a/testing/s2_tiles_supres.py b/testing/s2_tiles_supres.py
@@ -63,6 +63,7 @@ def __init__(self, data_file_path, clip_to_aoi, copy_original_bands, output_dir)
         self.clip_to_aoi = clip_to_aoi
         self.copy_original_bands = copy_original_bands
         self.output_dir = output_dir
+        self.data_name = os.path.basename(data_file_path)
 
         super().__init__(data_file_path)
 

diff --git a/tests/test_patches.py b/tests/test_patches.py
@@ -130,6 +130,10 @@ def test_downPixelAggr(dset_10, scale_20):
     r = patches.downPixelAggr(dset_10, scale_20)
     assert r.shape == (5490, 5490, 4)
 
+    dset_20_w = np.ones((5489, 5489, 6))
+    r = patches.downPixelAggr(dset_20_w, scale_20)
+    assert r.shape == (2744, 2744, 6)
+
 
 def test_recompose_images(dset_10, dset_20):
     p = patches.get_test_patches(dset_10, dset_20, 128, 8)

diff --git a/training/create_patches.py b/training/create_patches.py
@@ -7,10 +7,9 @@
 from typing import Tuple
 import numpy as np
 
-from utils.data_utils import DATA_UTILS, get_logger
-
+sys.path.append("..")
 
-sys.path.append("../")
+from utils.data_utils import DATA_UTILS, get_logger
 from utils.patches import (
     downPixelAggr,
     save_test_patches,
@@ -22,6 +21,58 @@
 LOGGER = get_logger(__name__)
 
 
+def parser_common(parser):
+    parser.add_argument(
+        "--test_data",
+        default=False,
+        action="store_true",
+        help="Store test patches in a separate dir.",
+    )
+    parser.add_argument(
+        "--rgb_images",
+        default=False,
+        action="store_true",
+        help=(
+            "If set, write PNG images for the original and the superresolved bands,"
+            " together with a composite rgb image (first three 10m bands), all with a "
+            "quick and dirty clipping to 99%% of the original bands dynamic range and "
+            "a quantization of the values to 256 levels."
+        ),
+    )
+    parser.add_argument(
+        "--save_prefix",
+        default="../data/",
+        help=(
+            "If set, speficies the name of a prefix for all output files. "
+            "Use a trailing / to save into a directory. The default of no prefix will "
+            "save into the current directory. Example: --save_prefix result/"
+        ),
+    )
+    parser.add_argument(
+        "--run_60",
+        default=False,
+        action="store_true",
+        help="If set, it will create patches also from the 60m channels.",
+    )
+    parser.add_argument(
+        "--true_data",
+        default=False,
+        action="store_true",
+        help=(
+            "If set, it will create patches for S2 without GT. This option is not "
+            "really useful here, please check the testing folder for predicting S2 images."
+        ),
+    )
+    parser.add_argument(
+        "--train_data",
+        default=False,
+        action="store_true",
+        help="Store train patches in a separate dir",
+    )
+    return parser
+
+
+# pylint: disable=unbalanced-tuple-unpacking
 class readS2fromFile(DATA_UTILS):
     def __init__(
         self,
@@ -41,8 +92,8 @@ def __init__(
         self.rgb_images = rgb_images
         self.run_60 = run_60
         self.true_data = true_data
-        self.test_data = test_data
         self.train_data = train_data
+        self.data_name = os.path.basename(data_file_path)
 
         super().__init__(data_file_path)
 
@@ -200,7 +251,7 @@ def saving_test_data(self, data10, data20, data60):
             self.save_band(
                 self.save_prefix,
                 data10_lr[:, :, 0:3],
-                "/test/" + self.data_name + "/RGB",
+                "test/" + self.data_name + "/RGB",
             )
         np.save(out_per_image + "no_tiling/" + "data10", data10_lr.astype(np.float32))
         np.save(out_per_image + "no_tiling/" + "data20", data20_lr.astype(np.float32))
@@ -230,9 +281,8 @@ def saving_true_data(self, data10, data20, data60):
         if not os.path.isdir(out_per_image):
             os.mkdir(out_per_image)
 
-        LOGGER.info(
-            f"Writing files for testing to:{out_per_image}"
-        )  # pylint: disable=logging-fstring-interpolation
+        # pylint: disable=logging-fstring-interpolation
+        LOGGER.info(f"Writing files for testing to:{out_per_image}")
         save_test_patches60(
             data10, data20, data60, out_per_image, patchSize=384, border=12
         )
@@ -290,66 +340,12 @@ def saving_train_data(self, data10, data20, data60):
             " or the S2A[...].xml file in a SAFE directory extracted from that ZIP."
         ),
     )
-    parser.add_argument(
-        "--clip_to_aoi",
-        default="",
-        help=(
-            "Sets the region of interest to extract as pixels locations on the 10m"
-            'bands. Use this syntax: x_1,y_1,x_2,y_2. E.g. --roi_x_y "2000,2000,3200,3200"'
-        ),
-    )
-    parser.add_argument(
-        "--test_data",
-        default=False,
-        action="store_true",
-        help="Store test patches in a separate dir.",
-    )
-    parser.add_argument(
-        "--rgb_images",
-        default=False,
-        action="store_true",
-        help=(
-            "If set, write PNG images for the original and the superresolved bands,"
-            " together with a composite rgb image (first three 10m bands), all with a "
-            "quick and dirty clipping to 99%% of the original bands dynamic range and "
-            "a quantization of the values to 256 levels."
-        ),
-    )
-    parser.add_argument(
-        "--save_prefix",
-        default="../data/",
-        help=(
-            "If set, speficies the name of a prefix for all output files. "
-            "Use a trailing / to save into a directory. The default of no prefix will "
-            "save into the current directory. Example: --save_prefix result/"
-        ),
-    )
-    parser.add_argument(
-        "--run_60",
-        default=False,
-        action="store_true",
-        help="If set, it will create patches also from the 60m channels.",
-    )
-    parser.add_argument(
-        "--true_data",
-        default=False,
-        action="store_true",
-        help=(
-            "If set, it will create patches for S2 without GT. This option is not "
-            "really useful here, please check the testing folder for predicting S2 images."
-        ),
-    )
-    parser.add_argument(
-        "--train_data",
-        default=False,
-        action="store_true",
-        help="Store train patches in a separate dir",
-    )
+    parser = parser_common(parser)
 
     args = parser.parse_args()
 
     LOGGER.info(
-        f"I will proceed with file {args.data_file}"
+        f"I will proceed with file {args.data_file_path}"
     )  # pylint: disable=logging-fstring-interpolation
     readS2fromFile(
         args.data_file_path,

diff --git a/training/generate_patches.md b/training/generate_patches.md
@@ -0,0 +1,16 @@
+# Create train datasets
+Place L1C and L2A data into train folder, and then run:
+```bash
+python generate_patches.py L1C --save_prefix "../data/l1c" --run_60 --train_data
+python generate_patches.py L1C --save_prefix "../data/l1c" --train_data
+python generate_patches.py L2A --save_prefix "../data/l2a" --run_60 --train_data
+python generate_patches.py L2A --save_prefix "../data/l2a" --train_data
+```
+
+# Create test datasets
+```bash
+python generate_patches.py L1C_test --save_prefix "../data/l1c" --run_60 --test_data
+python generate_patches.py L1C_test --save_prefix "../data/l1c" --test_data
+python generate_patches.py L2A_test --save_prefix "../data/l2a" --run_60 --test_data
+python generate_patches.py L2A_test --save_prefix "../data/l2a" --test_data
+```
diff --git a/training/generate_patches.py b/training/generate_patches.py
@@ -0,0 +1,48 @@
+import sys
+import argparse
+from pathlib import Path
+
+sys.path.append("..")
+
+from create_patches import readS2fromFile
+from create_patches import parser_common
+
+
+from utils.data_utils import get_logger
+
+LOGGER = get_logger(__name__)
+
+
+def arg_parse():
+    parser = argparse.ArgumentParser(
+        description="Read Sentinel-2 data. The code was adapted from N. Brodu.",
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+    )
+    parser.add_argument(
+        "data_folder_path", help=("Path to folder with S2 SAFE files."),
+    )
+    parser = parser_common(parser)
+    args = parser.parse_args()
+    return args
+
+
+def main(args):
+    # pylint: disable=logging-fstring-interpolation
+    LOGGER.info(f"I will proceed with file {args.data_folder_path}")
+
+    for file_path in Path(args.data_folder_path).glob("S2*"):
+        LOGGER.info(f"Processing {file_path}")
+        readS2fromFile(
+            str(file_path),
+            "",
+            args.save_prefix,
+            args.rgb_images,
+            args.run_60,
+            args.true_data,
+            args.test_data,
+            args.train_data,
+        ).process_patches()
+
+
+if __name__ == "__main__":
+    main(arg_parse())
diff --git a/training/supres_train.py b/training/supres_train.py
@@ -9,6 +9,7 @@
 import numpy as np
 
 import matplotlib as mpl
+import matplotlib.pyplot as plt
 
 mpl.use("Agg")
 from keras.optimizers import Nadam
@@ -40,6 +41,11 @@ class PlotLosses(Callback):
     def __init__(self, model_nr, lr):
         self.model_nr = model_nr
         self.lr = lr
+        self.losses = []
+        self.val_losses = []
+        self.i = 0
+        self.x = []
+        self.filename = out_path + self.model_nr + "_lr_{:.1e}.txt".format(self.lr)
 
     def on_train_begin(self, logs=None):
         self.losses = []
@@ -50,7 +56,6 @@ def on_train_begin(self, logs=None):
         open(self.filename, "w").close()
 
     def on_epoch_end(self, epoch, logs=None):
-        import matplotlib.pyplot as plt
 
         plt.ioff()
 
@@ -61,8 +66,8 @@ def on_epoch_end(self, epoch, logs=None):
         self.x.append(self.i)
         self.i += 1
         try:
-            with open(self.filename, "a") as self.f:
-                self.f.write(
+            with open(self.filename, "a") as f:
+                f.write(
                     "Finished epoch {:5d}: loss {:.3e}, valid: {:.3e}, lr: {:.1e}\n".format(
                         epoch, logs.get("loss"), logs.get("val_loss"), lr
                     )
@@ -141,9 +146,9 @@ def on_epoch_end(self, epoch, logs=None):
 
     # input_shape = ((4,32,32),(6,16,16))
     if args.run_60:
-        input_shape = ((4, None, None), (6, None, None), (2, None, None))
+        input_shape = ((4, None, None), (6, None, None), (2, None, None))  # type: ignore
     else:
-        input_shape = ((4, None, None), (6, None, None))
+        input_shape = ((4, None, None), (6, None, None))  # type: ignore
     # create model
     if args.deep:
         model = s2model(input_shape, num_layers=32, feature_size=256)

diff --git a/utils/data_utils.py b/utils/data_utils.py
@@ -58,20 +58,12 @@ def get_data(self) -> list:
         This method returns the raster data set of original image for
         all the available resolutions.
         """
-        # data_file_path is the path to the SAFE file.
-        if self.data_file_path.endswith("/"):
-            tmp = os.path.split(self.data_file_path)[0]
-            self.data_name = os.path.split(tmp)[1]
-        else:
-            self.data_name = os.path.split(self.data_file_path)[1]
-
-        data_path = ""
         data_folder = "MTD*.xml"
-        for file in glob.iglob(
-            os.path.join(self.data_name, data_folder), recursive=True,
-        ):
+        for file in glob.iglob(os.path.join(self.data_file_path, data_folder)):
             data_path = file
 
+        LOGGER.info(f"Data path is {data_path}")
+
         raster_data = rasterio.open(data_path)
         datasets = raster_data.subdatasets
 

diff --git a/utils/patches.py b/utils/patches.py
@@ -510,9 +510,13 @@ def downPixelAggr(img: np.ndarray, SCALE: int = 2) -> np.ndarray:
     img_lr = np.zeros(new_dims[0:2] + (img.shape[-1],))
     # Iterate through all the image channels with avg pooling (pixel aggregation)
     for i in range(0, img.shape[2]):
-        img_lr[:, :, i] = skimage.measure.block_reduce(
+        reduced = skimage.measure.block_reduce(
             img_blur[:, :, i], (SCALE, SCALE), np.mean
         )
+        img_lr_shape = img_lr[:, :, i].shape
+        if reduced.shape != img_lr_shape:
+            reduced = reduced[: img_lr_shape[0], : img_lr_shape[1]]
+        img_lr[:, :, i] = reduced
 
     return np.squeeze(img_lr)