Dev (#105)

* fix steps issues #48 and #49

* prepare sum of distances, not distances to 2 closest objects

* back to const erosion

* fixes

* fix

* [:-4] -> os.path.splitext()

* loss weighted by size of the object

* prepare masks, distances and sizes

* cleaning

* adapt models and loaders to handle size matrix and calculate size weights

* adapt models and loaders to handle size matrix and calculate size weights v2

* fix pipelines.py

* fix some issues with calculating size-weighted loss

* cleaning

* update mean and std

* fixes

* clean

* fix recall in evaluation

* fix bug in erosion (#91)

* Dev mosaic padding inference (#81)

* added mosaic seq, unet_mosaic pipe, mosaic loader

* added unet_weighted

* dropped input resize at inference

* dropped rescaling in loader, fixed postpro cropping

* local dev

* updated dilation/erosion, joined pipelines

* dropped unet mask saving

* added replication padding

* renamed mosaic->padded, moved params to configs

* padding->inference_padding

* config updates

* refactored padded unet

* refactored unet_padding

* Dev dice loss (#89)

* fix size weights

* add mixed dice + weighted ce loss

* fixes

* parametrize loss weights

* remove get_datagen function overriding

* dice loss per channel, some fixes

* fixes and smooth added to Dice loss instead of eps

* fixes and smooth added to Dice loss and eps, and parametrized

* sigmoid -> softmax in dice loss

* softmax2d

* move softmax to models.py

* parametrize softmax and sigmoid in dice loss

* Dev mask prep speed up (#94)

* distributed mask/distance/size generation added

* dropped deprecated

* dropped mask param

* Dev random crop (#97)

* local

* added random cropping, refactored augmentations

* Dev borders and dilation in preprocessing (#96)

* merge multithread

* preparing borders

* fix PR #96 and add update metadata generation

* Dev deeper archs (#102)

* dropped mask param

* added deeper resnets and spatial2d dropout

* updated config

* fixed casting

* updated index

* fix evaluate, add score builder in stream mode (#104)

* added initial version

* added simple evaluate on checkpoint script

* updated config

* added neptune file definition

* fixed conflicts

evaluate_checkpoint.py

@@ -0,0 +1,65 @@
+import yaml
+import subprocess
+import os
+
+import click
+
+MISSING_TRANSFORMERS = ['prediction_crop',
+                        'prediction_renamed',
+                        'mask_resize',
+                        'category_mapper',
+                        'mask_erosion',
+                        'labeler',
+                        'mask_dilation',
+                        'score_builder',
+                        'output']
+
+
+@click.group()
+def main():
+    pass
+
+
+@main.command()
+@click.option('-e', '--experiment_dir', help='experiment that you want to run evaluation on', required=True)
+@click.option('-t', '--temp_inference_dir', help='temporary directory', required=True)
+@click.option('-n', '--neptune_file', help='neptne file path', required=True)
+def run(temp_inference_dir, experiment_dir, neptune_file):
+    transformer_dir = os.path.join(temp_inference_dir, 'transformers')
+    checkpoints_dir = os.path.join(temp_inference_dir, 'checkpoints')
+
+    cmd = 'cp -rf {} {}'.format(experiment_dir, temp_inference_dir)
+    subprocess.call(cmd, shell=True)
+
+    cmd = 'cp {}/unet/best.torch {}/unet'.format(checkpoints_dir, transformer_dir)
+    subprocess.call(cmd, shell=True)
+
+    for missing_transformer in MISSING_TRANSFORMERS:
+        cmd = 'touch {}/{}'.format(transformer_dir, missing_transformer)
+        subprocess.call(cmd, shell=True)
+
+    cmd = 'cp {} temporary_neptune.yaml'.format(neptune_file, checkpoints_dir, transformer_dir)
+    subprocess.call(cmd, shell=True)
+
+    cmd = 'cp {} temporary_neptune.yaml'.format(neptune_file, checkpoints_dir, transformer_dir)
+    subprocess.call(cmd, shell=True)
+
+    with open("temporary_neptune.yaml", 'r+') as f:
+        doc = yaml.load(f)
+        doc['parameters']['experiment_dir'] = temp_inference_dir
+
+    with open("temporary_neptune.yaml", 'w+') as f:
+        yaml.dump(doc, f, default_flow_style=False)
+
+    cmd = 'neptune run --config temporary_neptune.yaml main.py -- evaluate -p unet_weighted_padded'
+    subprocess.call(cmd, shell=True)
+
+    cmd = 'rm temporary_neptune.yaml'
+    subprocess.call(cmd, shell=True)
+
+    cmd = 'rm -rf {}'.format(temp_inference_dir)
+    subprocess.call(cmd, shell=True)
+
+
+if __name__ == "__main__":
+    main()

main.py

@@ -53,7 +53,7 @@ def prepare_masks(dev_mode):
     for dataset in ["train", "val"]:
         logger.info('Overlaying masks, dataset: {}'.format(dataset))
         target_dir = "{}_eroded_{}_dilated_{}".format(params.masks_overlayed_dir[:-1],
-                                                    params.erode_selem_size, params.dilate_selem_size)
+                                                      params.erode_selem_size, params.dilate_selem_size)
         logger.info('Output directory: {}'.format(target_dir))
 
         overlay_masks(data_dir=params.data_dir,
@@ -249,9 +249,8 @@ def _generate_prediction(meta_data, pipeline, logger, category_ids):
     output = pipeline.transform(data)
     pipeline.clean_cache()
     y_pred = output['y_pred']
-    y_scores = output['y_scores']
 
-    prediction = create_annotations(meta_data, y_pred, y_scores, logger, category_ids)
+    prediction = create_annotations(meta_data, y_pred, logger, category_ids)
     return prediction
 
 
@@ -269,9 +268,8 @@ def _generate_prediction_in_chunks(meta_data, pipeline, logger, category_ids, ch
         output = pipeline.transform(data)
         pipeline.clean_cache()
         y_pred = output['y_pred']
-        y_scores = output['y_scores']
 
-        prediction_chunk = create_annotations(meta_chunk, y_pred, y_scores, logger, category_ids)
+        prediction_chunk = create_annotations(meta_chunk, y_pred, logger, category_ids)
         prediction.extend(prediction_chunk)
 
     return prediction

neptune.yaml

@@ -59,7 +59,7 @@ parameters:
   pool_kernel: 3
   pool_stride: 2
   repeat_blocks: 4
-  encoder: ResNet152
+  encoder: AlbuNet
 
 # U-Net loss weights (multi-output)
   bce_mask: 1.0
@@ -91,8 +91,8 @@ parameters:
   threshold: 0.5
   min_nuclei_size: 20
   erosion_percentages: '[10,20,30]'
-  erode_selem_size: 3
-  dilate_selem_size: 0
+  erode_selem_size: 2
+  dilate_selem_size: 3
 
 # Inference padding
   crop_image_h: 300

pipelines.py

@@ -78,8 +78,7 @@ def unet_padded(config, train_mode):
         output = Step(name='output',
                       transformer=Dummy(),
                       input_steps=[mask_postprocessed],
-                      adapter={'y_pred': ([(mask_postprocessed.name, 'images')]),
-                               'y_scores': ([(mask_postprocessed.name, 'scores')])
+                      adapter={'y_pred': ([(mask_postprocessed.name, 'images_with_scores')]),
                                },
                       cache_dirpath=config.env.cache_dirpath,
                       save_output=save_output)
@@ -246,7 +245,6 @@ def _preprocessing_multitask_generator(config, is_train, use_patching):
 
 
 def mask_postprocessing(loader, model, config, save_output=False):
-
     if config.postprocessor.crf.apply_crf:
         dense_crf = Step(name='dense_crf',
                          transformer=post.DenseCRFStream(**config.postprocessor.crf) if config.execution.stream_mode \

postprocessing.py

@@ -123,11 +123,10 @@ def transform(self, images):
 
 class ScoreBuilder(BaseTransformer):
     def transform(self, images, probabilities):
-        scores = []
+        images_with_scores = []
         for image, image_probabilities in tqdm(zip(images, probabilities)):
-            scores.append(build_score(image, image_probabilities))
-        return {'images': images,
-                'scores': scores}
+            images_with_scores.append((image, build_score(image, image_probabilities)))
+        return {'images_with_scores': images_with_scores}
 
 
 class MulticlassLabelerStream(BaseTransformer):
@@ -246,6 +245,15 @@ def _transform(self, images):
             yield crop_image_center_per_class(image, (self.h_crop, self.w_crop))
 
 
+class ScoreBuilderStream(BaseTransformer):
+    def transform(self, images, probabilities):
+        return {'images_with_scores': self._transform(images, probabilities)}
+
+    def _transform(self, images, probabilities):
+        for image, image_probabilities in tqdm(zip(images, probabilities)):
+            yield (image, build_score(image, image_probabilities))
+
+
 def label_multiclass_image(mask):
     labeled_channels = []
     for label_nr in range(0, mask.max() + 1):

utils.py

@@ -82,7 +82,7 @@ def decompose(labeled):
         return masks
 
 
-def create_annotations(meta, predictions, scores, logger, category_ids, save=False, experiment_dir='./'):
+def create_annotations(meta, predictions, logger, category_ids, save=False, experiment_dir='./'):
     '''
     :param meta: pd.DataFrame with metadata
     :param predictions: list of labeled masks or numpy array of size [n_images, im_height, im_width]
@@ -93,7 +93,7 @@ def create_annotations(meta, predictions, scores, logger, category_ids, save=Fal
     '''
     annotations = []
     logger.info('Creating annotations')
-    for image_id, prediction, image_scores in zip(meta["ImageId"].values, predictions, scores):
+    for image_id, (prediction, image_scores) in zip(meta["ImageId"].values, predictions):
         for category_nr, (category_instances, category_scores) in enumerate(zip(prediction, image_scores)):
             if category_ids[category_nr] != None:
                 masks = decompose(category_instances)
@@ -159,7 +159,7 @@ def _generate_metadata(dataset):
         if dataset != "test_images":
             images_path = os.path.join(images_path, "images")
 
-        if public_paths: # TODO: implement public generating public path
+        if public_paths:  # TODO: implement public generating public path
             raise NotImplementedError
         else:
             images_path_to_write = images_path
@@ -366,7 +366,7 @@ def coco_evaluation(gt_filepath, prediction_filepath, image_ids, category_ids, s
     cocoEval.accumulate()
     cocoEval.summarize()
 
-    return cocoEval.stats[0], cocoEval.stats[4]
+    return cocoEval.stats[0], cocoEval.stats[3]
 
 
 def label(mask):