bug fixing, CI (#3)

 * add doctest labels seq.
 * using pip GCO
 * update for PyPi
 * update arg params
 * use @ci unset DISPLAY
 * update experiments
 * fix annot. for train binary segmentation
 * fix index for py3
 * move CI configs
 * code cleaning w. Codacy

.gitignore

@@ -10,7 +10,7 @@ __pycache__/
 # installed libraries
 libs/
 
-# temporary strucure
+# temporary structure
 data/
 build/
 output/

shippable.yml → .shippable.yml

@@ -32,9 +32,9 @@ install:
   - root=$PWD
   - gcc --version
   - mkdir libs
-  - cd $root/libs && git clone https://github.com/Borda/pyGCO.git
-  - cd $root/libs/pyGCO && pip install -r requirements.txt
-  - cd $root/libs/pyGCO && python setup.py install
+#  - cd $root/libs && git clone https://github.com/Borda/pyGCO.git
+#  - cd $root/libs/pyGCO && pip install -r requirements.txt
+#  - cd $root/libs/pyGCO && python setup.py install
 #  - rm -r -f $here/libs/pyGCO
   - cd $root
 

.travis.yml

@@ -28,9 +28,9 @@ before_install:
 
 install:
   - root=$PWD
-  - cd $root/libs && git clone https://github.com/Borda/pyGCO.git
-  - cd $root/libs/pyGCO && pip install -r requirements.txt
-  - cd $root/libs/pyGCO && python setup.py install
+#  - cd $root/libs && git clone https://github.com/Borda/pyGCO.git
+#  - cd $root/libs/pyGCO && pip install -r requirements.txt
+#  - cd $root/libs/pyGCO && python setup.py install
 #  - rm -r -f $here/libs/pyGCO
   - cd $root
 

MANIFEST.in

@@ -0,0 +1,2 @@
+include README.md
+include requirements.txt

README.md

@@ -160,14 +160,14 @@ We utilize (un)supervised segmentation according to given training examples or s
     python experiments_segmentation/run_segm_slic_model_graphcut.py \
        -list images/langerhans_islets/list_lang-isl_imgs-annot.csv \
        -imgs "images/langerhans_islets/image/*.jpg" \
-       -out results -n langIsl --nb_classes 3 --visual 1 --nb_jobs 2
+       -out results -n langIsl --nb_classes 3 --visual --nb_jobs 2
     ```
 * Perform **Supervised** segmentation with afterwards evaluation.
     ```
     python experiments_segmentation/run_segm_slic_classif_graphcut.py \
         -list images/drosophila_ovary_slice/list_imgs-annot-struct.csv \
         -imgs "images/drosophila_ovary_slice/image/*.jpg" \
-        -out results -n Ovary --img_type 2d_gray --visual 1 --nb_jobs 2
+        -out results -n Ovary --img_type 2d_gray --visual --nb_jobs 2
     ```
 * For both experiment you can evaluate segmentation results.
     ```
@@ -275,7 +275,7 @@ python setup.py install
     ```
 1. Evaluate your segmentation results to expert annotation.
     ```
-    python experiments_ovary_detect/run_ovary_segm_evaluation.py
+    python experiments_ovary_detect/run_ovary_segm_evaluation.py --visual
     ```
 1. In the end, cut individual segmented objects comes as minimal bounding box.
     ```
@@ -295,7 +295,7 @@ python setup.py install
 
 ## References
 
-For complete references see [BibTex](references.bib).
+For complete references see [BibTex](docs/references.bib).
 1. Borovec J., Svihlik J., Kybic J., Habart D. (2017). **Supervised and unsupervised segmentation using superpixels, model estimation, and Graph Cut.** SPIE Journal of Electronic Imaging 26(6), 061610, http://doi.org/10.1117/1.JEI.26.6.061610
 1. Borovec J., Kybic J., Nava R. (2017) **Detection and Localization of Drosophila Egg Chambers in Microscopy Images.** In: Wang Q., Shi Y., Suk HI., Suzuki K. (eds) Machine Learning in Medical Imaging. MLMI 2017. LNCS, vol 10541. Springer, Cham. http://doi.org/10.1007/978-3-319-67389-9_3
 1. Borovec J., Kybic J., Sugimoto, A. (2017). **Region growing using superpixels with learned shape prior.** SPIE Journal of Electronic Imaging 26(6),  061611, http://doi.org/10.1117/1.JEI.26.6.061611

_config.yml

@@ -1 +1,2 @@
+show_downloads: true
 theme: jekyll-theme-leap-day

circle.yml

@@ -15,8 +15,8 @@ dependencies:
     - sudo apt-get install python-tk
     - mkdir libs
   override:
-    - cd libs && git clone https://github.com/Borda/pyGCO.git
-    - cd libs/pyGCO && pip install -r requirements.txt && python setup.py install
+    # - cd libs && git clone https://github.com/Borda/pyGCO.git
+    # - cd libs/pyGCO && pip install -r requirements.txt && python setup.py install
     # - rm -r -f libs/pyGCO
     # - pip install --upgrade setuptools
     - pip install -r requirements.txt
@@ -35,32 +35,35 @@ test:
     - coverage report && coverage xml -o $CIRCLE_TEST_REPORTS/coverage.xml
 
     # ANNOTATION section
-    - python handling_annotations/run_image_color_quantization.py -imgs "images/drosophila_ovary_slice/segm_rgb/*.png"
-    - python handling_annotations/run_image_color_quantization.py -imgs "images/drosophila_ovary_slice/segm_rgb/*.png" -m position
-    - python handling_annotations/run_image_convert_label_color.py -imgs "images/drosophila_ovary_slice/segm/*.png" -out images/drosophila_ovary_slice/segm_rgb
-    - python handling_annotations/run_image_convert_label_color.py -imgs "images/drosophila_ovary_slice/segm_rgb/*.png" -out images/drosophila_ovary_slice/segm
-    - python handling_annotations/run_overlap_images_segms.py  -imgs "images/drosophila_ovary_slice/image/*.jpg" -segs images/drosophila_ovary_slice/segm -out results/overlap_ovary_segment
-    - python handling_annotations/run_segm_annot_inpaint.py -imgs "images/drosophila_ovary_slice/segm/*.png" --label 0
-    - python handling_annotations/run_segm_annot_relabel.py -imgs "images/drosophila_ovary_slice/center_levels/*.png" -out results/relabel_center_levels
+    - unset DISPLAY && python handling_annotations/run_image_color_quantization.py -imgs "images/drosophila_ovary_slice/segm_rgb/*.png"
+    - unset DISPLAY && python handling_annotations/run_image_color_quantization.py -imgs "images/drosophila_ovary_slice/segm_rgb/*.png" -m position
+    - unset DISPLAY && python handling_annotations/run_image_convert_label_color.py -imgs "images/drosophila_ovary_slice/segm/*.png" -out images/drosophila_ovary_slice/segm_rgb
+    - unset DISPLAY && python handling_annotations/run_image_convert_label_color.py -imgs "images/drosophila_ovary_slice/segm_rgb/*.png" -out images/drosophila_ovary_slice/segm
+    - unset DISPLAY && python handling_annotations/run_overlap_images_segms.py  -imgs "images/drosophila_ovary_slice/image/*.jpg" -segs images/drosophila_ovary_slice/segm -out results/overlap_ovary_segment
+    - unset DISPLAY && python handling_annotations/run_segm_annot_inpaint.py -imgs "images/drosophila_ovary_slice/segm/*.png" --label 0
+    - unset DISPLAY && python handling_annotations/run_segm_annot_relabel.py -imgs "images/drosophila_ovary_slice/center_levels/*.png" -out results/relabel_center_levels
 
     # SEGMENTATION section
-    - python experiments_segmentation/run_compute-stat_annot-segm.py
-    - python experiments_segmentation/run_segm_slic_model_graphcut.py --visual 1
-    - python experiments_segmentation/run_segm_slic_classif_graphcut.py --visual 1
+    - unset DISPLAY && python experiments_segmentation/run_compute-stat_annot-segm.py
+    - unset DISPLAY && python experiments_segmentation/run_segm_slic_model_graphcut.py --visual
+    - unset DISPLAY && python experiments_segmentation/run_segm_slic_classif_graphcut.py --visual
 
     # CENTER DETECT. section
-    - python experiments_ovary_centres/run_create_annotation.py
-    - python experiments_ovary_centres/run_center_candidate_training.py
-    - python experiments_ovary_centres/run_center_prediction.py
-    - python experiments_ovary_centres/run_center_clustering.py
-    - python experiments_ovary_centres/run_center_evaluation.py
+    - unset DISPLAY && python experiments_ovary_centres/run_create_annotation.py
+    - unset DISPLAY && python experiments_ovary_centres/run_center_candidate_training.py
+    - unset DISPLAY && python experiments_ovary_centres/run_center_prediction.py
+    - unset DISPLAY && python experiments_ovary_centres/run_center_clustering.py
+    - unset DISPLAY && python experiments_ovary_centres/run_center_evaluation.py
 
     # download MorphSnake
     - cd libs && git clone https://github.com/Borda/morph-snakes.git
     - cd libs/morph-snakes && pip install -r requirements.txt && python setup.py install
     # REGION GROWING section
-    - python experiments_ovary_detect/run_RG2Sp_estim_shape-models.py
-    - python experiments_ovary_detect/run_ovary_egg-segmentation.py -m ellipse_moments ellipse_ransac_mmt ellipse_ransac_crit GC_pixels-large GC_pixels-shape GC_slic-small GC_slic-shape rg2sp_greedy-single rg2sp_GC-mixture watershed_morph
-    - python experiments_ovary_detect/run_ovary_segm_evaluation.py
-    - python experiments_ovary_detect/run_export_user-annot-segm.py
-    - python experiments_ovary_detect/run_cut_segmented_objects.py
+    - unset DISPLAY && python experiments_ovary_detect/run_RG2Sp_estim_shape-models.py
+    - unset DISPLAY && python experiments_ovary_detect/run_ovary_egg-segmentation.py -m ellipse_moments ellipse_ransac_mmt ellipse_ransac_crit GC_pixels-large GC_pixels-shape GC_slic-small GC_slic-shape rg2sp_greedy-single rg2sp_GC-mixture watershed_morph
+    - unset DISPLAY && python experiments_ovary_detect/run_ovary_segm_evaluation.py --visual
+    - unset DISPLAY && python experiments_ovary_detect/run_export_user-annot-segm.py
+    - unset DISPLAY && python experiments_ovary_detect/run_cut_segmented_objects.py
+    - unset DISPLAY && python experiments_ovary_detect/run_ellipse_annot_match.py
+    - unset DISPLAY && python experiments_ovary_detect/run_ellipse_cut_scale.py
+    - unset DISPLAY && python experiments_ovary_detect/run_egg_swap_orientation.py

experiments_ovary_centres/run_center_candidate_training.py

@@ -39,7 +39,8 @@
 import matplotlib
 if os.environ.get('DISPLAY', '') == '':
     logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
+    matplotlib.use('Agg')
+
 import matplotlib.pyplot as plt
 
 sys.path += [os.path.abspath('.'), os.path.abspath('..')]  # Add path to root
@@ -120,7 +121,7 @@
 })
 
 
-def arg_parse_params(params=CENTER_PARAMS):
+def arg_parse_params(params):
     """
     SEE: https://docs.python.org/3/library/argparse.html
     :return: {str: str}, int
@@ -704,7 +705,7 @@ def load_df_paths(params):
     return df_paths, path_csv
 
 
-def main_train(params=CENTER_PARAMS):
+def main_train(params):
     """ PIPELINE for training
     0) load triplets or create triplets from path to images, annotations
     1) load precomputed data or compute them now
@@ -724,7 +725,7 @@ def main_train(params=CENTER_PARAMS):
 
     tl_expt.create_subfolders(params['path_expt'], LIST_SUBDIRS)
 
-    df_paths, path_csv = load_df_paths(params)
+    df_paths, _ = load_df_paths(params)
 
     path_dump_data = os.path.join(params['path_expt'], NAME_DUMP_TRAIN_DATA)
     if not os.path.isfile(path_dump_data) or FORCE_RECOMP_DATA:
@@ -745,7 +746,7 @@ def main_train(params=CENTER_PARAMS):
 
     # concentrate features, labels
     features, labels, sizes = seg_clf.convert_set_features_labels_2_dataset(
-        dict_features, dict_labels, drop_labels=[-1], balance=params['balance'])
+        dict_features, dict_labels, drop_labels=[-1], balance_type=params['balance'])
     # remove all bad values from features space
     features[np.isnan(features)] = 0
     features[np.isinf(features)] = -1
@@ -777,5 +778,5 @@ def main_train(params=CENTER_PARAMS):
 
 if __name__ == '__main__':
     logging.basicConfig(level=logging.DEBUG)
-    params = arg_parse_params()
+    params = arg_parse_params(CENTER_PARAMS)
     main_train(params)

experiments_ovary_centres/run_center_clustering.py

@@ -22,7 +22,8 @@
 import matplotlib
 if os.environ.get('DISPLAY', '') == '':
     logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
+    matplotlib.use('Agg')
+
 import matplotlib.pylab as plt
 
 sys.path += [os.path.abspath('.'), os.path.abspath('..')]  # Add path to root

experiments_ovary_centres/run_center_evaluation.py

@@ -30,7 +30,8 @@
 import matplotlib
 if os.environ.get('DISPLAY', '') == '':
     logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
+    matplotlib.use('Agg')
+
 import matplotlib.pyplot as plt
 
 sys.path += [os.path.abspath('.'), os.path.abspath('..')]  # Add path to root
@@ -178,7 +179,7 @@ def load_center_evaluate(idx_row, df_annot, path_annot, path_visu=None,
         dict_stat = compute_statistic_eggs_centres(dict_row, centres, labels,
                                                    mask_eggs, img, segm,
                                                    path_visu, col_prefix)
-    except:
+    except Exception:
         logging.error(traceback.format_exc())
         dict_stat = dict_row
     return dict_stat

experiments_ovary_centres/run_center_prediction.py

@@ -23,11 +23,6 @@
 import tqdm
 import pandas as pd
 
-import matplotlib
-if os.environ.get('DISPLAY', '') == '':
-    logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
-
 sys.path += [os.path.abspath('.'), os.path.abspath('..')] # Add path to root
 import segmentation.utils.experiments as tl_expt
 import segmentation.utils.data_io as tl_io
@@ -92,7 +87,7 @@ def load_compute_detect_centers(idx_row, params, classif=None, path_classif='',
 
         dict_center = run_clust.cluster_points_draw_export(dict_center, params,
                                                            path_output)
-    except:
+    except Exception:
         logging.error(traceback.format_exc())
     gc.collect()
     time.sleep(1)

experiments_ovary_centres/run_create_annotation.py

@@ -33,10 +33,6 @@
 from scipy import ndimage
 from skimage import morphology, measure, draw
 
-import matplotlib
-if os.environ.get('DISPLAY', '') == '':
-    logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
 
 sys.path += [os.path.abspath('.'), os.path.abspath('..')]  # Add path to root
 import run_center_candidate_training as run_train

experiments_ovary_detect/run_RG2Sp_estim_shape-models.py

@@ -16,11 +16,6 @@
 import pickle
 import argparse
 
-import matplotlib
-if os.environ.get('DISPLAY', '') == '':
-    logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
-
 from PIL import Image
 import numpy as np
 import pandas as pd

experiments_ovary_detect/run_cut_segmented_objects.py

@@ -16,11 +16,6 @@
 import multiprocessing as mproc
 from functools import partial
 
-import matplotlib
-if os.environ.get('DISPLAY', '') == '':
-    logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
-
 import tqdm
 import numpy as np
 from PIL import Image
@@ -38,7 +33,7 @@
 }
 
 
-def arg_parse_params(dict_paths=PATHS):
+def arg_parse_params(dict_paths):
     """
     SEE: https://docs.python.org/3/library/argparse.html
     :return: {str: str}, int
@@ -133,5 +128,5 @@ def main(dict_paths, padding=0, use_mask=False, bg_color=None,
 
 if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO)
-    dict_paths, args = arg_parse_params()
+    dict_paths, args = arg_parse_params(PATHS)
     main(dict_paths, args.padding, args.mask, args.background, args.nb_jobs)

experiments_ovary_detect/run_egg_swap_orientation.py

@@ -45,7 +45,7 @@ def perform_orientation_swap(path_img, path_out):
     """
     img, _ = tl_io.load_image_2d(path_img)
 
-    part = img.shape[1] / 3
+    part = int(img.shape[1] / 3)
     sel_mask = img[:, :, IMAGE_CHANNEL] > np.min(img[:, :, IMAGE_CHANNEL])
     norm_val = np.mean(img[sel_mask, IMAGE_CHANNEL])
     val_left = np.sum(img[:, :part, IMAGE_CHANNEL] > norm_val)

experiments_ovary_detect/run_export_user-annot-segm.py

@@ -26,7 +26,7 @@
 import matplotlib
 if os.environ.get('DISPLAY', '') == '':
     logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
+    matplotlib.use('Agg')
 
 import tqdm
 from PIL import Image
@@ -55,7 +55,7 @@
 FIGURE_SIZE = 12
 
 
-def arg_parse_params(params=PARAMS):
+def arg_parse_params(params):
     """
     SEE: https://docs.python.org/3/library/argparse.html
     :return: {str: str}, int
@@ -160,7 +160,7 @@ def figure_draw_annot_csv(fig, img, row_slice, subfig_size=FIGURE_SIZE):
 #         figure_image_adjustment(fig, img)
 #         fig.savefig(os.path.join(path_out, n_img + '_segm_user-auto.png'))
 #         plt.close(fig)
-#     except:
+#     except Exception:
 #         print 'error for:', n_img
 #         traceback.print_exc()
 
@@ -197,7 +197,7 @@ def export_figure(idx_row, df_slices_info, path_out):
         tl_visu.figure_image_adjustment(fig, img.shape)
         fig.savefig(os.path.join(path_out, img_name + '.png'))
         plt.close(fig)
-    except:
+    except Exception:
         logging.error('failed for: %s', img_name)
         logging.error(traceback.format_exc())
 

experiments_ovary_detect/run_ovary_egg-segmentation.py

@@ -38,7 +38,7 @@
 import matplotlib
 if os.environ.get('DISPLAY', '') == '':
     logging.warning('No display found. Using non-interactive Agg backend')
-matplotlib.use('Agg')
+    matplotlib.use('Agg')
 
 import tqdm
 from PIL import Image
@@ -758,7 +758,7 @@ def image_segmentation(idx_row, params, debug_export=DEBUG_EXPORT):
             # export also centers
             centers = tl_data.swap_coord_x_y(centers)
             tl_data.save_landmarks_csv(path_centre, centers)
-        except:
+        except Exception:
             logging.error('segment fail for "%s" via %s with \n %s',
                           name, method, traceback.format_exc())