YAML configs

* move configs JSON -> YAML
* update CircleCI
* fix expt. RG

MANIFEST.in

@@ -27,7 +27,7 @@ exclude pytest.*
 exclude */*.sh
 
 # Include the experiments
-recursive-include experiments_* *.py *.json
+recursive-include experiments_* *.py *.yml
 recursive-include handling_annotations *.py
 
 # Include the figures

README.md

@@ -177,14 +177,14 @@ We utilize (un)supervised segmentation according to given training examples or s
     ```bash
     python experiments_segmentation/run_segm_slic_model_graphcut.py \
        -l ./data_images/langerhans_islets/list_lang-isl_imgs-annot.csv -i "" \
-       --cdf experiments_segmentation/sample_config.json \
+       --cdf experiments_segmentation/sample_config.yml \
        -o ./results -n langIsl --nb_classes 3 --visual --nb_workers 2
     ```
     OR specified on particular path:
     ```bash
     python experiments_segmentation/run_segm_slic_model_graphcut.py \
        -l "" -i "./data_images/langerhans_islets/image/*.jpg" \
-       -cfg ./experiments_segmentation/sample_config.json \
+       -cfg ./experiments_segmentation/sample_config.yml \
        -o ./results -n langIsl --nb_classes 3 --visual --nb_workers 2
     ```
     ![unsupervised](figures/imag-disk-20_gmm.jpg)
@@ -193,7 +193,7 @@ We utilize (un)supervised segmentation according to given training examples or s
     python experiments_segmentation/run_segm_slic_classif_graphcut.py \
         -l ./data_images/drosophila_ovary_slice/list_imgs-annot-struct.csv \
         -i "./data_images/drosophila_ovary_slice/image/*.jpg" \
-        --path_config ./experiments_segmentation/sample_config.json \
+        --path_config ./experiments_segmentation/sample_config.yml \
         -o ./results -n Ovary --img_type 2d_split --visual --nb_workers 2
     ```
     ![supervised](figures/imag-disk-20_train.jpg)
@@ -208,20 +208,19 @@ We utilize (un)supervised segmentation according to given training examples or s
     ```
     ![vusial](figures/segm-visual_D03_sy04_100x.jpg)
 
-The previous two (un)segmentation accept [configuration file](experiments_segmentation/sample_config.json) (JSON) by parameter `-cfg` with some extra parameters which was not passed in arguments, for instance:
-```json
-{
-    "slic_size": 35,
-    "slic_regul": 0.2,
-    "features": {"color_hsv": ["mean", "std", "eng"]},
-    "classif": "SVM",
-    "nb_classif_search": 150,
-    "gc_edge_type": "model",
-    "gc_regul": 3.0,
-    "run_LOO": false,
-    "run_LPO": true,
-    "cross_val": 0.1
-}
+The previous two (un)segmentation accept [configuration file](experiments_segmentation/sample_config.yml) (YAML) by parameter `-cfg` with some extra parameters which was not passed in arguments, for instance:
+```yaml
+slic_size: 35,
+slic_regul: 0.2,
+features: 
+  color_hsv: ['mean', 'std', 'eng']
+classif: 'SVM'
+nb_classif_search: 150
+gc_edge_type: 'model'
+gc_regul: 3.0
+run_LOO: false
+run_LPO: true
+cross_val: 0.1
 ```
 
 ### Center detection and ellipse fitting

circle.yml

@@ -1,69 +1,75 @@
 version: 2.0
 
-jobs:
-  Py2:
-      docker:
-        - image: circleci/python:2.7
-      steps: &steps
-        - checkout
-        # INSTALLATION
-        - run:
-            name: Install Packages
-            command: |
-              sudo apt-get update
-              sudo apt-get install tk-dev pkg-config python-dev python-tk
-              sudo pip install --upgrade pip setuptools
-              sudo pip install nose coverage pytest pytest-cov flake8
-              sudo pip install -U backports.functools_lru_cache  # required for matplotlib @py2
-              pip install -r requirements.txt --user
-              pip list
-
-        # BUILDING
-        - run:
-            name: Building project
-            command: |
-              mkdir libs && mkdir output && mkdir results && mkdir test-reports
-              gcc --version ; python --version ; pwd ; ls -l
-              pip --version ; pip freeze
-              python setup.py build_ext --inplace
+references:
 
-        # TESTING
-        - run:
-            name: Testing and Formating
-            command: |
-              unset DISPLAY
-              coverage run --source imsegm -m py.test imsegm -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
-              python setup.py check -m -s
-              flake8 . --ignore=E402,E731 --max-line-length=100
+  install_pips: &install_pips
+    run:
+      name: Install PyPI dependences
+      command: |
+        sudo apt-get update
+        sudo apt-get install tk-dev pkg-config python-dev python-tk
+        sudo pip install --upgrade pip setuptools
+        sudo pip install nose coverage pytest pytest-cov flake8
+        sudo pip install -U backports.functools_lru_cache  # required for matplotlib @py2
+        pip install -r requirements.txt --user
+        pip list
 
-        # ANNOTATION section
-        - run: bash handling_annotations/test_annotations.sh
+  test_coverage: &test_coverage
+    run:
+      name: Testing and Formating
+      command: |
+        unset DISPLAY
+        coverage run --source imsegm -m py.test imsegm -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
+        python setup.py check -m -s
+        flake8 . --ignore=E402,E731 --max-line-length=100
 
-        # SEGMENTATION section
-        - run: bash experiments_segmentation/test_segmentations.sh
+  build_project: &build_project
+   run:
+     name: Building project
+     command: |
+       mkdir libs && mkdir output && mkdir results && mkdir test-reports
+       gcc --version ; python --version ; pwd ; ls -l
+       pip --version ; pip freeze
+       python setup.py build_ext --inplace
 
-        # CENTER DETECT. section
-        - run: bash experiments_ovary_centres/test_ovary_cebters.sh
+jobs:
+  Py2:
+    docker:
+      - image: circleci/python:2.7
+    steps: &steps
+      - checkout
+      # INSTALLATION
+      - *install_pips
+      # BUILDING
+      - *build_project
+      # TESTING
+      - *test_coverage
 
-        # REGION GROWING section
-        - run: bash experiments_ovary_detect/test_ovary_detect.sh
+      # ANNOTATION section
+      - run: bash handling_annotations/test_annotations.sh
+      # SEGMENTATION section
+      - run: bash experiments_segmentation/test_segmentations.sh
+      # CENTER DETECT. section
+      - run: bash experiments_ovary_centres/test_ovary_cebters.sh
+      # REGION GROWING section
+      - run: bash experiments_ovary_detect/test_ovary_detect.sh
 
-        # PASSING
-        - run:
-            name: Finalise
-            command: |
-              python setup.py install --user
-              coverage report && coverage xml -o test-reports/coverage.xml
+      # PASSING
+      - run:
+          name: Finalise
+          command: |
+            python setup.py install --user
+            coverage report && coverage xml -o test-reports/coverage.xml
 
-        - store_test_results:
-            path: test-reports
-        - store_artifacts:
-            path: test-reports
+      - store_test_results:
+          path: test-reports
+      - store_artifacts:
+          path: test-reports
 
   Py3:
-      docker:
-        - image: circleci/python:3.6
-      steps: *steps
+    docker:
+      - image: circleci/python:3.6
+    steps: *steps
 
 workflows:
   version: 2

experiments_ovary_centres/run_center_candidate_training.py

@@ -25,11 +25,11 @@
 import os
 import sys
 import logging
-import json
 import argparse
 import multiprocessing as mproc
 from functools import partial
 
+import yaml
 import tqdm
 import pandas as pd
 import numpy as np
@@ -69,7 +69,7 @@
 
 NAME_CSV_TRIPLES = 'list_images_segms_centers.csv'
 NAME_CSV_STAT_TRAIN = 'statistic_train_centers.csv'
-NAME_JSON_PARAMS = 'configuration.json'
+NAME_YAML_PARAMS = 'configuration.yaml'
 NAME_DUMP_TRAIN_DATA = 'dump_training_data.npz'
 
 NB_THREADS = max(1, int(mproc.cpu_count() * 0.9))
@@ -165,10 +165,11 @@ def arg_parse_params(params):
         assert os.path.exists(p_dir), 'missing (%s) %s' % (k, p_dir)
     # load saved configuration
     if params['path_config'] is not None:
-        assert os.path.splitext(params['path_config'])[-1] == '.json', \
+        ext = os.path.splitext(params['path_config'])[-1]
+        assert (ext == '.yaml' or ext == '.yml'), \
             'wrong extension for %s' % params['path_config']
         with open(params['path_config'], 'r') as fd:
-            data = json.load(fd)
+            data = yaml.load(fd)
         params.update(data)
     params.update(paths)
     logging.info('ARG PARAMETERS: \n %r', params)
@@ -708,8 +709,8 @@ def main_train(params):
     tl_expt.set_experiment_logger(params['path_expt'])
     logging.info(tl_expt.string_dict(params, desc='PARAMETERS'))
 
-    with open(os.path.join(params['path_expt'], NAME_JSON_PARAMS), 'w') as f:
-        json.dump(params, f)
+    with open(os.path.join(params['path_expt'], NAME_YAML_PARAMS), 'w') as fp:
+        yaml.dump(params, fp, default_flow_style=False)
 
     tl_expt.create_subfolders(params['path_expt'], LIST_SUBDIRS)
 

experiments_ovary_centres/run_center_clustering.py

@@ -9,10 +9,10 @@
 import os
 import sys
 import logging
-import json
 # import multiprocessing as mproc
 from functools import partial
 
+import yaml
 import pandas as pd
 import numpy as np
 from sklearn import cluster
@@ -40,6 +40,7 @@
 # subfigure size for visualisations
 MAX_FIGURE_SIZE = 12
 FOLDER_EXPERIMENT = 'detect-centers-predict_%s'
+NAME_YAML_PARAMS = 'config_clustering.yaml'
 
 # The asumtion is that the max distance is about 3 * sampling distance
 CLUSTER_PARAMS = {
@@ -194,9 +195,8 @@ def main(params):
     """
     params['path_expt'] = os.path.join(params['path_output'],
                                        FOLDER_EXPERIMENT % params['name'])
-    with open(os.path.join(params['path_expt'],
-                           'config_clustering.json'), 'w') as fp:
-        json.dump(params, fp)
+    with open(os.path.join(params['path_expt'], NAME_YAML_PARAMS), 'w') as fp:
+        yaml.dump(params, fp, default_flow_style=False)
 
     tl_expt.create_subfolders(params['path_expt'], LIST_SUBDIRS)
 

experiments_ovary_detect/run_ovary_egg-segmentation.py

@@ -24,7 +24,6 @@
 import os
 import sys
 import time
-import json
 import argparse
 import logging
 import pickle
@@ -36,6 +35,7 @@
     print('No display found. Using non-interactive Agg backend.')
     matplotlib.use('Agg')
 
+import yaml
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
@@ -91,9 +91,8 @@
 }
 COLUMNS_ELLIPSE = ('xc', 'yc', 'a', 'b', 'theta')
 
-PATH_DATA = tl_data.update_path('data', absolute=True)
-PATH_IMAGES = os.path.join(tl_data.update_path('data_images'),
-                           'drosophila_ovary_slice')
+PATH_DATA = tl_data.update_path('data_images', absolute=True)
+PATH_IMAGES = os.path.join(PATH_DATA, 'drosophila_ovary_slice')
 # sample segmentation methods
 LIST_SAMPLE_METHODS = (
     'ellipse_moments', 'ellipse_ransac_mmt', 'ellipse_ransac_crit',
@@ -154,10 +153,11 @@ def arg_parse_params(params):
         params['path_config'] = tl_data.update_path(params['path_config'])
         assert os.path.isfile(params['path_config']), \
             'missing file: %s' % params['path_config']
-        assert os.path.splitext(params['path_config'])[-1] == '.json', \
-            '"%s" should be JSON file' % params['path_config']
+        ext = os.path.splitext(params['path_config'])[-1]
+        assert (ext == '.yaml' or ext == '.yml'), \
+            '"%s" should be YAML file' % os.path.basename(params['path_config'])
         with open(params['path_config'], 'r') as fd:
-            data = json.load(fd)
+            data = yaml.load(fd)
         params.update(data)
         params.update(arg_params)
     for k in (k for k in arg_params if 'path' in k):
@@ -527,13 +527,12 @@ def segment_rg2sp_greedy(slic, seg, centers, labels_fg_prob, path_model,
         shape_model = pickle.load(open(path_model, 'rb'))
     dict_debug = dict() if os.path.isdir(debug_export) else None
 
-    slic_prob_fg = seg_rg.compute_segm_prob_fg(slic, seg, [0.1, 0.9])
+    slic_prob_fg = seg_rg.compute_segm_prob_fg(slic, seg, labels_fg_prob)
     labels_greedy = seg_rg.region_growing_shape_slic_greedy(
         slic, slic_prob_fg, centers, (shape_model['mix_model'], shape_model['cdfs']),
-        shape_model['name'], coef_shape, coef_pairwise,
-        prob_label_trans, greedy_tol=1e-1, allow_obj_swap=allow_obj_swap,
-        dict_thresholds=dict_thresholds, nb_iter=1000,
-        debug_history=dict_debug)
+        shape_model['name'], coef_shape=coef_shape, coef_pairwise=coef_pairwise,
+        prob_label_trans=prob_label_trans, greedy_tol=1e-1, allow_obj_swap=allow_obj_swap,
+        dict_thresholds=dict_thresholds, nb_iter=1000, debug_history=dict_debug)
 
     if dict_debug is not None:
         nb_iter = len(dict_debug['energy'])
@@ -560,10 +559,9 @@ def segment_rg2sp_graphcut(slic, seg, centers, labels_fg_prob, path_model,
     slic_prob_fg = seg_rg.compute_segm_prob_fg(slic, seg, labels_fg_prob)
     labels_gc = seg_rg.region_growing_shape_slic_graphcut(
         slic, slic_prob_fg, centers, (shape_model['mix_model'], shape_model['cdfs']),
-        shape_model['name'], coef_shape, coef_pairwise, prob_label_trans,
-        optim_global=True, allow_obj_swap=allow_obj_swap,
-        dict_thresholds=dict_thresholds, nb_iter=250,
-        debug_history=dict_debug)
+        shape_model['name'], coef_shape=coef_shape, coef_pairwise=coef_pairwise,
+        prob_label_trans=prob_label_trans, optim_global=True, allow_obj_swap=allow_obj_swap,
+        dict_thresholds=dict_thresholds, nb_iter=250, debug_history=dict_debug)
 
     if dict_debug is not None:
         nb_iter = len(dict_debug['energy'])

experiments_segmentation/run_segm_slic_classif_graphcut.py

@@ -20,7 +20,7 @@
 >> python run_segm_slic_classif_graphcut.py \
     -l data_images/drosophila_ovary_slice/list_imgs-annot-struct.csv \
     -i "data_images/drosophila_ovary_slice/image/*.jpg" \
-   --path_config experiments_segmentation/sample_config.json \
+   --path_config experiments_segmentation/sample_config.yml \
     -o results -n Ovary --img_type 2d_split --visual
 
 Copyright (C) 2016-2018 Jiri Borovec <jiri.borovec@fel.cvut.cz>

experiments_segmentation/run_segm_slic_model_graphcut.py

@@ -21,7 +21,6 @@
 
 import os
 import sys
-import json
 import glob
 import pickle
 import argparse
@@ -36,6 +35,7 @@
     print('No display found. Using non-interactive Agg backend.')
     matplotlib.use('Agg')
 
+import yaml
 from PIL import Image
 import numpy as np
 import pandas as pd
@@ -161,7 +161,7 @@ def arg_parse_params(params):
     # if the config path is set load the it otherwise use default
     if os.path.isfile(args.get('path_config', '')):
         with open(args['path_config'], 'r') as fd:
-            config = json.load(fd)
+            config = yaml.load(fd)
         params.update(config)
     params.update(args)
     return params
@@ -275,7 +275,7 @@ def export_visual(idx_name, img, segm, debug_visual=None,
     :param str path_visu: path to dir with debug images
     """
     logging.info('export results and visualization...')
-    if set(np.unique(segm)) <= set([0, 1]):
+    if set(np.unique(segm)) <= {0, 1}:
         segm *= 255
 
     path_img = os.path.join(path_out, str(idx_name) + '.png')