jupyter notebook plots for parameter importance

README.md

@@ -2,7 +2,7 @@
 
 **NOTE** *This repository is waiting for adoption and there might be longer waiting times for answers.
 However, feel free to report bugs or ask questions in the issues.
-Last known working versions of major dependencies can be found [here](https://github.com/automl/issues/*
+Last known working versions of major dependencies can be found [here](https://github.com/automl/issues/).*
 
 ## Configuration Assessment, Visualization and Evaluation
 

cave/analyzer/configurator/parallel_coordinates.py

@@ -5,6 +5,9 @@
 import pandas as pd
 from ConfigSpace.configuration_space import ConfigurationSpace, Configuration
 from ConfigSpace.hyperparameters import NumericalHyperparameter, CategoricalHyperparameter
+from bokeh.io import output_notebook
+from bokeh.models.annotations import Title
+from bokeh.plotting import show
 from bokeh.embed import components
 from bokeh.layouts import column
 from bokeh.models import Div
@@ -235,3 +238,14 @@ def get_html(self, d=None, tooltip=None):
             result["tooltip"] = self.__doc__
             d["Parallel Coordinates"] = result
         return result
+
+    def get_jupyter(self):
+        bokeh_plots = self.plot_bokeh()
+        output_notebook()
+        if len(self.result) == 0:
+            if(bokeh_plots):
+                show(bokeh_plots)
+            return
+        for budget, bokeh_plot in bokeh_plots.items():
+            plot = column( Div(text=budget), bokeh_plot) # to add a title
+            show(plot)

cave/analyzer/parameter_importance/base_parameter_importance.py

@@ -109,3 +109,22 @@ def get_html(self, d=None, tooltip=None):
                                                       "and the whiskers are quartiles."}
 
         return super().get_html(d, tooltip)
+
+    def get_jupyter(self):
+        from IPython.core.display import HTML, display
+        import matplotlib.pyplot as plt
+        import matplotlib
+        from importlib import reload
+        matplotlib.use('nbAgg')  # GUI backend
+        matplotlib = reload(matplotlib)
+        for b, data in self.result['Importances Per Parameter'].items():
+            im_list = []
+            for component, _ in data.items():
+                if(component == "figure"):
+                    im_list += [plt.imread(f) for f in data[component]]
+            f, axes = plt.subplots(1, len(im_list), figsize = (6*len(im_list),5))
+            for img, ax in zip(im_list, axes):
+                ax.imshow(img)
+                ax.axis('off')
+            f.suptitle(b, fontsize=14)
+            plt.show()

cave/analyzer/parameter_importance/fanova.py

@@ -99,19 +99,54 @@ def parse_pairwise(p):
 
         return result
 
+    def _plot_img_lst(self, plt, img_lst, max_cols, sup_title):
+        import math
+        rows = math.ceil( len(img_lst)/max_cols )
+        figure, axes = plt.subplots(rows, max_cols, figsize = (5*max_cols,5*rows))
+        i = 0
+        for r in range(rows):
+            for c in range(max_cols):
+                if(rows > 1):
+                    if(i < len(img_lst)):
+                        axes[r, c].imshow(img_lst[i])
+                    axes[r, c].axis('off')
+                else:
+                    if(i < len(img_lst)):
+                        axes[c].imshow(img_lst[i])
+                    axes[c].axis('off')
+                i += 1
+
+        figure.suptitle(sup_title, fontsize=14)
+        return figure
+
     def get_jupyter(self):
-        from IPython.core.display import HTML, Image, display
-        for b, result in self.result.items():
-            error = self.result[b]['else'] if 'else' in self.result[b] else None
-            if error:
-                display(HTML(error))
-            else:
-                # Show table
-                display(HTML(self.result[b]["Importance"]["table"]))
-                # Show plots
-                display(*list([Image(filename=d["figure"]) for d in self.result[b]['Marginals'].values()]))
-                display(*list([Image(filename=d["figure"]) for d in self.result[b]['Pairwise Marginals'].values()]))
-                # While working for a prettier solution, this might be an option:
-                # display(HTML(figure_to_html([d["figure"] for d in self.result[b]['Marginals'].values()] +
-                #                            [d["figure"] for d in self.result[b]['Pairwise Marginals'].values()],
-                #                            max_in_a_row=3, true_break_between_rows=True)))
+        # from IPython.core.display import HTML, Image, display
+        # for b, result in self.result['Importances Per Parameter'].items():
+        #     display(HTML(result["Importance"]["table"]))
+        #     # Show plots
+        #     display(*list([Image(filename=d["figure"]) for d in result['Marginals'].values()]))
+        #     display(*list([Image(filename=d["figure"]) for d in result['Pairwise Marginals'].values()]))
+        # Reload matplotlib backend to avoid GUI blank plots
+        from IPython.core.display import HTML, display
+        import matplotlib
+        import matplotlib.pyplot as plt
+        from importlib import reload
+        matplotlib.use('nbAgg')  # GUI backend
+        matplotlib = reload(matplotlib)
+
+        for b, result in self.result['Importances Per Parameter'].items():
+            # Show Table
+            display(HTML(result["Importance"]["table"]))
+
+            # Show plots
+            marginals_lst, pairws_marginals_lst = [], []
+            marginals_lst = [plt.imread(hp['figure']) for hp in result['Marginals'].values()]
+            pairws_marginals_lst = [plt.imread(hp['figure']) for hp in result['Pairwise Marginals'].values()]
+
+            f_marginals = self._plot_img_lst(plt, marginals_lst,
+                                                max_cols = 2, 
+                                                sup_title = b + ": Marginals")
+            f_pairwise_marginals = self._plot_img_lst(plt, pairws_marginals_lst,\
+                                                        max_cols = 2, 
+                                                        sup_title = b + ": Pairwise Marginals")
+            plt.show()

cave/analyzer/parameter_importance/local_parameter_importance.py

@@ -27,16 +27,50 @@ def get_name(self):
     def postprocess(self, pimp, output_dir):
         param_imp = pimp.evaluator.evaluated_parameter_importance
         plots = OrderedDict()
-        for p, i in [(k, v) for k, v in sorted(param_imp.items(),
+        for p, _ in [(k, v) for k, v in sorted(param_imp.items(),
                                                key=operator.itemgetter(1), reverse=True)]:
             plots[p] = os.path.join(output_dir, 'lpi', p + '.png')
         return OrderedDict([
             (p, {'figure': path}) for p, path in plots.items()
         ])
 
     def get_jupyter(self):
+        # from IPython.core.display import HTML, display, Image
+        # for b, budget_dict in self.result['Importances Per Parameter'].items():
+        #     for plots, d in budget_dict.items():
+        #         if(plots != 'Interactive Plots'):
+        #             display(Image(filename = d["figure"]))
+        # Reload matplotlib backend to avoid GUI blank plots
         from IPython.core.display import HTML, display
-        display(HTML(figure_to_html(self.get_plots(), max_in_a_row=3, true_break_between_rows=True)))
+        import matplotlib
+        import matplotlib.pyplot as plt
+        from importlib import reload
+        import math
+        matplotlib.use('nbAgg')  # GUI backend
+        matplotlib = reload(matplotlib)
+
+        max_cols = 2
+        for b, data in self.result['Importances Per Parameter'].items():
+            img_lst = [ plt.imread( c['figure'] ) for c in data.values() if 'figure' in c ]
+            # Plot in grid
+            rows = math.ceil( len(img_lst)/max_cols )
+            figure, axes = plt.subplots(rows, max_cols, figsize = (5*max_cols,4*rows))
+            i = 0
+            for r in range(rows):
+                for c in range(max_cols):
+                    if(rows > 1):
+                        if(i < len(img_lst)):
+                            axes[r, c].imshow(img_lst[i])
+                        axes[r, c].axis('off')
+                    else:
+                        if(i < len(img_lst)):
+                            axes[c].imshow(img_lst[i])
+                        axes[c].axis('off')
+                    i += 1
+
+            figure.suptitle(b, fontsize=14)
+            plt.show()
+
         if self.runscontainer.analyzing_options['Parameter Importance'].getboolean('whisker_quantiles_plot'):
             output_notebook()
             show(self.plot_whiskers())

cave/plot/configurator_footprint.py

@@ -21,7 +21,7 @@
 from bokeh.models.widgets import CheckboxButtonGroup, RadioButtonGroup, Button, Div
 from bokeh.plotting import figure, ColumnDataSource
 from sklearn.decomposition import PCA
-from sklearn.manifold.mds import MDS
+from sklearn.manifold import MDS
 from sklearn.preprocessing import StandardScaler
 from smac.epm.rf_with_instances import RandomForestWithInstances
 from smac.runhistory.runhistory import RunHistory