ENH permutation importance

examples/plot_model_card.py

@@ -20,6 +20,7 @@
 from sklearn.datasets import load_breast_cancer
 from sklearn.ensemble import HistGradientBoostingClassifier
 from sklearn.experimental import enable_halving_search_cv  # noqa
+from sklearn.inspection import permutation_importance
 from sklearn.metrics import (
     ConfusionMatrixDisplay,
     accuracy_score,
@@ -148,6 +149,11 @@
 disp.figure_.savefig(Path(local_repo) / "confusion_matrix.png")
 model_card.add_plot(**{"Confusion matrix": "confusion_matrix.png"})
 
+importances = permutation_importance(model, X_test, y_test, n_repeats=10)
+model_card.add_permutation_importances(
+    importances, X_test.columns, "importance.png", "Permutation Importance"
+)
+
 cv_results = model.cv_results_
 clf_report = classification_report(
     y_test, y_pred, output_dict=True, target_names=["malignant", "benign"]

skops/card/_model_card.py

@@ -10,6 +10,7 @@
 from reprlib import Repr
 from typing import Any, Optional, Union
 
+import matplotlib.pyplot as plt
 from huggingface_hub import CardData, ModelCard
 from sklearn.utils import estimator_html_repr
 from tabulate import tabulate  # type: ignore
@@ -373,6 +374,47 @@ def add_metrics(self, **kwargs: str) -> "Card":
             self._eval_results[metric] = value
         return self
 
+    def add_permutation_importances(
+        self, feature_importances, columns, plot_file, plot_name
+    ) -> "Card":
+        """Plots permutation importance and saves it to model card.
+
+        Parameters
+        ----------
+        feature_importances : sklearn.utils.Bunch
+            Output of sklearn.inspection.permutation_importance()
+
+        columns :
+            Column names of the data used to generate importances.
+
+        plot_file :
+            Filename for the plot.
+
+        plot_name :
+            Name of the plot.
+
+        Returns
+        -------
+        self : object
+            Card object.
+        """
+        sorted_importances_idx = feature_importances.importances_mean.argsort()
+        fig, ax = plt.subplots()
+        ax.boxplot(
+            x=feature_importances.importances[sorted_importances_idx].T,
+            labels=columns[sorted_importances_idx],
+            vert=False,
+        )
+        ax.set_title(plot_name)
+        ax.set_xlabel("Decrease in Score")
+        if plot_name is not None and plot_file is not None:
+            plt.savefig(plot_file)
+            self.add_plot(**{plot_name: plot_file})
+        else:
+            plt.savefig("feature_importances.png")
+            self.add_plot(**{"Feature Importances": "feature_importances.png"})
+        return self
+
     def _generate_card(self) -> ModelCard:
         """Generate the ModelCard object
 

skops/card/tests/test_card.py

@@ -10,6 +10,7 @@
 import sklearn
 from huggingface_hub import CardData, metadata_load
 from sklearn.datasets import load_iris
+from sklearn.inspection import permutation_importance
 from sklearn.linear_model import LinearRegression, LogisticRegression
 
 import skops
@@ -169,6 +170,19 @@ def test_add_plot(destination_path, model_card):
     assert "![fig1](fig1.png)" in model_card
 
 
+def test_permutation_importances(
+    iris_estimator, iris_data, model_card, destination_path
+):
+    X, y = iris_data
+    result = permutation_importance(
+        iris_estimator, X, y, n_repeats=10, random_state=42, n_jobs=2
+    )
+    model_card.add_permutation_importances(
+        result, X.columns, "importance.png", "Permutation Importance"
+    )
+    assert "![Permutation Importance](importance.png)" in model_card.render()
+
+
 def test_temporary_plot(destination_path, model_card):
     # test if the additions are made to a temporary template file
     # and not to default template or template provided