Add probability calibration visualization

docs/extras/sitemap.xml

@@ -19,6 +19,7 @@
 <url><loc>https://interpret.ml/docs/python/api/RegressionPerf.html</loc></url>
 <url><loc>https://interpret.ml/docs/python/api/ROC.html</loc></url>
 <url><loc>https://interpret.ml/docs/python/api/PR.html</loc></url>
+<url><loc>https://interpret.ml/docs/python/api/CalibrationCurve.html</loc></url>
 <url><loc>https://interpret.ml/docs/python/api/ClassHistogram.html</loc></url>
 <url><loc>https://interpret.ml/docs/python/api/Marginal.html</loc></url>
 <url><loc>https://interpret.ml/docs/python/api/DPExplainableBoostingClassifier.html</loc></url>

docs/interpret/_toc.yml

@@ -28,6 +28,7 @@ chapters:
     - file: python/api/RegressionPerf
     - file: python/api/ROC
     - file: python/api/PR
+    - file: python/api/CalibrationCurve
   - file: python/api/interpret-data
     sections:
     - file: python/api/ClassHistogram

docs/interpret/python/api/CalibrationCurve.ipynb

@@ -0,0 +1,25 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "api785f6",
+   "metadata": {},
+   "source": [
+    "# CalibrationCurve\n",
+    "\n",
+    "```{eval-rst}\n",
+    ".. autoclass:: interpret.perf.CalibrationCurve\n",
+    "   :members:\n",
+    "   :inherited-members:\n",
+    "```"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

python/interpret-core/interpret/perf/__init__.py

@@ -1,5 +1,5 @@
 # Copyright (c) 2023 The InterpretML Contributors
 # Distributed under the MIT software license
 
-from ._curve import PR, ROC  # noqa: F401
+from ._curve import PR, ROC, CalibrationCurve  # noqa: F401
 from ._regression import RegressionPerf  # noqa: F401

python/interpret-core/interpret/perf/_curve.py

@@ -2,10 +2,11 @@
 # Distributed under the MIT software license
 
 from itertools import count
+from numbers import Integral
 
 import numpy as np
 
-from ..core.base import PerfExplainer, BaseExplanation
+from ..core.base import BaseExplanation, PerfExplainer
 from ..utils._clean_simple import clean_dimensions, typify_classification
 from ..utils._clean_x import preclean_X
 from ..utils._explanation import gen_name_from_class
@@ -74,7 +75,7 @@ def explain_perf(self, X, y, name=None):
         scores = predict_fn(X)
 
         try:
-            from sklearn.metrics import precision_recall_curve, average_precision_score
+            from sklearn.metrics import average_precision_score, precision_recall_curve
         except ImportError:
             raise ImportError(
                 "scikit-learn is required for PR curves. Install it with: pip install scikit-learn"
@@ -165,7 +166,7 @@ def explain_perf(self, X, y, name=None):
         scores = predict_fn(X)
 
         try:
-            from sklearn.metrics import roc_curve, auc
+            from sklearn.metrics import auc, roc_curve
         except ImportError:
             raise ImportError(
                 "scikit-learn is required for ROC curves. Install it with: pip install scikit-learn"
@@ -196,6 +197,131 @@ def explain_perf(self, X, y, name=None):
         )
 
 
+class CalibrationCurve(PerfExplainer):
+    """Produces probability calibration curves."""
+
+    def __init__(
+        self,
+        model,
+        feature_names=None,
+        feature_types=None,
+        n_bins=5,
+        strategy="uniform",
+    ):
+        """Initializes class.
+
+        Args:
+            model: Model or prediction function of model.
+            feature_names: List of feature names.
+            feature_types: List of feature types.
+            n_bins: Number of bins used to calculate the calibration curve.
+            strategy: Strategy used to define bin widths. Either "uniform" or
+                "quantile".
+        """
+        if not isinstance(n_bins, Integral) or isinstance(n_bins, bool) or n_bins < 1:
+            raise ValueError("n_bins must be a positive integer.")
+        if strategy not in ("uniform", "quantile"):
+            raise ValueError('strategy must be either "uniform" or "quantile".')
+
+        self.model = model
+        self.feature_names = feature_names
+        self.feature_types = feature_types
+        self.n_bins = int(n_bins)
+        self.strategy = strategy
+
+    def explain_perf(self, X, y, name=None):
+        """Produce a probability calibration curve.
+
+        Args:
+            X: NumPy array for X to compare predict function against.
+            y: NumPy vector for y to compare predict function against.
+            name: User-defined explanation name.
+
+        Returns:
+            An explanation object.
+        """
+        if name is None:
+            name = gen_name_from_class(self)
+
+        y = clean_dimensions(y, "y")
+        if y.ndim != 1:
+            msg = (
+                f"y must be 1 dimensional, but got {y.ndim} dimensions "
+                f"with shape {y.shape}"
+            )
+            raise ValueError(msg)
+
+        X, n_samples = preclean_X(X, self.feature_names, self.feature_types, len(y))
+
+        predict_fn, n_classes, classes = determine_classes(self.model, X, n_samples)
+        if n_classes != 2:
+            msg = (
+                "Only binary classification is supported by the CalibrationCurve "
+                "class. The model must have exactly 2 classes."
+            )
+            raise ValueError(msg)
+        predict_fn = unify_predict_fn(predict_fn, X, 1)
+
+        X, feature_names, feature_types = unify_data(
+            X, n_samples, self.feature_names, self.feature_types, True, 0
+        )
+
+        y = typify_classification(y)
+        if classes is None:
+            # scikit-learn requires that classes are sorted with np.unique
+            classes, y = np.unique(y, return_inverse=True)
+            if len(classes) != n_classes:
+                msg = (
+                    f"Class count mismatch: model predicted {n_classes} classes "
+                    f"but y contains {len(classes)} unique values"
+                )
+                raise ValueError(msg)
+        else:
+            invert_classes = dict(zip(classes, count()))
+            y = np.array([invert_classes[el] for el in y], dtype=np.int64)
+
+        scores = np.asarray(predict_fn(X), dtype=np.float64)
+
+        try:
+            from sklearn.calibration import calibration_curve
+        except ImportError:
+            raise ImportError(
+                "scikit-learn is required for calibration curves. "
+                "Install it with: pip install scikit-learn"
+            )
+
+        prob_true, prob_pred = calibration_curve(
+            y,
+            scores,
+            n_bins=self.n_bins,
+            strategy=self.strategy,
+        )
+        counts, values = np.histogram(
+            scores,
+            bins=self.n_bins,
+            range=(0.0, 1.0),
+        )
+
+        overall_dict = {
+            "type": "perf_curve",
+            "density": {"names": values, "scores": counts},
+            "scores": scores,
+            "x_values": prob_pred,
+            "y_values": prob_true,
+            "n_bins": self.n_bins,
+            "strategy": self.strategy,
+        }
+        internal_obj = {"overall": overall_dict, "specific": None}
+
+        return CalibrationCurveExplanation(
+            "perf",
+            internal_obj,
+            feature_names=feature_names,
+            feature_types=feature_types,
+            name=name,
+        )
+
+
 class ROCExplanation(BaseExplanation):
     """Explanation object specific to ROC explainer."""
 
@@ -338,3 +464,71 @@ def visualize(self, key=None):
             title="PR Curve: " + self.name,
             auc_prefix="Average Precision",
         )
+
+
+class CalibrationCurveExplanation(BaseExplanation):
+    """Explanation object specific to probability calibration curves."""
+
+    explanation_type = None
+
+    def __init__(
+        self,
+        explanation_type,
+        internal_obj,
+        feature_names=None,
+        feature_types=None,
+        name=None,
+        selector=None,
+    ):
+        """Initializes class.
+
+        Args:
+            explanation_type: Type of explanation.
+            internal_obj: A jsonable object that backs the explanation.
+            feature_names: List of feature names.
+            feature_types: List of feature types.
+            name: User-defined name of explanation.
+            selector: A dict with "columns" and "data" keys whose entries
+                correspond to explanation entries.
+        """
+        self.explanation_type = explanation_type
+        self._internal_obj = internal_obj
+        self.feature_names = feature_names
+        self.feature_types = feature_types
+        self.name = name
+        self.selector = selector
+
+    def data(self, key=None):
+        """Provides specific explanation data.
+
+        Args:
+            key: A number/string that references a specific data item.
+
+        Returns:
+            A serializable dictionary.
+        """
+        if key is None:
+            return self._internal_obj["overall"]
+        return None
+
+    def visualize(self, key=None):
+        """Provides interactive visualizations.
+
+        Args:
+            key: Either a scalar or list that indexes the internal object for
+                sub-plotting. If an overall visualization is requested, pass
+                None.
+
+        Returns:
+            A Plotly figure.
+        """
+        from ..visual.plot import plot_calibration_curve
+
+        data_dict = self.data(key)
+        if data_dict is None:
+            return None
+
+        return plot_calibration_curve(
+            data_dict,
+            title="Calibration Curve: " + self.name,
+        )

python/interpret-core/interpret/visual/plot.py

@@ -98,6 +98,56 @@ def plot_performance_curve(
     return figure
 
 
+def plot_calibration_curve(data_dict, title=""):
+    x_values = data_dict["x_values"]
+    y_values = data_dict["y_values"]
+
+    width = 2
+
+    calibration_trace = go.Scatter(
+        x=x_values,
+        y=y_values,
+        mode="lines+markers",
+        hovertemplate=(
+            "Mean predicted probability: %{x:.3f}<br>"
+            "Fraction of positives: %{y:.3f}<extra></extra>"
+        ),
+        line={"color": "darkorange", "width": width},
+        marker={"color": "darkorange"},
+        name="Calibration",
+        showlegend=False,
+    )
+    baseline_trace = go.Scatter(
+        x=[0, 1],
+        y=[0, 1],
+        mode="lines",
+        line={"color": "navy", "width": width, "dash": "dash"},
+        hoverinfo="skip",
+        name="Perfect calibration",
+        showlegend=False,
+    )
+
+    layout = go.Layout(
+        xaxis={"title": "Mean Predicted Probability", "range": [0.0, 1.0]},
+        yaxis={"title": "Fraction of Positives", "range": [0.0, 1.0]},
+        showlegend=False,
+    )
+    main_fig = go.Figure(
+        data=[calibration_trace, baseline_trace],
+        layout=layout,
+    )
+
+    density_fig = plot_density(data_dict["density"])
+    figure = _two_plot(main_fig, density_fig, title=title, share_xaxis=False)
+    figure["layout"]["xaxis1"].update(
+        title="Mean Predicted Probability", range=[0.0, 1.0]
+    )
+    figure["layout"]["yaxis1"].update(title="Fraction of Positives", range=[0.0, 1.0])
+    figure["layout"]["xaxis2"].update(title="Predicted Probability")
+    figure["layout"]["yaxis2"].update(title="Density")
+    return figure
+
+
 def plot_continuous_bar(
     data_dict, multiclass=False, show_error=True, title=None, xtitle="", ytitle=""
 ):

python/interpret-core/tests/perf/__init__.py

@@ -0,0 +1,2 @@
+# Copyright (c) 2026 The InterpretML Contributors
+# Distributed under the MIT software license