From 6ba60ff4a7869ef33ba017456b7607a9ea58f023 Mon Sep 17 00:00:00 2001
From: ChidiebereNjoku <144056487+ChidiebereNjoku@users.noreply.github.com>
Date: Tue, 18 Aug 2026 09:38:56 +0000
Subject: [PATCH] Add probability calibration visualization
Signed-off-by: ChidiebereNjoku <144056487+ChidiebereNjoku@users.noreply.github.com>
---
docs/extras/sitemap.xml | 1 +
docs/interpret/_toc.yml | 1 +
.../python/api/CalibrationCurve.ipynb | 25 +++
.../interpret-core/interpret/perf/__init__.py | 2 +-
.../interpret-core/interpret/perf/_curve.py | 200 +++++++++++++++++-
.../interpret-core/interpret/visual/plot.py | 50 +++++
python/interpret-core/tests/perf/__init__.py | 2 +
.../interpret-core/tests/perf/test_curve.py | 77 +++++++
python/interpret-core/tests/tutils.py | 11 +-
9 files changed, 362 insertions(+), 7 deletions(-)
create mode 100644 docs/interpret/python/api/CalibrationCurve.ipynb
create mode 100644 python/interpret-core/tests/perf/__init__.py
create mode 100644 python/interpret-core/tests/perf/test_curve.py
diff --git a/docs/extras/sitemap.xml b/docs/extras/sitemap.xml
index 3265d1deb..60a25fd70 100644
--- a/docs/extras/sitemap.xml
+++ b/docs/extras/sitemap.xml
@@ -19,6 +19,7 @@
https://interpret.ml/docs/python/api/RegressionPerf.html
https://interpret.ml/docs/python/api/ROC.html
https://interpret.ml/docs/python/api/PR.html
+https://interpret.ml/docs/python/api/CalibrationCurve.html
https://interpret.ml/docs/python/api/ClassHistogram.html
https://interpret.ml/docs/python/api/Marginal.html
https://interpret.ml/docs/python/api/DPExplainableBoostingClassifier.html
diff --git a/docs/interpret/_toc.yml b/docs/interpret/_toc.yml
index 55425c2d5..0ca90324c 100644
--- a/docs/interpret/_toc.yml
+++ b/docs/interpret/_toc.yml
@@ -28,6 +28,7 @@ chapters:
- file: python/api/RegressionPerf
- file: python/api/ROC
- file: python/api/PR
+ - file: python/api/CalibrationCurve
- file: python/api/interpret-data
sections:
- file: python/api/ClassHistogram
diff --git a/docs/interpret/python/api/CalibrationCurve.ipynb b/docs/interpret/python/api/CalibrationCurve.ipynb
new file mode 100644
index 000000000..bb4a39dea
--- /dev/null
+++ b/docs/interpret/python/api/CalibrationCurve.ipynb
@@ -0,0 +1,25 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "api785f6",
+ "metadata": {},
+ "source": [
+ "# CalibrationCurve\n",
+ "\n",
+ "```{eval-rst}\n",
+ ".. autoclass:: interpret.perf.CalibrationCurve\n",
+ " :members:\n",
+ " :inherited-members:\n",
+ "```"
+ ]
+ }
+ ],
+ "metadata": {
+ "language_info": {
+ "name": "python"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/python/interpret-core/interpret/perf/__init__.py b/python/interpret-core/interpret/perf/__init__.py
index bf2cb4f1e..6e5ecd9f8 100644
--- a/python/interpret-core/interpret/perf/__init__.py
+++ b/python/interpret-core/interpret/perf/__init__.py
@@ -1,5 +1,5 @@
# Copyright (c) 2023 The InterpretML Contributors
# Distributed under the MIT software license
-from ._curve import PR, ROC # noqa: F401
+from ._curve import PR, ROC, CalibrationCurve # noqa: F401
from ._regression import RegressionPerf # noqa: F401
diff --git a/python/interpret-core/interpret/perf/_curve.py b/python/interpret-core/interpret/perf/_curve.py
index 962687acc..46bd255c3 100644
--- a/python/interpret-core/interpret/perf/_curve.py
+++ b/python/interpret-core/interpret/perf/_curve.py
@@ -2,10 +2,11 @@
# Distributed under the MIT software license
from itertools import count
+from numbers import Integral
import numpy as np
-from ..core.base import PerfExplainer, BaseExplanation
+from ..core.base import BaseExplanation, PerfExplainer
from ..utils._clean_simple import clean_dimensions, typify_classification
from ..utils._clean_x import preclean_X
from ..utils._explanation import gen_name_from_class
@@ -74,7 +75,7 @@ def explain_perf(self, X, y, name=None):
scores = predict_fn(X)
try:
- from sklearn.metrics import precision_recall_curve, average_precision_score
+ from sklearn.metrics import average_precision_score, precision_recall_curve
except ImportError:
raise ImportError(
"scikit-learn is required for PR curves. Install it with: pip install scikit-learn"
@@ -165,7 +166,7 @@ def explain_perf(self, X, y, name=None):
scores = predict_fn(X)
try:
- from sklearn.metrics import roc_curve, auc
+ from sklearn.metrics import auc, roc_curve
except ImportError:
raise ImportError(
"scikit-learn is required for ROC curves. Install it with: pip install scikit-learn"
@@ -196,6 +197,131 @@ def explain_perf(self, X, y, name=None):
)
+class CalibrationCurve(PerfExplainer):
+ """Produces probability calibration curves."""
+
+ def __init__(
+ self,
+ model,
+ feature_names=None,
+ feature_types=None,
+ n_bins=5,
+ strategy="uniform",
+ ):
+ """Initializes class.
+
+ Args:
+ model: Model or prediction function of model.
+ feature_names: List of feature names.
+ feature_types: List of feature types.
+ n_bins: Number of bins used to calculate the calibration curve.
+ strategy: Strategy used to define bin widths. Either "uniform" or
+ "quantile".
+ """
+ if not isinstance(n_bins, Integral) or isinstance(n_bins, bool) or n_bins < 1:
+ raise ValueError("n_bins must be a positive integer.")
+ if strategy not in ("uniform", "quantile"):
+ raise ValueError('strategy must be either "uniform" or "quantile".')
+
+ self.model = model
+ self.feature_names = feature_names
+ self.feature_types = feature_types
+ self.n_bins = int(n_bins)
+ self.strategy = strategy
+
+ def explain_perf(self, X, y, name=None):
+ """Produce a probability calibration curve.
+
+ Args:
+ X: NumPy array for X to compare predict function against.
+ y: NumPy vector for y to compare predict function against.
+ name: User-defined explanation name.
+
+ Returns:
+ An explanation object.
+ """
+ if name is None:
+ name = gen_name_from_class(self)
+
+ y = clean_dimensions(y, "y")
+ if y.ndim != 1:
+ msg = (
+ f"y must be 1 dimensional, but got {y.ndim} dimensions "
+ f"with shape {y.shape}"
+ )
+ raise ValueError(msg)
+
+ X, n_samples = preclean_X(X, self.feature_names, self.feature_types, len(y))
+
+ predict_fn, n_classes, classes = determine_classes(self.model, X, n_samples)
+ if n_classes != 2:
+ msg = (
+ "Only binary classification is supported by the CalibrationCurve "
+ "class. The model must have exactly 2 classes."
+ )
+ raise ValueError(msg)
+ predict_fn = unify_predict_fn(predict_fn, X, 1)
+
+ X, feature_names, feature_types = unify_data(
+ X, n_samples, self.feature_names, self.feature_types, True, 0
+ )
+
+ y = typify_classification(y)
+ if classes is None:
+ # scikit-learn requires that classes are sorted with np.unique
+ classes, y = np.unique(y, return_inverse=True)
+ if len(classes) != n_classes:
+ msg = (
+ f"Class count mismatch: model predicted {n_classes} classes "
+ f"but y contains {len(classes)} unique values"
+ )
+ raise ValueError(msg)
+ else:
+ invert_classes = dict(zip(classes, count()))
+ y = np.array([invert_classes[el] for el in y], dtype=np.int64)
+
+ scores = np.asarray(predict_fn(X), dtype=np.float64)
+
+ try:
+ from sklearn.calibration import calibration_curve
+ except ImportError:
+ raise ImportError(
+ "scikit-learn is required for calibration curves. "
+ "Install it with: pip install scikit-learn"
+ )
+
+ prob_true, prob_pred = calibration_curve(
+ y,
+ scores,
+ n_bins=self.n_bins,
+ strategy=self.strategy,
+ )
+ counts, values = np.histogram(
+ scores,
+ bins=self.n_bins,
+ range=(0.0, 1.0),
+ )
+
+ overall_dict = {
+ "type": "perf_curve",
+ "density": {"names": values, "scores": counts},
+ "scores": scores,
+ "x_values": prob_pred,
+ "y_values": prob_true,
+ "n_bins": self.n_bins,
+ "strategy": self.strategy,
+ }
+ internal_obj = {"overall": overall_dict, "specific": None}
+
+ return CalibrationCurveExplanation(
+ "perf",
+ internal_obj,
+ feature_names=feature_names,
+ feature_types=feature_types,
+ name=name,
+ )
+
+
class ROCExplanation(BaseExplanation):
"""Explanation object specific to ROC explainer."""
@@ -338,3 +464,71 @@ def visualize(self, key=None):
title="PR Curve: " + self.name,
auc_prefix="Average Precision",
)
+
+
+class CalibrationCurveExplanation(BaseExplanation):
+ """Explanation object specific to probability calibration curves."""
+
+ explanation_type = None
+
+ def __init__(
+ self,
+ explanation_type,
+ internal_obj,
+ feature_names=None,
+ feature_types=None,
+ name=None,
+ selector=None,
+ ):
+ """Initializes class.
+
+ Args:
+ explanation_type: Type of explanation.
+ internal_obj: A jsonable object that backs the explanation.
+ feature_names: List of feature names.
+ feature_types: List of feature types.
+ name: User-defined name of explanation.
+ selector: A dict with "columns" and "data" keys whose entries
+ correspond to explanation entries.
+ """
+ self.explanation_type = explanation_type
+ self._internal_obj = internal_obj
+ self.feature_names = feature_names
+ self.feature_types = feature_types
+ self.name = name
+ self.selector = selector
+
+ def data(self, key=None):
+ """Provides specific explanation data.
+
+ Args:
+ key: A number/string that references a specific data item.
+
+ Returns:
+ A serializable dictionary.
+ """
+ if key is None:
+ return self._internal_obj["overall"]
+ return None
+
+ def visualize(self, key=None):
+ """Provides interactive visualizations.
+
+ Args:
+ key: Either a scalar or list that indexes the internal object for
+ sub-plotting. If an overall visualization is requested, pass
+ None.
+
+ Returns:
+ A Plotly figure.
+ """
+ from ..visual.plot import plot_calibration_curve
+
+ data_dict = self.data(key)
+ if data_dict is None:
+ return None
+
+ return plot_calibration_curve(
+ data_dict,
+ title="Calibration Curve: " + self.name,
+ )
diff --git a/python/interpret-core/interpret/visual/plot.py b/python/interpret-core/interpret/visual/plot.py
index 61f7b69ba..f96d8bb94 100644
--- a/python/interpret-core/interpret/visual/plot.py
+++ b/python/interpret-core/interpret/visual/plot.py
@@ -98,6 +98,56 @@ def plot_performance_curve(
return figure
+def plot_calibration_curve(data_dict, title=""):
+ x_values = data_dict["x_values"]
+ y_values = data_dict["y_values"]
+
+ width = 2
+
+ calibration_trace = go.Scatter(
+ x=x_values,
+ y=y_values,
+ mode="lines+markers",
+ hovertemplate=(
+ "Mean predicted probability: %{x:.3f}
"
+ "Fraction of positives: %{y:.3f}"
+ ),
+ line={"color": "darkorange", "width": width},
+ marker={"color": "darkorange"},
+ name="Calibration",
+ showlegend=False,
+ )
+ baseline_trace = go.Scatter(
+ x=[0, 1],
+ y=[0, 1],
+ mode="lines",
+ line={"color": "navy", "width": width, "dash": "dash"},
+ hoverinfo="skip",
+ name="Perfect calibration",
+ showlegend=False,
+ )
+
+ layout = go.Layout(
+ xaxis={"title": "Mean Predicted Probability", "range": [0.0, 1.0]},
+ yaxis={"title": "Fraction of Positives", "range": [0.0, 1.0]},
+ showlegend=False,
+ )
+ main_fig = go.Figure(
+ data=[calibration_trace, baseline_trace],
+ layout=layout,
+ )
+
+ density_fig = plot_density(data_dict["density"])
+ figure = _two_plot(main_fig, density_fig, title=title, share_xaxis=False)
+ figure["layout"]["xaxis1"].update(
+ title="Mean Predicted Probability", range=[0.0, 1.0]
+ )
+ figure["layout"]["yaxis1"].update(title="Fraction of Positives", range=[0.0, 1.0])
+ figure["layout"]["xaxis2"].update(title="Predicted Probability")
+ figure["layout"]["yaxis2"].update(title="Density")
+ return figure
+
+
def plot_continuous_bar(
data_dict, multiclass=False, show_error=True, title=None, xtitle="", ytitle=""
):
diff --git a/python/interpret-core/tests/perf/__init__.py b/python/interpret-core/tests/perf/__init__.py
new file mode 100644
index 000000000..b677d5257
--- /dev/null
+++ b/python/interpret-core/tests/perf/__init__.py
@@ -0,0 +1,2 @@
+# Copyright (c) 2026 The InterpretML Contributors
+# Distributed under the MIT software license
diff --git a/python/interpret-core/tests/perf/test_curve.py b/python/interpret-core/tests/perf/test_curve.py
new file mode 100644
index 000000000..65631ec2e
--- /dev/null
+++ b/python/interpret-core/tests/perf/test_curve.py
@@ -0,0 +1,77 @@
+# Copyright (c) 2026 The InterpretML Contributors
+# Distributed under the MIT software license
+
+import numpy as np
+import pytest
+from interpret.perf import CalibrationCurve
+from sklearn.base import BaseEstimator, ClassifierMixin
+
+
+class BinaryProbabilityModel(ClassifierMixin, BaseEstimator):
+ classes_ = np.array([0, 1])
+
+ def predict_proba(self, X):
+ scores = np.asarray(X)[:, 0]
+ return np.column_stack((1.0 - scores, scores))
+
+
+def test_calibration_curve_data():
+ X = np.array([[0.1], [0.2], [0.8], [0.9]])
+ y = np.array([0, 0, 1, 1])
+
+ explanation = CalibrationCurve(
+ BinaryProbabilityModel(),
+ n_bins=2,
+ ).explain_perf(X, y)
+ data = explanation.data()
+
+ np.testing.assert_allclose(data["x_values"], [0.15, 0.85])
+ np.testing.assert_allclose(data["y_values"], [0.0, 1.0])
+ np.testing.assert_array_equal(data["density"]["scores"], [2, 2])
+ np.testing.assert_allclose(data["density"]["names"], [0.0, 0.5, 1.0])
+ assert data["n_bins"] == 2
+ assert data["strategy"] == "uniform"
+
+
+def test_calibration_curve_quantile_strategy():
+ X = np.array([[0.1], [0.2], [0.3], [0.9]])
+ y = np.array([0, 0, 1, 1])
+
+ explanation = CalibrationCurve(
+ BinaryProbabilityModel(),
+ n_bins=2,
+ strategy="quantile",
+ ).explain_perf(X, y)
+ data = explanation.data()
+
+ np.testing.assert_allclose(data["x_values"], [0.15, 0.6])
+ np.testing.assert_allclose(data["y_values"], [0.0, 1.0])
+ assert data["strategy"] == "quantile"
+
+
+def test_calibration_curve_visualize():
+ X = np.array([[0.1], [0.2], [0.8], [0.9]])
+ y = np.array([0, 0, 1, 1])
+
+ explanation = CalibrationCurve(
+ BinaryProbabilityModel(),
+ n_bins=2,
+ ).explain_perf(X, y, name="Test model")
+ figure = explanation.visualize()
+
+ assert len(figure.data) == 3
+ assert figure.data[0].mode == "lines+markers"
+ assert figure.data[0].name == "Calibration"
+ assert figure.data[1].name == "Perfect calibration"
+ assert figure.layout.title.text == "Calibration Curve: Test model"
+
+
+@pytest.mark.parametrize("n_bins", [0, -1, 1.5, True])
+def test_calibration_curve_rejects_invalid_n_bins(n_bins):
+ with pytest.raises(ValueError, match="positive integer"):
+ CalibrationCurve(BinaryProbabilityModel(), n_bins=n_bins)
+
+
+def test_calibration_curve_rejects_invalid_strategy():
+ with pytest.raises(ValueError, match="uniform"):
+ CalibrationCurve(BinaryProbabilityModel(), strategy="invalid")
diff --git a/python/interpret-core/tests/tutils.py b/python/interpret-core/tests/tutils.py
index 3e0911958..afb88c9e1 100644
--- a/python/interpret-core/tests/tutils.py
+++ b/python/interpret-core/tests/tutils.py
@@ -5,6 +5,7 @@
import dash.development.base_component as dash_base
import numpy as np
import pandas as pd
+import sklearn.datasets
from interpret.blackbox import (
PartialDependence,
)
@@ -19,18 +20,22 @@
)
# from ..blackbox import PermutationImportance
-from interpret.perf import PR, ROC, RegressionPerf
+from interpret.perf import PR, ROC, CalibrationCurve, RegressionPerf
from pandas.core.generic import NDFrame
from plotly import graph_objs as go
from sklearn.base import is_classifier
from sklearn.model_selection import train_test_split
-import sklearn.datasets
def get_all_explainers():
# True means run on classification. False means run on regression
data_explainer_classes = [(ClassHistogram, None), (Marginal, None)]
- perf_explainer_classes = [(ROC, True), (PR, True), (RegressionPerf, False)]
+ perf_explainer_classes = [
+ (ROC, True),
+ (PR, True),
+ (CalibrationCurve, True),
+ (RegressionPerf, False),
+ ]
model_explainer_classes = [
(ClassificationTree, True),
(RegressionTree, False),