Probability Calibration for ML Models

from sklearn.calibration import CalibratedClassifierCV, calibration_curve
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.datasets import make_classification
import numpy as np

X, y = make_classification(n_samples=1000, n_features=20, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

base_model = GradientBoostingClassifier(n_estimators=100, random_state=42)
base_model.fit(X_train, y_train)

platt = CalibratedClassifierCV(GradientBoostingClassifier(n_estimators=100), method='sigmoid', cv=5)
isotonic = CalibratedClassifierCV(GradientBoostingClassifier(n_estimators=100), method='isotonic', cv=5)

for name, model in [('Base', base_model), ('Platt', platt), ('Isotonic', isotonic)]:
    if name != 'Base':
        model.fit(X_train, y_train)
    proba = model.predict_proba(X_test)[:, 1]
    fraction_pos, mean_pred = calibration_curve(y_test, proba, n_bins=10)
    brier = np.mean((proba - y_test) ** 2)
    print(f'{name:10} | Brier score: {brier:.4f}')