{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Probability Calibration for 3-class classification\n\n\nThis example illustrates how sigmoid calibration changes predicted\nprobabilities for a 3-class classification problem. Illustrated is the\nstandard 2-simplex, where the three corners correspond to the three classes.\nArrows point from the probability vectors predicted by an uncalibrated\nclassifier to the probability vectors predicted by the same classifier after\nsigmoid calibration on a hold-out validation set. Colors indicate the true\nclass of an instance (red: class 1, green: class 2, blue: class 3).\n\nThe base classifier is a random forest classifier with 25 base estimators\n(trees). If this classifier is trained on all 800 training datapoints, it is\noverly confident in its predictions and thus incurs a large log-loss.\nCalibrating an identical classifier, which was trained on 600 datapoints, with\nmethod='sigmoid' on the remaining 200 datapoints reduces the confidence of the\npredictions, i.e., moves the probability vectors from the edges of the simplex\ntowards the center. This calibration results in a lower log-loss. Note that an\nalternative would have been to increase the number of base estimators which\nwould have resulted in a similar decrease in log-loss.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "print(__doc__)\n\n# Author: Jan Hendrik Metzen <jhm@informatik.uni-bremen.de>\n# License: BSD Style.\n\n\nimport matplotlib.pyplot as plt\n\nimport numpy as np\n\nfrom sklearn.datasets import make_blobs\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.calibration import CalibratedClassifierCV\nfrom sklearn.metrics import log_loss\n\nnp.random.seed(0)\n\n# Generate data\nX, y = make_blobs(n_samples=1000, n_features=2, random_state=42,\n                  cluster_std=5.0)\nX_train, y_train = X[:600], y[:600]\nX_valid, y_valid = X[600:800], y[600:800]\nX_train_valid, y_train_valid = X[:800], y[:800]\nX_test, y_test = X[800:], y[800:]\n\n# Train uncalibrated random forest classifier on whole train and validation\n# data and evaluate on test data\nclf = RandomForestClassifier(n_estimators=25)\nclf.fit(X_train_valid, y_train_valid)\nclf_probs = clf.predict_proba(X_test)\nscore = log_loss(y_test, clf_probs)\n\n# Train random forest classifier, calibrate on validation data and evaluate\n# on test data\nclf = RandomForestClassifier(n_estimators=25)\nclf.fit(X_train, y_train)\nclf_probs = clf.predict_proba(X_test)\nsig_clf = CalibratedClassifierCV(clf, method=\"sigmoid\", cv=\"prefit\")\nsig_clf.fit(X_valid, y_valid)\nsig_clf_probs = sig_clf.predict_proba(X_test)\nsig_score = log_loss(y_test, sig_clf_probs)\n\n# Plot changes in predicted probabilities via arrows\nplt.figure(0)\ncolors = [\"r\", \"g\", \"b\"]\nfor i in range(clf_probs.shape[0]):\n    plt.arrow(clf_probs[i, 0], clf_probs[i, 1],\n              sig_clf_probs[i, 0] - clf_probs[i, 0],\n              sig_clf_probs[i, 1] - clf_probs[i, 1],\n              color=colors[y_test[i]], head_width=1e-2)\n\n# Plot perfect predictions\nplt.plot([1.0], [0.0], 'ro', ms=20, label=\"Class 1\")\nplt.plot([0.0], [1.0], 'go', ms=20, label=\"Class 2\")\nplt.plot([0.0], [0.0], 'bo', ms=20, label=\"Class 3\")\n\n# Plot boundaries of unit simplex\nplt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label=\"Simplex\")\n\n# Annotate points on the simplex\nplt.annotate(r'($\\frac{1}{3}$, $\\frac{1}{3}$, $\\frac{1}{3}$)',\n             xy=(1.0/3, 1.0/3), xytext=(1.0/3, .23), xycoords='data',\n             arrowprops=dict(facecolor='black', shrink=0.05),\n             horizontalalignment='center', verticalalignment='center')\nplt.plot([1.0/3], [1.0/3], 'ko', ms=5)\nplt.annotate(r'($\\frac{1}{2}$, $0$, $\\frac{1}{2}$)',\n             xy=(.5, .0), xytext=(.5, .1), xycoords='data',\n             arrowprops=dict(facecolor='black', shrink=0.05),\n             horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($0$, $\\frac{1}{2}$, $\\frac{1}{2}$)',\n             xy=(.0, .5), xytext=(.1, .5), xycoords='data',\n             arrowprops=dict(facecolor='black', shrink=0.05),\n             horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($\\frac{1}{2}$, $\\frac{1}{2}$, $0$)',\n             xy=(.5, .5), xytext=(.6, .6), xycoords='data',\n             arrowprops=dict(facecolor='black', shrink=0.05),\n             horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($0$, $0$, $1$)',\n             xy=(0, 0), xytext=(.1, .1), xycoords='data',\n             arrowprops=dict(facecolor='black', shrink=0.05),\n             horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($1$, $0$, $0$)',\n             xy=(1, 0), xytext=(1, .1), xycoords='data',\n             arrowprops=dict(facecolor='black', shrink=0.05),\n             horizontalalignment='center', verticalalignment='center')\nplt.annotate(r'($0$, $1$, $0$)',\n             xy=(0, 1), xytext=(.1, 1), xycoords='data',\n             arrowprops=dict(facecolor='black', shrink=0.05),\n             horizontalalignment='center', verticalalignment='center')\n# Add grid\nplt.grid(\"off\")\nfor x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:\n    plt.plot([0, x], [x, 0], 'k', alpha=0.2)\n    plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)\n    plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)\n\nplt.title(\"Change of predicted probabilities after sigmoid calibration\")\nplt.xlabel(\"Probability class 1\")\nplt.ylabel(\"Probability class 2\")\nplt.xlim(-0.05, 1.05)\nplt.ylim(-0.05, 1.05)\nplt.legend(loc=\"best\")\n\nprint(\"Log-loss of\")\nprint(\" * uncalibrated classifier trained on 800 datapoints: %.3f \"\n      % score)\nprint(\" * classifier trained on 600 datapoints and calibrated on \"\n      \"200 datapoint: %.3f\" % sig_score)\n\n# Illustrate calibrator\nplt.figure(1)\n# generate grid over 2-simplex\np1d = np.linspace(0, 1, 20)\np0, p1 = np.meshgrid(p1d, p1d)\np2 = 1 - p0 - p1\np = np.c_[p0.ravel(), p1.ravel(), p2.ravel()]\np = p[p[:, 2] >= 0]\n\ncalibrated_classifier = sig_clf.calibrated_classifiers_[0]\nprediction = np.vstack([calibrator.predict(this_p)\n                        for calibrator, this_p in\n                        zip(calibrated_classifier.calibrators_, p.T)]).T\nprediction /= prediction.sum(axis=1)[:, None]\n\n# Plot modifications of calibrator\nfor i in range(prediction.shape[0]):\n    plt.arrow(p[i, 0], p[i, 1],\n              prediction[i, 0] - p[i, 0], prediction[i, 1] - p[i, 1],\n              head_width=1e-2, color=colors[np.argmax(p[i])])\n# Plot boundaries of unit simplex\nplt.plot([0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 1.0, 0.0], 'k', label=\"Simplex\")\n\nplt.grid(\"off\")\nfor x in [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]:\n    plt.plot([0, x], [x, 0], 'k', alpha=0.2)\n    plt.plot([0, 0 + (1-x)/2], [x, x + (1-x)/2], 'k', alpha=0.2)\n    plt.plot([x, x + (1-x)/2], [0, 0 + (1-x)/2], 'k', alpha=0.2)\n\nplt.title(\"Illustration of sigmoid calibrator\")\nplt.xlabel(\"Probability class 1\")\nplt.ylabel(\"Probability class 2\")\nplt.xlim(-0.05, 1.05)\nplt.ylim(-0.05, 1.05)\n\nplt.show()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}