{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# Comparing randomized search and grid search for hyperparameter estimation\n\n\nCompare randomized search and grid search for optimizing hyperparameters of a\nrandom forest.\nAll parameters that influence the learning are searched simultaneously\n(except for the number of estimators, which poses a time / quality tradeoff).\n\nThe randomized search and the grid search explore exactly the same space of\nparameters. The result in parameter settings is quite similar, while the run\ntime for randomized search is drastically lower.\n\nThe performance is slightly worse for the randomized search, though this\nis most likely a noise effect and would not carry over to a held-out test set.\n\nNote that in practice, one would not search over this many different parameters\nsimultaneously using grid search, but pick only the ones deemed most important.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "print(__doc__)\n\nimport numpy as np\n\nfrom time import time\nfrom scipy.stats import randint as sp_randint\n\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.model_selection import RandomizedSearchCV\nfrom sklearn.datasets import load_digits\nfrom sklearn.ensemble import RandomForestClassifier\n\n# get some data\ndigits = load_digits()\nX, y = digits.data, digits.target\n\n# build a classifier\nclf = RandomForestClassifier(n_estimators=20)\n\n\n# Utility function to report best scores\ndef report(results, n_top=3):\n    for i in range(1, n_top + 1):\n        candidates = np.flatnonzero(results['rank_test_score'] == i)\n        for candidate in candidates:\n            print(\"Model with rank: {0}\".format(i))\n            print(\"Mean validation score: {0:.3f} (std: {1:.3f})\".format(\n                  results['mean_test_score'][candidate],\n                  results['std_test_score'][candidate]))\n            print(\"Parameters: {0}\".format(results['params'][candidate]))\n            print(\"\")\n\n\n# specify parameters and distributions to sample from\nparam_dist = {\"max_depth\": [3, None],\n              \"max_features\": sp_randint(1, 11),\n              \"min_samples_split\": sp_randint(2, 11),\n              \"bootstrap\": [True, False],\n              \"criterion\": [\"gini\", \"entropy\"]}\n\n# run randomized search\nn_iter_search = 20\nrandom_search = RandomizedSearchCV(clf, param_distributions=param_dist,\n                                   n_iter=n_iter_search, cv=5)\n\nstart = time()\nrandom_search.fit(X, y)\nprint(\"RandomizedSearchCV took %.2f seconds for %d candidates\"\n      \" parameter settings.\" % ((time() - start), n_iter_search))\nreport(random_search.cv_results_)\n\n# use a full grid over all parameters\nparam_grid = {\"max_depth\": [3, None],\n              \"max_features\": [1, 3, 10],\n              \"min_samples_split\": [2, 3, 10],\n              \"bootstrap\": [True, False],\n              \"criterion\": [\"gini\", \"entropy\"]}\n\n# run grid search\ngrid_search = GridSearchCV(clf, param_grid=param_grid, cv=5)\nstart = time()\ngrid_search.fit(X, y)\n\nprint(\"GridSearchCV took %.2f seconds for %d candidate parameter settings.\"\n      % (time() - start, len(grid_search.cv_results_['params'])))\nreport(grid_search.cv_results_)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}