hyperopt-xgb-skl.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.model_selection import (cross_val_score, train_test_split,\n",
    "                                    GridSearchCV, RandomizedSearchCV)\n",
    "from sklearn.preprocessing import Imputer\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# SEED = 42 # Initial results were performed with a SEED of 42, but let's change things up.\n",
    "SEED = 25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/train_final.csv')\n",
    "test = pd.read_csv('data/test_final.csv')\n",
    "train = train.drop(['id'], axis=1)\n",
    "test = test.drop(['id'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Divide dataset into X and y\n",
    "y = train.Y\n",
    "X = train.drop([\"Y\"], axis=1)\n",
    "X_test = test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Impute missing features\n",
    "from sklearn.preprocessing import Imputer\n",
    "\n",
    "# train = train.apply(lambda x: x.fillna(x.value_counts().index[0]))\n",
    "imp = Imputer(missing_values='NaN', strategy='mean', axis=0)\n",
    "train_xform = imp.fit_transform(X)\n",
    "# TODO: Impute dataframe so that F5 uses median\n",
    "# and F19 uses mean. For now, we'll impute via mean for both.\n",
    "\n",
    "X = pd.DataFrame(train_xform, columns=X.columns)\n",
    "test_xform = imp.transform(X_test)\n",
    "X_test = pd.DataFrame(test_xform, columns=X_test.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X\n",
    "y_train = y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['OMP_NUM_THREADS'] = '4' # Let OpenMP use 1 thread to avoid possible subprocess call hangs\n",
    "import hyperopt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set up logging for XGBoost param tuning.\n",
    "\n",
    "import logging\n",
    "logging.basicConfig(filename=\"logs/xgb_skl_hyperopt.log\", level=logging.INFO)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.823153+0.0165849\ttest-auc:0.815401+0.0144789\n"
     ]
    }
   ],
   "source": [
    "# import xgboost as xgb\n",
    "# from sklearn.metrics import roc_auc_score\n",
    "\n",
    "# dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "# params = {  'n_estimators': 100, \n",
    "#             'eta': 0.1,\n",
    "#             # A problem with max_depth casted to float instead of int with\n",
    "#             # the hp.quniform method.\n",
    "#             'max_depth': 25,\n",
    "#             'min_child_weight': 6,\n",
    "#             'subsample': 0.7,\n",
    "#             'gamma': 0.3,\n",
    "#             'colsample_bytree': 0.7,\n",
    "#             'eval_metric': 'auc',\n",
    "#             'objective': 'binary:logistic',\n",
    "#             # Increase this number if you have more cores. Otherwise, remove it and it will default \n",
    "#             # to the maxium number. \n",
    "#             'nthread': 4,\n",
    "#             'booster': 'gbtree',\n",
    "#             'tree_method': 'exact',\n",
    "#             'silent': 1,\n",
    "#             'seed': SEED\n",
    "#         }\n",
    "# scores = xgb.cv(params, dtrain, nfold=5, stratified=True, verbose_eval=500)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0098138053455324153"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# scores.tail(1).iloc[0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.metrics import roc_auc_score\n",
    "# Hyperparameters tuning\n",
    "from hyperopt import STATUS_OK, Trials, fmin, hp, tpe\n",
    "\n",
    "# Scoring and optimization functions\n",
    "\n",
    "def score(params):\n",
    "    logging.info(\"Training with params: \")\n",
    "    logging.info(params)\n",
    "    num_round = int(params['n_estimators'])\n",
    "    del params['n_estimators']\n",
    "    dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    score_history = xgb.cv(params, dtrain, num_round,\n",
    "                    nfold=5, stratified=True,\n",
    "                    early_stopping_rounds=250,\n",
    "                    verbose_eval=500)\n",
    "    # Only use scores from last boosting round for this set of hyperparams.\n",
    "    mean_last_round = score_history.tail(1).iloc[0, 0]\n",
    "    std_last_round = score_history.tail(1).iloc[0, 1]\n",
    "    # TODO: Add the importance for the selected features\n",
    "    logging.info(\"\\tMean Score: {0}\\n\".format(mean_last_round))\n",
    "    logging.info(\"\\tStd Dev: {0}\\n\\n\".format(std_last_round))\n",
    "    # The score function should return the loss (1-score)\n",
    "    # since the optimize function looks for the minimum\n",
    "    loss = 1 - mean_last_round\n",
    "    return {'loss': loss, 'status': STATUS_OK}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def optimize(\n",
    "             #trials, \n",
    "             random_state=SEED):\n",
    "    \"\"\"\n",
    "    This is the optimization function that given a space (space here) of \n",
    "    hyperparameters and a scoring function (score here), finds the best hyperparameters.\n",
    "    \"\"\"\n",
    "    # To learn more about XGBoost parameters, head to this page: \n",
    "    # https://github.com/dmlc/xgboost/blob/master/doc/parameter.md\n",
    "    space = {\n",
    "        'n_estimators': hp.quniform('n_estimators', 100, 1000, 1),\n",
    "        'eta': hp.quniform('eta', 0.01, 0.1, 0.025),\n",
    "        # A problem with max_depth casted to float instead of int with\n",
    "        # the hp.quniform method.\n",
    "        'max_depth':  hp.choice('max_depth', np.arange(5, 30, 4, dtype=int)),\n",
    "        'min_child_weight': hp.quniform('min_child_weight', 1, 7, 2),\n",
    "        'subsample': hp.quniform('subsample', 0.6, 1, 0.1),\n",
    "        'gamma': hp.quniform('gamma', 0.1, 1, 0.2),\n",
    "        'colsample_bytree': hp.quniform('colsample_bytree', 0.5, 1, 0.1),\n",
    "        'eval_metric': 'auc',\n",
    "        'objective': 'binary:logistic',\n",
    "        # Increase this number if you have more cores. Otherwise, remove it and it will default \n",
    "        # to the maxium number. \n",
    "        'nthread': 4,\n",
    "        'booster': 'gbtree',\n",
    "        'tree_method': 'exact',\n",
    "        'silent': 1,\n",
    "        'seed': random_state\n",
    "    }\n",
    "    # Use the fmin function from Hyperopt to find the best hyperparameters\n",
    "    best = fmin(score, space, algo=tpe.suggest, \n",
    "                # trials=trials, \n",
    "                max_evals=250)\n",
    "    return best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.824938+0.0166366\ttest-auc:0.813122+0.01288\n",
      "[0]\ttrain-auc:0.853022+0.00442814\ttest-auc:0.837581+0.0117243\n",
      "[0]\ttrain-auc:0.856823+0.0056061\ttest-auc:0.834943+0.0125085\n",
      "[0]\ttrain-auc:0.808594+0.0244222\ttest-auc:0.793251+0.0239726\n",
      "[0]\ttrain-auc:0.847141+0.00284951\ttest-auc:0.814121+0.0130268\n",
      "[0]\ttrain-auc:0.858265+0.00546674\ttest-auc:0.834697+0.0125382\n",
      "[0]\ttrain-auc:0.847142+0.00454815\ttest-auc:0.835843+0.0126234\n",
      "[0]\ttrain-auc:0.823617+0.016144\ttest-auc:0.817208+0.0117573\n",
      "[0]\ttrain-auc:0.815403+0.0125306\ttest-auc:0.815038+0.0126083\n",
      "[0]\ttrain-auc:0.854088+0.00518082\ttest-auc:0.836857+0.012655\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.821155+0.0165794\ttest-auc:0.809751+0.0156826\n",
      "[0]\ttrain-auc:0.817899+0.0171294\ttest-auc:0.81166+0.0145074\n",
      "[0]\ttrain-auc:0.853713+0.00482869\ttest-auc:0.833493+0.0129354\n",
      "[0]\ttrain-auc:0.811689+0.0152934\ttest-auc:0.811075+0.0153527\n",
      "[0]\ttrain-auc:0.855371+0.00515924\ttest-auc:0.837334+0.0129644\n",
      "[0]\ttrain-auc:0.846948+0.00477786\ttest-auc:0.836205+0.0126647\n",
      "[0]\ttrain-auc:0.851244+0.00513961\ttest-auc:0.837774+0.0119987\n",
      "[0]\ttrain-auc:0.855851+0.00530847\ttest-auc:0.835693+0.01186\n",
      "[0]\ttrain-auc:0.817612+0.0172665\ttest-auc:0.811892+0.0142096\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.826247+0.0163872\ttest-auc:0.816387+0.0120603\n",
      "[0]\ttrain-auc:0.815529+0.0178744\ttest-auc:0.814581+0.012575\n",
      "[500]\ttrain-auc:0.903176+0.00240248\ttest-auc:0.860042+0.0120043\n",
      "[0]\ttrain-auc:0.831104+0.0144919\ttest-auc:0.821109+0.00807171\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.818471+0.0169606\ttest-auc:0.819043+0.0110089\n",
      "[500]\ttrain-auc:0.904287+0.0030465\ttest-auc:0.859832+0.0115967\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.858099+0.0037183\ttest-auc:0.836325+0.0129353\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.845941+0.00280662\ttest-auc:0.814467+0.0119753\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.83252+0.00369776\ttest-auc:0.799399+0.0103267\n",
      "[0]\ttrain-auc:0.796128+0.0277401\ttest-auc:0.794508+0.0211233\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.811354+0.0234006\ttest-auc:0.797029+0.0223164\n",
      "[0]\ttrain-auc:0.8321+0.00395579\ttest-auc:0.799308+0.00959034\n",
      "[0]\ttrain-auc:0.815258+0.0227592\ttest-auc:0.79829+0.0212508\n",
      "[0]\ttrain-auc:0.827648+0.0165888\ttest-auc:0.812343+0.0136262\n",
      "[0]\ttrain-auc:0.815529+0.0212639\ttest-auc:0.800163+0.0201633\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.814986+0.0182588\ttest-auc:0.814486+0.0125838\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.82791+0.0165453\ttest-auc:0.812004+0.0135028\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.810983+0.0229328\ttest-auc:0.797439+0.0227567\n",
      "[0]\ttrain-auc:0.830463+0.0149418\ttest-auc:0.814614+0.0125816\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.815258+0.0227592\ttest-auc:0.79829+0.0212508\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.813498+0.0230455\ttest-auc:0.799878+0.020877\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.814986+0.0182588\ttest-auc:0.814486+0.0125838\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.814986+0.0182588\ttest-auc:0.814486+0.0125838\n",
      "[0]\ttrain-auc:0.815529+0.0178744\ttest-auc:0.814581+0.012575\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.81452+0.00332802\ttest-auc:0.806333+0.0130761\n",
      "[0]\ttrain-auc:0.796124+0.0277419\ttest-auc:0.794541+0.0211114\n",
      "[500]\ttrain-auc:0.90206+0.00259324\ttest-auc:0.86027+0.0116243\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.812221+0.0234498\ttest-auc:0.79663+0.0221651\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.810559+0.0244015\ttest-auc:0.797572+0.0213919\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.814119+0.00410532\ttest-auc:0.805139+0.0121388\n",
      "[500]\ttrain-auc:0.897698+0.00292972\ttest-auc:0.859751+0.0118746\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.812701+0.0039932\ttest-auc:0.803893+0.0110887\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.820045+0.00428814\ttest-auc:0.803247+0.0107248\n",
      "[0]\ttrain-auc:0.841235+0.00592749\ttest-auc:0.836269+0.0140717\n",
      "[0]\ttrain-auc:0.847009+0.00476827\ttest-auc:0.821748+0.0105361\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.793102+0.0240091\ttest-auc:0.791374+0.0204367\n",
      "[0]\ttrain-auc:0.819885+0.0176397\ttest-auc:0.808761+0.015831\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.810983+0.0229328\ttest-auc:0.797439+0.0227567\n",
      "[0]\ttrain-auc:0.818277+0.017024\ttest-auc:0.819116+0.0109769\n",
      "[500]\ttrain-auc:0.903873+0.00276227\ttest-auc:0.859663+0.0116902\n",
      "[0]\ttrain-auc:0.833799+0.0139307\ttest-auc:0.820281+0.00853339\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.830807+0.00519054\ttest-auc:0.80237+0.0104081\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.81452+0.00332802\ttest-auc:0.806333+0.0130761\n",
      "[500]\ttrain-auc:0.900166+0.00271002\ttest-auc:0.859474+0.011851\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.827298+0.00563263\ttest-auc:0.803616+0.0105045\n",
      "[0]\ttrain-auc:0.818455+0.0169677\ttest-auc:0.819029+0.0110853\n",
      "[0]\ttrain-auc:0.811437+0.0243131\ttest-auc:0.797711+0.020999\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.812495+0.0234296\ttest-auc:0.796693+0.0221196\n",
      "[0]\ttrain-auc:0.83317+0.0143429\ttest-auc:0.81983+0.00742986\n",
      "[0]\ttrain-auc:0.812877+0.00358108\ttest-auc:0.804084+0.0112015\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.818095+0.0229729\ttest-auc:0.797527+0.0206594\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.829894+0.00576096\ttest-auc:0.801993+0.00991412\n",
      "[0]\ttrain-auc:0.81817+0.0170009\ttest-auc:0.819133+0.0108896\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.797115+0.0271858\ttest-auc:0.79123+0.0245173\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.822512+0.00451946\ttest-auc:0.801995+0.0101865\n",
      "[0]\ttrain-auc:0.821635+0.0170217\ttest-auc:0.809046+0.0154781\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.815352+0.0178991\ttest-auc:0.814669+0.0125128\n",
      "[500]\ttrain-auc:0.903271+0.00251541\ttest-auc:0.860034+0.0118757\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.796628+0.0273666\ttest-auc:0.793741+0.0212763\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.814817+0.00319665\ttest-auc:0.805893+0.0127265\n",
      "[0]\ttrain-auc:0.830899+0.014644\ttest-auc:0.819843+0.0114969\n",
      "[0]\ttrain-auc:0.796124+0.0277419\ttest-auc:0.794541+0.0211114\n",
      "[0]\ttrain-auc:0.814758+0.0228129\ttest-auc:0.798853+0.0212439\n",
      "[0]\ttrain-auc:0.827578+0.0161809\ttest-auc:0.81589+0.0118377\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.831239+0.0144894\ttest-auc:0.81943+0.0109651\n",
      "[0]\ttrain-auc:0.815017+0.0181176\ttest-auc:0.814708+0.0124711\n",
      "[0]\ttrain-auc:0.814648+0.0232012\ttest-auc:0.798883+0.0209958\n",
      "[0]\ttrain-auc:0.854831+0.00540902\ttest-auc:0.836556+0.0125271\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.832532+0.0145792\ttest-auc:0.819666+0.0074132\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.814036+0.00408193\ttest-auc:0.805247+0.0119421\n",
      "[500]\ttrain-auc:0.899611+0.00284712\ttest-auc:0.859977+0.0118005\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.796162+0.0277111\ttest-auc:0.794741+0.0209683\n",
      "[0]\ttrain-auc:0.812701+0.0039932\ttest-auc:0.803893+0.0110887\n",
      "[0]\ttrain-auc:0.796124+0.0277419\ttest-auc:0.794541+0.0211114\n",
      "[0]\ttrain-auc:0.796124+0.0277419\ttest-auc:0.794541+0.0211114\n",
      "[0]\ttrain-auc:0.815529+0.0178744\ttest-auc:0.814581+0.012575\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.827067+0.016587\ttest-auc:0.816305+0.0111994\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.826522+0.0162018\ttest-auc:0.813614+0.0134433\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.83133+0.014546\ttest-auc:0.81503+0.0135067\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.831243+0.0146047\ttest-auc:0.815082+0.013427\n",
      "[0]\ttrain-auc:0.796628+0.0273666\ttest-auc:0.793741+0.0212763\n",
      "[500]\ttrain-auc:0.902399+0.0025795\ttest-auc:0.860101+0.0116363\n",
      "[0]\ttrain-auc:0.819677+0.0177628\ttest-auc:0.808982+0.015822\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.812043+0.0235152\ttest-auc:0.79688+0.0223673\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.812105+0.0234917\ttest-auc:0.796737+0.0222863\n",
      "[0]\ttrain-auc:0.81695+0.0177204\ttest-auc:0.815287+0.0145307\n",
      "[500]\ttrain-auc:0.908696+0.00257948\ttest-auc:0.860046+0.0112742\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.796121+0.0277424\ttest-auc:0.794517+0.0211328\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.796162+0.0277111\ttest-auc:0.794741+0.0209683\n",
      "[500]\ttrain-auc:0.904762+0.00241888\ttest-auc:0.860747+0.0118329\n",
      "[0]\ttrain-auc:0.867005+0.00370824\ttest-auc:0.835031+0.0128708\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.811745+0.00300357\ttest-auc:0.804848+0.0122982\n",
      "[500]\ttrain-auc:0.902172+0.00289545\ttest-auc:0.859509+0.0115287\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.811781+0.00298053\ttest-auc:0.804815+0.0122739\n",
      "[500]\ttrain-auc:0.90215+0.00290689\ttest-auc:0.859715+0.0116559\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.862056+0.0040946\ttest-auc:0.834117+0.0136664\n",
      "[0]\ttrain-auc:0.823019+0.00484679\ttest-auc:0.80405+0.0122136\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.815008+0.0180648\ttest-auc:0.814419+0.0126864\n",
      "[500]\ttrain-auc:0.90052+0.00274078\ttest-auc:0.860008+0.0117791\n",
      "[0]\ttrain-auc:0.814805+0.0232358\ttest-auc:0.798503+0.0209434\n",
      "[0]\ttrain-auc:0.811781+0.00298053\ttest-auc:0.804815+0.0122739\n",
      "[0]\ttrain-auc:0.827719+0.00456645\ttest-auc:0.802804+0.0114574\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.815008+0.0180648\ttest-auc:0.814419+0.0126864\n",
      "[500]\ttrain-auc:0.900717+0.00284093\ttest-auc:0.86013+0.0119738\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.826827+0.00509532\ttest-auc:0.802745+0.0108868\n",
      "[0]\ttrain-auc:0.830423+0.0148361\ttest-auc:0.820978+0.00763702\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.812706+0.0233248\ttest-auc:0.796691+0.0219597\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.812877+0.00358108\ttest-auc:0.804084+0.0112015\n",
      "[500]\ttrain-auc:0.898002+0.00276145\ttest-auc:0.859555+0.011703\n",
      "[0]\ttrain-auc:0.830955+0.014631\ttest-auc:0.81975+0.0114215\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.796628+0.0273666\ttest-auc:0.793741+0.0212763\n",
      "[500]\ttrain-auc:0.902399+0.0025795\ttest-auc:0.860101+0.0116363\n",
      "[0]\ttrain-auc:0.830166+0.0151332\ttest-auc:0.814786+0.0123065\n",
      "[0]\ttrain-auc:0.796156+0.0276523\ttest-auc:0.794244+0.021341\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.796124+0.0277419\ttest-auc:0.794541+0.0211114\n",
      "[0]\ttrain-auc:0.83133+0.014546\ttest-auc:0.81503+0.0135067\n",
      "[0]\ttrain-auc:0.830816+0.00487764\ttest-auc:0.802922+0.00986443\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "[0]\ttrain-auc:0.857861+0.00374926\ttest-auc:0.83648+0.0128615\n",
      "[0]\ttrain-auc:0.814673+0.0183747\ttest-auc:0.814525+0.0125377\n",
      "[500]\ttrain-auc:0.905797+0.0022948\ttest-auc:0.860417+0.0119553\n",
      "[0]\ttrain-auc:0.826045+0.00495638\ttest-auc:0.802784+0.0110796\n",
      "[0]\ttrain-auc:0.5+0\ttest-auc:0.5+0\n",
      "The best hyperparameters are:  \n",
      "\n",
      "{'subsample': 0.9, 'colsample_bytree': 0.5, 'max_depth': 0, 'n_estimators': 724.0, 'gamma': 0.6000000000000001, 'min_child_weight': 2.0, 'eta': 0.025}\n"
     ]
    }
   ],
   "source": [
    "best_hyperparams = optimize(\n",
    "                            #trials\n",
    "                            )\n",
    "print(\"The best hyperparameters are: \", \"\\n\")\n",
    "print(best_hyperparams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:datasci]",
   "language": "python",
   "name": "conda-env-datasci-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}