07-04-Ensemble Methods 4 - AdaBoost & Gradient Boosting Machine.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bagging Machine Learning Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df = sns.load_dataset('titanic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891, 15)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>survived</th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>sibsp</th>\n",
       "      <th>parch</th>\n",
       "      <th>fare</th>\n",
       "      <th>embarked</th>\n",
       "      <th>class</th>\n",
       "      <th>who</th>\n",
       "      <th>adult_male</th>\n",
       "      <th>deck</th>\n",
       "      <th>embark_town</th>\n",
       "      <th>alive</th>\n",
       "      <th>alone</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Cherbourg</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>S</td>\n",
       "      <td>First</td>\n",
       "      <td>woman</td>\n",
       "      <td>False</td>\n",
       "      <td>C</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>yes</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>S</td>\n",
       "      <td>Third</td>\n",
       "      <td>man</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Southampton</td>\n",
       "      <td>no</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   survived  pclass     sex   age  sibsp  parch     fare embarked  class  \\\n",
       "0         0       3    male  22.0      1      0   7.2500        S  Third   \n",
       "1         1       1  female  38.0      1      0  71.2833        C  First   \n",
       "2         1       3  female  26.0      0      0   7.9250        S  Third   \n",
       "3         1       1  female  35.0      1      0  53.1000        S  First   \n",
       "4         0       3    male  35.0      0      0   8.0500        S  Third   \n",
       "\n",
       "     who  adult_male deck  embark_town alive  alone  \n",
       "0    man        True  NaN  Southampton    no  False  \n",
       "1  woman       False    C    Cherbourg   yes  False  \n",
       "2  woman       False  NaN  Southampton   yes   True  \n",
       "3  woman       False    C  Southampton   yes  False  \n",
       "4    man        True  NaN  Southampton    no   True  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 3, 2])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['pclass'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    157\n",
       "2     15\n",
       "3     10\n",
       "Name: pclass, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['pclass'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['female', 'male'], dtype=object)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sex'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "male      94\n",
       "female    88\n",
       "Name: sex, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['sex'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAEpJJREFUeJzt3X2MZXV9x/H3t+AD7OgCRaZ0l3Q0MVTDKLo3FKW1M1J0BSJNY1MINdDSzB+1iGaNXWJa4x+mNK1PaZs2RBHTEqYRsZrFqgSZkjaKnQV0FxfE6gZZkMUiSwdJdeu3f8zZOBlndu455z4cf/t+JZO598y553zuOed+cubMvb+JzESS9PPvF8YdQJI0GBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRDHj3Jlp556ak5NTfU9/zPPPMOmTZuGF6ihruaC7mbrai7obrau5oLuZutqLmiXbffu3d/PzBdtOGNmjuxr27ZtWcedd95Za/5R6WquzO5m62quzO5m62quzO5m62quzHbZgMXso2O95CJJhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYUY6Uf/pUGb2nnbmtP3X3fRiJNI4+cZuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFWLDQo+IGyLiYETsXeNn74qIjIhThxNPktSvfs7QbwS2r54YEWcAFwAPDziTJKmBDQs9M+8CnlzjRx8C3g3koENJkuprdA09It4MHMjMrw04jySpocjc+AQ7IqaAXZl5VkScCNwJvCEzD0XEfqCXmd9f57FzwBzA5OTktvn5+b7DLS0tMTEx0ff8o9LVXNDdbMPKtefAoTWnT2/Z3PcyjrVtNghdzdbVXNAu2+zs7O7M7G00X5NCnwbuAH5Y/Xgr8ChwTmZ+72jL6fV6ubi4uOH6jlhYWGBmZqbv+Uelq7mgu9mGlWsQw+cea9tsELqarau5oF22iOir0GuPh56Ze4DTVqxoP0c5Q5ckjUY/b1u8GfgycGZEPBIRVw0/liSprg3P0DPzsg1+PjWwNJKkxvykqCQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQvTzP0VviIiDEbF3xbS/iogHIuLrEfHpiDhpuDElSRvp5wz9RmD7qmm3A2dl5iuAbwLXDjiXJKmmDQs9M+8Cnlw17YuZebi6+xVg6xCySZJqGMQ19D8E/nUAy5EktRCZufFMEVPArsw8a9X09wA94HdynQVFxBwwBzA5Obltfn6+73BLS0tMTEz0Pf+odDUXdDfbsHLtOXBozenTWzb3vYxjbZsNQlezdTUXtMs2Ozu7OzN7G813fKOlAxFxBXAxcP56ZQ6QmdcD1wP0er2cmZnpex0LCwvUmX9UupoLupttWLmu3HnbmtP3X97/uo61bTYIXc3W1VwwmmyNCj0itgN/CvxmZv5wsJEkSU3087bFm4EvA2dGxCMRcRXwt8ALgNsj4r6I+Ich55QkbWDDM/TMvGyNyR8bQhZJUgt+UlSSCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiEsdEkqRD//JPqGiDgYEXtXTDslIm6PiIeq7ycPN6YkaSP9nKHfCGxfNW0ncEdmvhS4o7ovSRqjDQs9M+8Cnlw1+RLgE9XtTwC/PeBckqSaml5Dn8zMxwCq76cNLpIkqYnIzI1nipgCdmXmWdX9pzLzpBU//0FmrnkdPSLmgDmAycnJbfPz832HW1paYmJiou/5R6WruaC72YaVa8+BQ2tOn96yue9ljGqb1c3a1X0J3c3W1VzQLtvs7OzuzOxtNN/xjZYOj0fE6Zn5WEScDhxcb8bMvB64HqDX6+XMzEzfK1lYWKDO/KPS1VzQ3WzDynXlztvWnL7/8v7XNaptVjdrV/cldDdbV3PBaLI1veTyWeCK6vYVwGcGE0eS1FQ/b1u8GfgycGZEPBIRVwHXARdExEPABdV9SdIYbXjJJTMvW+dH5w84iySpBT8pKkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWg6OJcKMLXeYFHXXVTsutda747pw8wMda3SaHiGLkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWhV6BHxzoi4PyL2RsTNEfH8QQWTJNXTuNAjYgvwdqCXmWcBxwGXDiqYJKmetpdcjgdOiIjjgROBR9tHkiQ1EZnZ/MER1wDvB54FvpiZl68xzxwwBzA5Obltfn6+7+UvLS0xMTHRON+wdDUX1Mu258ChNadPb9k8yEjAz+Ya1LrrLmet+SdPgNNOGfxz7mfdsH7WUo6zUepqLmiXbXZ2dndm9jaar3GhR8TJwKeA3wOeAj4J3JKZ/7TeY3q9Xi4uLva9joWFBWZmZhrlG6au5oJ62UY54uHqXINad93lrDfa4tWXX1JrvU3UzVrKcTZKXc0F7bJFRF+F3uaSy28B38nMJzLzx8CtwGtbLE+S1EKbQn8YODciToyIAM4H9g0mliSprsaFnpl3A7cA9wB7qmVdP6BckqSaWv3Hosx8L/DeAWWRJLXgJ0UlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWj1wSIN17AHsBqkUQ70JWltnqFLUiEsdEkqhIUuSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhWhV6RJwUEbdExAMRsS8iXjOoYJKketp+9P8jwOcz8y0R8VzgxAFkkiQ10LjQI+KFwOuAKwEy80fAjwYTS5JUV5tLLi8BngA+HhH3RsRHI2LTgHJJkmqKzGz2wIge8BXgvMy8OyI+AjydmX+2ar45YA5gcnJy2/z8fN/rWFpaYmJiolG+YRpVrj0HDtWaf3rL5jWzNVlOXeut48iyVueqm6mu9Z7DWuudPAFOO6X+c65ro2202sEnD/H4s/3PP0rH+muziTbZZmdnd2dmb6P52hT6LwFfycyp6v5vADszc93xUnu9Xi4uLva9joWFBWZmZhrlG6ZR5ao77O3+6y5aM1uT5dS10fC5q3MNe0jf9Z7DWuvdMX2Yqy+/ZKh51ls3rJ/1b276DB/Y87NXRbswJPGx/tpsok22iOir0BtfcsnM7wHfjYgzq0nnA99oujxJUjtt3+VyNXBT9Q6XbwN/0D6SJKmJVoWemfcBG/4aIEkaPj8pKkmFsNAlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhWj70f+RqTuwkZob9sBZpRjUMbnecnZMD3e9dfP4Wus+z9AlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQrQs9Io6LiHsjYtcgAkmSmhnEGfo1wL4BLEeS1EKrQo+IrcBFwEcHE0eS1FTbM/QPA+8GfjKALJKkFiIzmz0w4mLgwsz844iYAd6VmRevMd8cMAcwOTm5bX5+vu91LC0tMTExAcCeA4dq5ZvesrnW/HWszDVMTZ7zWtnqLmcYJk+Ax58dd4q1HS3b0Y6j9bbreo+pux/qbrO6622Tc2W2Yb7W6hrVa7OJNtlmZ2d3Z2Zvo/naFPpfAG8FDgPPB14I3JqZv7/eY3q9Xi4uLva9joWFBWZmZoD6Q7oOc6jPlbmGqclzXitbF4bD3TF9mA/s6eZozUfLdrTjqO4ws3X3Q91tVne9bXKuzNalYXVH9dpsok22iOir0BtfcsnMazNza2ZOAZcCXzpamUuShsv3oUtSIQbyO3BmLgALg1iWJKkZz9AlqRAWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5Jhejm4BrHmEGNtTK18zZ2TB/myg6M3XIsG9fYOV0Ys0fj5Rm6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKoSFLkmFsNAlqRAWuiQVwkKXpEI0LvSIOCMi7oyIfRFxf0RcM8hgkqR62ozlchjYkZn3RMQLgN0RcXtmfmNA2SRJNTQ+Q8/MxzLznur2/wD7gC2DCiZJqicys/1CIqaAu4CzMvPpVT+bA+YAJicnt83Pz/e93KWlJSYmJgDYc+BQrUzTWzbXmr+OlbnqqPscmpg8AR5/duirqa2rueDo2Y52HA17fw57m6333Pp5XiuzDXIbtX3dNn1tjkKbbLOzs7szs7fRfK0LPSImgH8D3p+Ztx5t3l6vl4uLi30ve2FhgZmZGaD+0KD7r7uo1vx1rMxVxyiGN90xfZgP7OneqMhdzQVHz3a042jY+3PY22y959bP81qZbZDbqO3rtulrcxTaZIuIvgq91btcIuI5wKeAmzYqc0nScLV5l0sAHwP2ZeYHBxdJktREmzP084C3Aq+PiPuqrwsHlEuSVFPjC3SZ+e9ADDCLJKkFPykqSYWw0CWpEBa6JBXCQpekQljoklQIC12SCmGhS1IhLHRJKkQ3R0saoqMNFlR3YKD1ljXMgcE0WqMYUG1cBvXcxrmNVq97x/Rhrhzga7xtnpVu3L5pqOsGz9AlqRgWuiQVwkKXpEJY6JJUCAtdkgphoUtSISx0SSqEhS5JhbDQJakQrQo9IrZHxIMR8a2I2DmoUJKk+hoXekQcB/wd8Cbg5cBlEfHyQQWTJNXT5gz9HOBbmfntzPwRMA9cMphYkqS62hT6FuC7K+4/Uk2TJI1BZGazB0b8LvDGzPyj6v5bgXMy8+pV880Bc9XdM4EHa6zmVOD7jQIOV1dzQXezdTUXdDdbV3NBd7N1NRe0y/YrmfmijWZqM3zuI8AZK+5vBR5dPVNmXg9c32QFEbGYmb1m8Yanq7mgu9m6mgu6m62ruaC72bqaC0aTrc0ll/8EXhoRL46I5wKXAp8dTCxJUl2Nz9Az83BE/AnwBeA44IbMvH9gySRJtbT6j0WZ+TngcwPKspZGl2pGoKu5oLvZupoLuputq7mgu9m6mgtGkK3xH0UlSd3iR/8lqRCdLPQuDSkQETdExMGI2Lti2ikRcXtEPFR9P3kMuc6IiDsjYl9E3B8R13Qo2/Mj4qsR8bUq2/uq6S+OiLurbP9c/TF95CLiuIi4NyJ2dSzX/ojYExH3RcRiNa0L+/OkiLglIh6ojrfXdCTXmdW2OvL1dES8oyPZ3lkd+3sj4ubqNTH046xzhd7BIQVuBLavmrYTuCMzXwrcUd0ftcPAjsx8GXAu8LZqO3Uh2/8Cr8/MVwJnA9sj4lzgL4EPVdl+AFw1hmwA1wD7VtzvSi6A2cw8e8Xb27qwPz8CfD4zfxV4Jcvbbuy5MvPBaludDWwDfgh8etzZImIL8Hagl5lnsfymkUsZxXGWmZ36Al4DfGHF/WuBa8ecaQrYu+L+g8Dp1e3TgQc7sN0+A1zQtWzAicA9wK+x/KGK49fazyPMs5XlF/nrgV1AdCFXte79wKmrpo11fwIvBL5D9fe2ruRaI+cbgP/oQjZ++in6U1h+48ku4I2jOM46d4bOz8eQApOZ+RhA9f20cYaJiCngVcDddCRbdVnjPuAgcDvwX8BTmXm4mmVc+/XDwLuBn1T3f7EjuQAS+GJE7K4+YQ3j358vAZ4APl5dpvpoRGzqQK7VLgVurm6PNVtmHgD+GngYeAw4BOxmBMdZFws91pjmW3HWERETwKeAd2Tm0+POc0Rm/l8u/yq8leWB3F621myjzBQRFwMHM3P3yslrzDqu4+28zHw1y5cb3xYRrxtTjpWOB14N/H1mvgp4hvFc9llXdS36zcAnx50FoLpmfwnwYuCXgU0s79PVBn6cdbHQ+xpSYMwej4jTAarvB8cRIiKew3KZ35SZt3Yp2xGZ+RSwwPJ1/pMi4shnH8axX88D3hwR+1keHfT1LJ+xjzsXAJn5aPX9IMvXgs9h/PvzEeCRzLy7un8LywU/7lwrvQm4JzMfr+6PO9tvAd/JzCcy88fArcBrGcFx1sVC/3kYUuCzwBXV7StYvn49UhERwMeAfZn5wY5le1FEnFTdPoHlA3wfcCfwlnFly8xrM3NrZk6xfFx9KTMvH3cugIjYFBEvOHKb5WvCexnz/szM7wHfjYgzq0nnA98Yd65VLuOnl1tg/NkeBs6NiBOr1+mRbTb842ycf8g4yh8VLgS+yfJ11/eMOcvNLF8H+zHLZytXsXzd9Q7goer7KWPI9ess/8r2deC+6uvCjmR7BXBvlW0v8OfV9JcAXwW+xfKvx88b436dAXZ1JVeV4WvV1/1HjvuO7M+zgcVqf/4LcHIXclXZTgT+G9i8YtrYswHvAx6ojv9/BJ43iuPMT4pKUiG6eMlFktSAhS5JhbDQJakQFrokFcJCl6RCWOiSVAgLXZIKYaFLUiH+H59OIbKbrAMjAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['age'].hist(bins=50);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "X = df[['pclass', 'sex', 'age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "lb = preprocessing.LabelBinarizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/anthonyn/anaconda3/envs/tensorflow/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "X['sex'] = lb.fit_transform(X['sex'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>38.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>35.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>54.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>58.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    pclass  sex   age\n",
       "1        1    0  38.0\n",
       "3        1    0  35.0\n",
       "6        1    1  54.0\n",
       "10       3    0   4.0\n",
       "11       1    0  58.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(182, 3)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pclass</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>182.000000</td>\n",
       "      <td>182.000000</td>\n",
       "      <td>182.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.192308</td>\n",
       "      <td>0.516484</td>\n",
       "      <td>35.623187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.516411</td>\n",
       "      <td>0.501107</td>\n",
       "      <td>15.671615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.920000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>36.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>47.750000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>80.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           pclass         sex         age\n",
       "count  182.000000  182.000000  182.000000\n",
       "mean     1.192308    0.516484   35.623187\n",
       "std      0.516411    0.501107   15.671615\n",
       "min      1.000000    0.000000    0.920000\n",
       "25%      1.000000    0.000000   24.000000\n",
       "50%      1.000000    1.000000   36.000000\n",
       "75%      1.000000    1.000000   47.750000\n",
       "max      3.000000    1.000000   80.000000"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 182 entries, 1 to 889\n",
      "Data columns (total 3 columns):\n",
      "pclass    182 non-null int64\n",
      "sex       182 non-null int64\n",
      "age       182 non-null float64\n",
      "dtypes: float64(1), int64(2)\n",
      "memory usage: 5.7 KB\n"
     ]
    }
   ],
   "source": [
    "X.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "y = df['survived']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    123\n",
       "0     59\n",
       "Name: survived, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import cross_val_score, cross_val_predict\n",
    "from sklearn.metrics import accuracy_score, classification_report\n",
    "from sklearn.metrics import confusion_matrix, roc_auc_score\n",
    "def print_score(clf, X_train, X_test, y_train, y_test, train=True):\n",
    "    '''\n",
    "    v0.1 Follow the scikit learn library format in terms of input\n",
    "    print the accuracy score, classification report and confusion matrix of classifier\n",
    "    '''\n",
    "    lb = preprocessing.LabelBinarizer()\n",
    "    lb.fit(y_train)\n",
    "    if train:\n",
    "        '''\n",
    "        training performance\n",
    "        '''\n",
    "        res = clf.predict(X_train)\n",
    "        print(\"Train Result:\\n\")\n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_train, \n",
    "                                                                res)))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_train, \n",
    "                                                                            res)))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_train, \n",
    "                                                                  res)))\n",
    "        print(\"ROC AUC: {0:.4f}\\n\".format(roc_auc_score(lb.transform(y_train), \n",
    "                                                      lb.transform(res))))\n",
    "\n",
    "        res = cross_val_score(clf, X_train, y_train, cv=10, scoring='accuracy')\n",
    "        print(\"Average Accuracy: \\t {0:.4f}\".format(np.mean(res)))\n",
    "        print(\"Accuracy SD: \\t\\t {0:.4f}\".format(np.std(res)))\n",
    "        \n",
    "    elif train==False:\n",
    "        '''\n",
    "        test performance\n",
    "        '''\n",
    "        res_test = clf.predict(X_test)\n",
    "        print(\"Test Result:\\n\")        \n",
    "        print(\"accuracy score: {0:.4f}\\n\".format(accuracy_score(y_test, \n",
    "                                                                res_test)))\n",
    "        print(\"Classification Report: \\n {}\\n\".format(classification_report(y_test, \n",
    "                                                                            res_test)))\n",
    "        print(\"Confusion Matrix: \\n {}\\n\".format(confusion_matrix(y_test, \n",
    "                                                                  res_test)))   \n",
    "        print(\"ROC AUC: {0:.4f}\\n\".format(roc_auc_score(lb.transform(y_test), \n",
    "                                                      lb.transform(res_test))))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Boosting (Hypothesis Boosting)\n",
    "\n",
    "* Combine several weak learners into a strong learner. \n",
    "\n",
    "* Train predictors sequentially"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AdaBoost / Adaptive Boosting\n",
    "\n",
    "[Robert Schapire](http://rob.schapire.net/papers/explaining-adaboost.pdf)\n",
    "\n",
    "[Wikipedia](https://en.wikipedia.org/wiki/AdaBoost)\n",
    "\n",
    "[Chris McCormick](http://mccormickml.com/2013/12/13/adaboost-tutorial/)\n",
    "\n",
    "[Scikit Learn AdaBoost](http://scikit-learn.org/stable/modules/ensemble.html#adaboost)\n",
    "\n",
    "1995\n",
    "\n",
    "As above for Boosting:\n",
    "* Similar to human learning, the algo learns from past mistakes by focusing more on difficult problems it did not get right in prior learning. \n",
    "* In machine learning speak, it pays more attention to training instances that previously underfitted.\n",
    "\n",
    "Source: Scikit-Learn:\n",
    "\n",
    "* Fit a sequence of weak learners (i.e., models that are only slightly better than random guessing, such as small decision trees) on repeatedly modified versions of the data. \n",
    "* The predictions from all of them are then combined through a weighted majority vote (or sum) to produce the final prediction.\n",
    "* The data modifications at each so-called boosting iteration consist of applying weights $w_1, w_2, …, w_N$ to each of the training samples. \n",
    "* Initially, those weights are all set to $w_i = 1/N$, so that the first step simply trains a weak learner on the original data. \n",
    "* For each successive iteration, the sample weights are individually modified and the learning algorithm is reapplied to the reweighted data. \n",
    "* At a given step, those training examples that were incorrectly predicted by the boosted model induced at the previous step have their weights increased, whereas the weights are decreased for those that were predicted correctly. \n",
    "* As iterations proceed, examples that are difficult to predict receive ever-increasing influence. Each subsequent weak learner is thereby forced to concentrate on the examples that are missed by the previous ones in the sequence.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import AdaBoostClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "ada_clf = AdaBoostClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None,\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[SAMME16](https://web.stanford.edu/~hastie/Papers/samme.pdf) (Stagewise Additive Modeling using a Multiclass Exponential loss function).\n",
    "\n",
    "R stands for real"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.8740\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.88      0.70      0.78        40\n",
      "          1       0.87      0.95      0.91        87\n",
      "\n",
      "avg / total       0.87      0.87      0.87       127\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[28 12]\n",
      " [ 4 83]]\n",
      "\n",
      "ROC AUC: 0.8270\n",
      "\n",
      "Average Accuracy: \t 0.7321\n",
      "Accuracy SD: \t\t 0.1158\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.7818\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.77      0.53      0.62        19\n",
      "          1       0.79      0.92      0.85        36\n",
      "\n",
      "avg / total       0.78      0.78      0.77        55\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[10  9]\n",
      " [ 3 33]]\n",
      "\n",
      "ROC AUC: 0.7215\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(ada_clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(ada_clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaBoost with Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "ada_clf = AdaBoostClassifier(RandomForestClassifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "          base_estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9449\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.93      0.91        40\n",
      "          1       0.97      0.95      0.96        87\n",
      "\n",
      "avg / total       0.95      0.94      0.95       127\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[37  3]\n",
      " [ 4 83]]\n",
      "\n",
      "ROC AUC: 0.9395\n",
      "\n",
      "Average Accuracy: \t 0.7628\n",
      "Accuracy SD: \t\t 0.0871\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.7455\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.61      0.74      0.67        19\n",
      "          1       0.84      0.75      0.79        36\n",
      "\n",
      "avg / total       0.76      0.75      0.75        55\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[14  5]\n",
      " [ 9 27]]\n",
      "\n",
      "ROC AUC: 0.7434\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(ada_clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(ada_clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "ada_clf = AdaBoostClassifier(base_estimator=RandomForestClassifier())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R',\n",
       "          base_estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=1,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9449\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.93      0.91        40\n",
      "          1       0.97      0.95      0.96        87\n",
      "\n",
      "avg / total       0.95      0.94      0.95       127\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[37  3]\n",
      " [ 4 83]]\n",
      "\n",
      "ROC AUC: 0.9395\n",
      "\n",
      "Average Accuracy: \t 0.7474\n",
      "Accuracy SD: \t\t 0.0936\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.7455\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.61      0.74      0.67        19\n",
      "          1       0.84      0.75      0.79        36\n",
      "\n",
      "avg / total       0.76      0.75      0.75        55\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[14  5]\n",
      " [ 9 27]]\n",
      "\n",
      "ROC AUC: 0.7434\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(ada_clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(ada_clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gradient Boosting / Gradient Boosting Machine (GBM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Works for both regression and classification\n",
    "\n",
    "[Wikipedia](https://en.wikipedia.org/wiki/Gradient_boosting)\n",
    "\n",
    "* Sequentially adding predictors\n",
    "* Each one correcting its predecessor\n",
    "* Fit new predictor to the residual errors\n",
    "\n",
    "Compare this to AdaBoost: \n",
    "* Alter instance weights at every iteration\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Step 1. **\n",
    "\n",
    "  $$Y = F(x) + \\epsilon$$\n",
    "\n",
    "**Step 2. **\n",
    "\n",
    "  $$\\epsilon = G(x) + \\epsilon_2$$\n",
    "\n",
    "  Substituting (2) into (1), we get:\n",
    "  \n",
    "  $$Y = F(x) + G(x) + \\epsilon_2$$\n",
    "    \n",
    "**Step 3. **\n",
    "\n",
    "  $$\\epsilon_2 = H(x)  + \\epsilon_3$$\n",
    "\n",
    "Now:\n",
    "  \n",
    "  $$Y = F(x) + G(x) + H(x)  + \\epsilon_3$$\n",
    "  \n",
    "Finally, by adding weighting  \n",
    "  \n",
    "  $$Y = \\alpha F(x) + \\beta G(x) + \\gamma H(x)  + \\epsilon_4$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gradient boosting involves three elements:\n",
    "\n",
    "* **Loss function to be optimized**: Loss function depends on the type of problem being solved. In the case of regression problems, mean squared error is used, and in classification problems, logarithmic loss will be used. In boosting, at each stage, unexplained loss from prior iterations will be optimized rather than starting from scratch.\n",
    "\n",
    "* **Weak learner to make predictions**: Decision trees are used as a weak learner in gradient boosting.\n",
    "\n",
    "* **Additive model to add weak learners to minimize the loss function**: Trees are added one at a time and existing trees in the model are not changed. The gradient descent procedure is used to minimize the loss when adding trees."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "              presort='auto', random_state=None, subsample=1.0, verbose=0,\n",
       "              warm_start=False)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gbc_clf = GradientBoostingClassifier()\n",
    "gbc_clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Result:\n",
      "\n",
      "accuracy score: 0.9449\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.90      0.93      0.91        40\n",
      "          1       0.97      0.95      0.96        87\n",
      "\n",
      "avg / total       0.95      0.94      0.95       127\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[37  3]\n",
      " [ 4 83]]\n",
      "\n",
      "ROC AUC: 0.9395\n",
      "\n",
      "Average Accuracy: \t 0.7314\n",
      "Accuracy SD: \t\t 0.0752\n",
      "Test Result:\n",
      "\n",
      "accuracy score: 0.8182\n",
      "\n",
      "Classification Report: \n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          0       0.74      0.74      0.74        19\n",
      "          1       0.86      0.86      0.86        36\n",
      "\n",
      "avg / total       0.82      0.82      0.82        55\n",
      "\n",
      "\n",
      "Confusion Matrix: \n",
      " [[14  5]\n",
      " [ 5 31]]\n",
      "\n",
      "ROC AUC: 0.7990\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_score(gbc_clf, X_train, X_test, y_train, y_test, train=True)\n",
    "print_score(gbc_clf, X_train, X_test, y_train, y_test, train=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "| Classifier | Decision Tree | Bagging | Random Forest | Optimised RF | Extra-Trees | AdaBoost (CART) | AdaBoost (RF) | Gradient Boosting |\n",
    "|:-|:-|:- |:- |:- |:- |:-|:-| :- |\n",
    "| Train accuracy score | 0.9528 | 0.9528 | 0.9325 | 0.9264 | 0.9448 | 0.8661 | 0.9528 | 0.9449 |\n",
    "| Average accuracy score | 0.7724 | 0.7879 | 0.7801 | 0.7059 | 0.7548 | 0.7793 | 0.7353 | 0.7906 |\n",
    "| SD | 0.1018 | 0.1008 | 0.1474 | 0.1308 | 0.1406 | 0.1172 | 0.0881 | 0.0912 |\n",
    "| Test accuracy score | 0.7636 | 0.7455 | 0.7895 | 0.6316 | 0.7895 | 0.6545 | 0.7818 | 0.7818 |\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}