HW3_ScikitLearn.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "xORZjoj_lULf"
      },
      "outputs": [],
      "source": [
        "# Enter your name(s) here\n",
        "# Tushar Kohli tk22994\n",
        "# Shizuka Bai sb55983"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yebR9i5SlULg"
      },
      "source": [
        "# Assignment 3 : Using `scikit-learn`\n",
        "\n",
        "Scikit-learn provides a range of supervised and unsupervised learning algorithms via a consistent interface in Python. In this assigment you'll explore how to train various classifiers using the `scikit-learn` library. The scikit-learn documentation can be found [here](http://scikit-learn.org/stable/documentation.html).\n",
        "\n",
        "In this assignment we'll attempt to classify patients as either having or not having diabetic retinopathy, using the same Diabetic Retinopathy data set from your previous assignments. Recall that this dataset contains 1151 records and 20 attributes (some categorical, some continuous). You can find additional details about the dataset [here](http://archive.ics.uci.edu/ml/datasets/Diabetic+Retinopathy+Debrecen+Data+Set)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "nWWiSmitlULh"
      },
      "outputs": [],
      "source": [
        "#You may add additional imports\n",
        "import warnings\n",
        "#warnings.simplefilter(\"ignore\")\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import sklearn as sk\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "from sklearn.model_selection import train_test_split\n",
        "import sklearn.metrics as met\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn import tree\n",
        "import sklearn.model_selection as ms\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.decomposition import PCA\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from warnings import simplefilter\n",
        "from sklearn.exceptions import ConvergenceWarning\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.ensemble import AdaBoostClassifier"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "uLj5UB59lULh"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 445
        },
        "id": "Vwk4ve_4lULh",
        "outputId": "45652893-e5f6-4c6e-90fd-ae937a2573d9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1151, 20)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>quality</th>\n",
              "      <th>prescreen</th>\n",
              "      <th>ma2</th>\n",
              "      <th>ma3</th>\n",
              "      <th>ma4</th>\n",
              "      <th>ma5</th>\n",
              "      <th>ma6</th>\n",
              "      <th>ma7</th>\n",
              "      <th>exudate8</th>\n",
              "      <th>exudate9</th>\n",
              "      <th>exudate10</th>\n",
              "      <th>exudate11</th>\n",
              "      <th>exudate12</th>\n",
              "      <th>exudate13</th>\n",
              "      <th>exudate14</th>\n",
              "      <th>exudate15</th>\n",
              "      <th>euDist</th>\n",
              "      <th>diameter</th>\n",
              "      <th>amfm_class</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "      <td>22</td>\n",
              "      <td>22</td>\n",
              "      <td>19</td>\n",
              "      <td>18</td>\n",
              "      <td>14</td>\n",
              "      <td>49.895756</td>\n",
              "      <td>17.775994</td>\n",
              "      <td>5.270920</td>\n",
              "      <td>0.771761</td>\n",
              "      <td>0.018632</td>\n",
              "      <td>0.006864</td>\n",
              "      <td>0.003923</td>\n",
              "      <td>0.003923</td>\n",
              "      <td>0.486903</td>\n",
              "      <td>0.100025</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>22</td>\n",
              "      <td>18</td>\n",
              "      <td>16</td>\n",
              "      <td>13</td>\n",
              "      <td>57.709936</td>\n",
              "      <td>23.799994</td>\n",
              "      <td>3.325423</td>\n",
              "      <td>0.234185</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.520908</td>\n",
              "      <td>0.144414</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>62</td>\n",
              "      <td>60</td>\n",
              "      <td>59</td>\n",
              "      <td>54</td>\n",
              "      <td>47</td>\n",
              "      <td>33</td>\n",
              "      <td>55.831441</td>\n",
              "      <td>27.993933</td>\n",
              "      <td>12.687485</td>\n",
              "      <td>4.852282</td>\n",
              "      <td>1.393889</td>\n",
              "      <td>0.373252</td>\n",
              "      <td>0.041817</td>\n",
              "      <td>0.007744</td>\n",
              "      <td>0.530904</td>\n",
              "      <td>0.128548</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>55</td>\n",
              "      <td>53</td>\n",
              "      <td>53</td>\n",
              "      <td>50</td>\n",
              "      <td>43</td>\n",
              "      <td>31</td>\n",
              "      <td>40.467228</td>\n",
              "      <td>18.445954</td>\n",
              "      <td>9.118901</td>\n",
              "      <td>3.079428</td>\n",
              "      <td>0.840261</td>\n",
              "      <td>0.272434</td>\n",
              "      <td>0.007653</td>\n",
              "      <td>0.001531</td>\n",
              "      <td>0.483284</td>\n",
              "      <td>0.114790</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>44</td>\n",
              "      <td>44</td>\n",
              "      <td>44</td>\n",
              "      <td>41</td>\n",
              "      <td>39</td>\n",
              "      <td>27</td>\n",
              "      <td>18.026254</td>\n",
              "      <td>8.570709</td>\n",
              "      <td>0.410381</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.475935</td>\n",
              "      <td>0.123572</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>44</td>\n",
              "      <td>43</td>\n",
              "      <td>41</td>\n",
              "      <td>41</td>\n",
              "      <td>37</td>\n",
              "      <td>29</td>\n",
              "      <td>28.356400</td>\n",
              "      <td>6.935636</td>\n",
              "      <td>2.305771</td>\n",
              "      <td>0.323724</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.502831</td>\n",
              "      <td>0.126741</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>29</td>\n",
              "      <td>29</td>\n",
              "      <td>29</td>\n",
              "      <td>27</td>\n",
              "      <td>25</td>\n",
              "      <td>16</td>\n",
              "      <td>15.448398</td>\n",
              "      <td>9.113819</td>\n",
              "      <td>1.633493</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.541743</td>\n",
              "      <td>0.139575</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>20.679649</td>\n",
              "      <td>9.497786</td>\n",
              "      <td>1.223660</td>\n",
              "      <td>0.150382</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.576318</td>\n",
              "      <td>0.071071</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "      <td>21</td>\n",
              "      <td>18</td>\n",
              "      <td>15</td>\n",
              "      <td>13</td>\n",
              "      <td>10</td>\n",
              "      <td>66.691933</td>\n",
              "      <td>23.545543</td>\n",
              "      <td>6.151117</td>\n",
              "      <td>0.496372</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.500073</td>\n",
              "      <td>0.116793</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>79</td>\n",
              "      <td>75</td>\n",
              "      <td>73</td>\n",
              "      <td>71</td>\n",
              "      <td>64</td>\n",
              "      <td>47</td>\n",
              "      <td>22.141784</td>\n",
              "      <td>10.054384</td>\n",
              "      <td>0.874633</td>\n",
              "      <td>0.099780</td>\n",
              "      <td>0.023386</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.560959</td>\n",
              "      <td>0.109134</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   quality  prescreen  ma2  ma3  ma4  ma5  ma6  ma7   exudate8   exudate9  \\\n",
              "0        1          1   22   22   22   19   18   14  49.895756  17.775994   \n",
              "1        1          1   24   24   22   18   16   13  57.709936  23.799994   \n",
              "2        1          1   62   60   59   54   47   33  55.831441  27.993933   \n",
              "3        1          1   55   53   53   50   43   31  40.467228  18.445954   \n",
              "4        1          1   44   44   44   41   39   27  18.026254   8.570709   \n",
              "5        1          1   44   43   41   41   37   29  28.356400   6.935636   \n",
              "6        1          0   29   29   29   27   25   16  15.448398   9.113819   \n",
              "7        1          1    6    6    6    6    2    1  20.679649   9.497786   \n",
              "8        1          1   22   21   18   15   13   10  66.691933  23.545543   \n",
              "9        1          1   79   75   73   71   64   47  22.141784  10.054384   \n",
              "\n",
              "   exudate10  exudate11  exudate12  exudate13  exudate14  exudate15    euDist  \\\n",
              "0   5.270920   0.771761   0.018632   0.006864   0.003923   0.003923  0.486903   \n",
              "1   3.325423   0.234185   0.003903   0.003903   0.003903   0.003903  0.520908   \n",
              "2  12.687485   4.852282   1.393889   0.373252   0.041817   0.007744  0.530904   \n",
              "3   9.118901   3.079428   0.840261   0.272434   0.007653   0.001531  0.483284   \n",
              "4   0.410381   0.000000   0.000000   0.000000   0.000000   0.000000  0.475935   \n",
              "5   2.305771   0.323724   0.000000   0.000000   0.000000   0.000000  0.502831   \n",
              "6   1.633493   0.000000   0.000000   0.000000   0.000000   0.000000  0.541743   \n",
              "7   1.223660   0.150382   0.000000   0.000000   0.000000   0.000000  0.576318   \n",
              "8   6.151117   0.496372   0.000000   0.000000   0.000000   0.000000  0.500073   \n",
              "9   0.874633   0.099780   0.023386   0.000000   0.000000   0.000000  0.560959   \n",
              "\n",
              "   diameter  amfm_class  label  \n",
              "0  0.100025           1      0  \n",
              "1  0.144414           0      0  \n",
              "2  0.128548           0      1  \n",
              "3  0.114790           0      0  \n",
              "4  0.123572           0      1  \n",
              "5  0.126741           0      1  \n",
              "6  0.139575           0      1  \n",
              "7  0.071071           1      0  \n",
              "8  0.116793           0      1  \n",
              "9  0.109134           0      1  "
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Read the data from csv file\n",
        "col_names = []\n",
        "for i in range(20):\n",
        "    if i == 0:\n",
        "        col_names.append('quality')\n",
        "    if i == 1:\n",
        "        col_names.append('prescreen')\n",
        "    if i >= 2 and i <= 7:\n",
        "        col_names.append('ma' + str(i))\n",
        "    if i >= 8 and i <= 15:\n",
        "        col_names.append('exudate' + str(i))\n",
        "    if i == 16:\n",
        "        col_names.append('euDist')\n",
        "    if i == 17:\n",
        "        col_names.append('diameter')\n",
        "    if i == 18:\n",
        "        col_names.append('amfm_class')\n",
        "    if i == 19:\n",
        "        col_names.append('label')\n",
        "\n",
        "data = pd.read_csv(\"messidor_features.txt\", names = col_names)\n",
        "# data.to_csv(\"/Users/tusharkohli/Desktop/Dm/messidor_features.csv\")\n",
        "print(data.shape)\n",
        "data.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xwf6x6SGnB8T",
        "outputId": "83a25cdf-e1fc-4715-f349-40975b272414"
      },
      "outputs": [
        {
          "ename": "ModuleNotFoundError",
          "evalue": "No module named 'google'",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-d5df0069828e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'google'"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kgZmyaT1lULi"
      },
      "source": [
        "### A. Data prep"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ATYaBdTglULj"
      },
      "source": [
        "Q1. All of the classifiers in `scikit-learn` require that you separate the feature columns from the class label column, so go ahead and do that first. You should end up with two separate data frames: one that contains all of the feature values and one that contains the class labels. \n",
        "\n",
        "Note: Later in this assignment, you may get a warning stating \"a column-vector was passed when a 1d array was expected.\" This indicates that some function wants a _flat array_ of labels, rather than a 2D DataFrame of labels. You can go ahead and transform the labels into a flat array here by doing either `labels.values.ravel()` or `labels.iloc[:,0]`. And you can just use that flat array for everything.\n",
        "\n",
        "Print the `shape` of your features data frame, the shape or len of your labels dataframe or array, and the `head` of the features data frame."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "id": "5OTBzRhclULj",
        "outputId": "1f8b27c5-5193-4127-f29d-72fc0146c94b",
        "scrolled": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1151, 19)\n",
            "(1151,)\n"
          ]
        },
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>quality</th>\n",
              "      <th>prescreen</th>\n",
              "      <th>ma2</th>\n",
              "      <th>ma3</th>\n",
              "      <th>ma4</th>\n",
              "      <th>ma5</th>\n",
              "      <th>ma6</th>\n",
              "      <th>ma7</th>\n",
              "      <th>exudate8</th>\n",
              "      <th>exudate9</th>\n",
              "      <th>exudate10</th>\n",
              "      <th>exudate11</th>\n",
              "      <th>exudate12</th>\n",
              "      <th>exudate13</th>\n",
              "      <th>exudate14</th>\n",
              "      <th>exudate15</th>\n",
              "      <th>euDist</th>\n",
              "      <th>diameter</th>\n",
              "      <th>amfm_class</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>22</td>\n",
              "      <td>22</td>\n",
              "      <td>22</td>\n",
              "      <td>19</td>\n",
              "      <td>18</td>\n",
              "      <td>14</td>\n",
              "      <td>49.895756</td>\n",
              "      <td>17.775994</td>\n",
              "      <td>5.270920</td>\n",
              "      <td>0.771761</td>\n",
              "      <td>0.018632</td>\n",
              "      <td>0.006864</td>\n",
              "      <td>0.003923</td>\n",
              "      <td>0.003923</td>\n",
              "      <td>0.486903</td>\n",
              "      <td>0.100025</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>24</td>\n",
              "      <td>24</td>\n",
              "      <td>22</td>\n",
              "      <td>18</td>\n",
              "      <td>16</td>\n",
              "      <td>13</td>\n",
              "      <td>57.709936</td>\n",
              "      <td>23.799994</td>\n",
              "      <td>3.325423</td>\n",
              "      <td>0.234185</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.003903</td>\n",
              "      <td>0.520908</td>\n",
              "      <td>0.144414</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>62</td>\n",
              "      <td>60</td>\n",
              "      <td>59</td>\n",
              "      <td>54</td>\n",
              "      <td>47</td>\n",
              "      <td>33</td>\n",
              "      <td>55.831441</td>\n",
              "      <td>27.993933</td>\n",
              "      <td>12.687485</td>\n",
              "      <td>4.852282</td>\n",
              "      <td>1.393889</td>\n",
              "      <td>0.373252</td>\n",
              "      <td>0.041817</td>\n",
              "      <td>0.007744</td>\n",
              "      <td>0.530904</td>\n",
              "      <td>0.128548</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>55</td>\n",
              "      <td>53</td>\n",
              "      <td>53</td>\n",
              "      <td>50</td>\n",
              "      <td>43</td>\n",
              "      <td>31</td>\n",
              "      <td>40.467228</td>\n",
              "      <td>18.445954</td>\n",
              "      <td>9.118901</td>\n",
              "      <td>3.079428</td>\n",
              "      <td>0.840261</td>\n",
              "      <td>0.272434</td>\n",
              "      <td>0.007653</td>\n",
              "      <td>0.001531</td>\n",
              "      <td>0.483284</td>\n",
              "      <td>0.114790</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>44</td>\n",
              "      <td>44</td>\n",
              "      <td>44</td>\n",
              "      <td>41</td>\n",
              "      <td>39</td>\n",
              "      <td>27</td>\n",
              "      <td>18.026254</td>\n",
              "      <td>8.570709</td>\n",
              "      <td>0.410381</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.475935</td>\n",
              "      <td>0.123572</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   quality  prescreen  ma2  ma3  ma4  ma5  ma6  ma7   exudate8   exudate9  \\\n",
              "0        1          1   22   22   22   19   18   14  49.895756  17.775994   \n",
              "1        1          1   24   24   22   18   16   13  57.709936  23.799994   \n",
              "2        1          1   62   60   59   54   47   33  55.831441  27.993933   \n",
              "3        1          1   55   53   53   50   43   31  40.467228  18.445954   \n",
              "4        1          1   44   44   44   41   39   27  18.026254   8.570709   \n",
              "\n",
              "   exudate10  exudate11  exudate12  exudate13  exudate14  exudate15    euDist  \\\n",
              "0   5.270920   0.771761   0.018632   0.006864   0.003923   0.003923  0.486903   \n",
              "1   3.325423   0.234185   0.003903   0.003903   0.003903   0.003903  0.520908   \n",
              "2  12.687485   4.852282   1.393889   0.373252   0.041817   0.007744  0.530904   \n",
              "3   9.118901   3.079428   0.840261   0.272434   0.007653   0.001531  0.483284   \n",
              "4   0.410381   0.000000   0.000000   0.000000   0.000000   0.000000  0.475935   \n",
              "\n",
              "   diameter  amfm_class  \n",
              "0  0.100025           1  \n",
              "1  0.144414           0  \n",
              "2  0.128548           0  \n",
              "3  0.114790           0  \n",
              "4  0.123572           0  "
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# your code goes here\n",
        "data_X = data.drop(['label'],axis=1)\n",
        "data_Y = data['label']\n",
        "\n",
        "print(data_X.shape)\n",
        "print(data_Y.shape)\n",
        "data_X.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wd4cs21ylULk"
      },
      "source": [
        "### B. Decision Trees (DT) & Cross Validation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s6i7YI15lULk"
      },
      "source": [
        "**Train/Test Split**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rFcVkcktlULk"
      },
      "source": [
        "Q2. You can train a classifier using the holdout method by splitting your data into a  training set and a  test set, then you can evaluate the classifier on the held-out test set. \n",
        "\n",
        "Let's try this with a decision tree classifier. \n",
        "\n",
        "* Use `sklearn.model_selection.train_test_split` to split your dataset into training and test sets (do an 80%-20% split). Display how many records are in the training set and how many are in the test set.\n",
        "* Use `sklearn.tree.DecisionTreeClassifier` to fit a decision tree classifier on the training set. Use entropy as the split criterion. \n",
        "* Now that the tree has been learned from the training data, we can run the test data through and predict classes for the test data. Use the `predict` method of `DecisionTreeClassifier` to classify the test data. \n",
        "* Then use `sklearn.metrics.accuracy_score` to print out the accuracy of the classifier on the test set."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZbSla7v6lULk",
        "outputId": "05c12574-2a29-48cc-be35-8f3c0a8e3560"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "X_train= (920, 19)  y_train= (920,)\n",
            "X_test= (231, 19)  y_test= (231,)\n",
            "Accuracy on test data is: 0.6233766233766234\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "# Split dataset to training and test sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(data_X, data_Y, test_size=0.20)\n",
        "print(type(X_train))\n",
        "print(\"X_train=\",X_train.shape, \" y_train=\", y_train.shape)\n",
        "print(\"X_test=\",X_test.shape, \" y_test=\", y_test.shape)\n",
        "# Fit Decision Tree Classifier\n",
        "clf = tree.DecisionTreeClassifier(criterion='entropy')\n",
        "clf = clf.fit(X_train, y_train)\n",
        "preds = clf.predict(X_test)\n",
        "print('Accuracy on test data is:', (met.accuracy_score(y_test, preds)))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZGlBL_5plULl"
      },
      "source": [
        "Q3. Note that the DecisionTree classifier has many parameters that can be set. Try tweaking parameters like split criterion, max_depth, min_impurity_decrease, min_samples_leaf, min_samples_split, etc. to see how they affect accuracy. Print the accuracy of a few different variations."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XkoOhdRxlULl",
        "outputId": "27db16b2-f071-458b-e445-d1efe589de1e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0.48917748917748916"
            ]
          },
          "execution_count": 15,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# your code goes here\n",
        "clf = DecisionTreeClassifier(criterion='gini')\n",
        "clf = clf.fit(X_train, y_train)\n",
        "preds = clf.predict(X_test)\n",
        "met.accuracy_score(y_test, preds)\n",
        "clf = DecisionTreeClassifier(splitter='random')\n",
        "clf = clf.fit(X_train, y_train)\n",
        "preds = clf.predict(X_test)\n",
        "met.accuracy_score(y_test, preds)\n",
        "clf = DecisionTreeClassifier(max_depth=3)\n",
        "clf = clf.fit(X_train, y_train)\n",
        "preds = clf.predict(X_test)\n",
        "met.accuracy_score(y_test, preds)\n",
        "clf = DecisionTreeClassifier(min_samples_split=3)\n",
        "clf = clf.fit(X_train, y_train)\n",
        "preds = clf.predict(X_test)\n",
        "met.accuracy_score(y_test, preds)\n",
        "clf = DecisionTreeClassifier(min_samples_leaf=2)\n",
        "clf = clf.fit(X_train, y_train)\n",
        "preds = clf.predict(X_test)\n",
        "met.accuracy_score(y_test, preds)\n",
        "clf = DecisionTreeClassifier(min_impurity_decrease=1)\n",
        "clf = clf.fit(X_train, y_train)\n",
        "preds = clf.predict(X_test)\n",
        "met.accuracy_score(y_test, preds)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_T4_f5RolULl"
      },
      "source": [
        "**Cross Validation**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XymoQstalULl"
      },
      "source": [
        "Q4. You have now built a decision tree and tested it's accuracy using the \"holdout\" method. But as discussed in class, this is not sufficient for estimating generalization accuracy. Instead, we should use Cross Validation to get a better estimate of accuracy. \n",
        "\n",
        "Use `sklearn.model_selection.cross_val_score` to perform 10-fold cross validation on a decision tree. You will pass the FULL dataset into `cross_val_score` which will automatically divide it into the number of folds you tell it to, train a decision tree model on the training set for each fold, and test it on the test set for each fold. It will return a numpy array with the accuracy out of each fold. Average these accuracies to print out the generalization accuracy of the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "arhQh703lULl",
        "outputId": "7b345f2c-a30e-463f-a7a9-ba18712e0110",
        "scrolled": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy: 59.89130434782608\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "clf = tree.DecisionTreeClassifier() # set any params you may want to set\n",
        "\n",
        "# Run a 10-fold cross validation\n",
        "scores = ms.cross_val_score(clf, data_X, data_Y, cv=10)                                     \n",
        "\n",
        "print(\"Accuracy:\", scores.mean()*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UsAyj29nlULl"
      },
      "source": [
        "**Nested Cross Validation**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "53xkbptTlULl"
      },
      "source": [
        "Q5. Now we want to tune our model to use the best parameters to avoid overfitting to our training data. Grid search is an approach to parameter tuning that will methodically build and evaluate a model for each combination of algorithm parameters (hyperparameters) specified in a grid. \n",
        "* Use `sklearn.model_selection.GridSearchCV` to find the best `max_depth`, `max_features`, and `min_samples_leaf` for your tree. Use a 5-fold-CV and 'accuracy' for the scoring criteria.\n",
        "* Try the values [5,10,15,20] for `max_depth` and `min_samples_leaf`. Try [5,10,15] for `max_features`. \n",
        "* Print out the best value for each of the tested parameters (`best_params_`).\n",
        "* Print out the accuracy of the model with these best values (`best_score_`)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D0oejKFklULl",
        "outputId": "9593492b-2aed-4fa3-b7bd-f1d87b918f26"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Best param: {'max_depth': 15, 'max_features': 15, 'min_samples_leaf': 10}\n",
            "Best score: 64.46343740934145\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "\n",
        "clf = tree.DecisionTreeClassifier()\n",
        "\n",
        "params = {\"max_depth\": [5,10,15,20], \"min_samples_leaf\": [5,10,15,20],\n",
        "          \"max_features\": [5,10,15]}\n",
        "grid_search = ms.GridSearchCV(clf, params, cv=3, scoring='accuracy')\n",
        "grid_search.fit(data_X, data_Y)\n",
        "print(\"Best param:\", grid_search.best_params_)\n",
        "print(\"Best score:\", grid_search.best_score_*100)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ElWv24k3lULm"
      },
      "source": [
        "Q6. What you did in Q5 performed the _inner_ loop of a nested CV (no test set was held out). What you did in Q4 performed an _outer_ loop of CV (holds out a test set). Now we need to combine them to perform the nested cross-validation that we discussed in class. To do this, you'll need to pass the a `GridSearchCV` into a `cross_val_score`. \n",
        "\n",
        "What this does is: the `cross_val_score` splits the data in to train and test sets for the first outer fold, and it passes the train set into `GridSearchCV`. `GridSearchCV` then splits that set into train and validation sets for k number of folds (the inner CV loop). The hyper-parameters for which the average score over all inner iterations is best, is reported as the `best_params_`, `best_score_`, and `best_estimator_`(best decision tree). This best decision tree is then evaluated with the test set from the `cross_val_score` (the outer CV loop). And this whole thing is repeated for the remaining k folds of the `cross_val_score` (the outer CV loop). \n",
        "\n",
        "That is a lot of explanation for a very complex (but IMPORTANT) process, which can all be performed with a single line of code!\n",
        "\n",
        "Be patient for this one to run. The nested cross-validation loop can take some time. A [ * ] next to the cell indicates that it is still running.\n",
        "\n",
        "Print the accuracy of your tuned, cross-validated model. This is the official accuracy that you would report for your model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_xaz3f6llULm",
        "outputId": "8fce7652-6e19-4e6b-ab68-e769cd45c49c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy: 63.15817091454272\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "# Run a 10-fold cross validation\n",
        "scores = ms.cross_val_score(grid_search, data_X, data_Y, cv=10)                                     \n",
        "\n",
        "print(\"Accuracy:\", scores.mean()*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xiqgsEN8lULm"
      },
      "source": [
        "### C. Naive Bayes (NB) & Evaluation Metrics\n",
        "\n",
        "`sklearn.naive_bayes.GaussianNB` implements the Gaussian Naive Bayes algorithm for classification. This means that the liklihood of continuous features is estimated using a Gaussian distribution. (Refer to slide 13 of the Naive Bayes powerpoint notes.)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NZMLkh4RlULm"
      },
      "source": [
        "Q7. Create a `sklearn.naive_bayes.GaussianNB` classifier. Use `sklearn.model_selection.cross_val_score` to do a 10-fold cross validation on the classifier. Display the accuracy."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iL8x9OkTlULm",
        "outputId": "715a5377-6eb7-40c2-dcfb-bb934aeb41be"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Scores: [0.61206897 0.56521739 0.63478261 0.53913043 0.6        0.66086957\n",
            " 0.56521739 0.54782609 0.60869565 0.64347826]\n",
            "Accuracy: 59.77286356821588\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "clf = GaussianNB()\n",
        "scores = ms.cross_val_score(clf, data_X, data_Y, cv=10)\n",
        "print(\"Scores:\", scores)\n",
        "print(\"Accuracy:\", scores.mean()*100)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rrlQe-4PlULm"
      },
      "source": [
        "Q8. `cross_val_score` returns the scores of every test fold. There is another function called `cross_val_predict` that returns predicted y values for every record in the test fold. In other words, for each element in the input, `cross_val_predict` returns the prediction that was obtained for that element when it was in the test set. \n",
        "\n",
        "* Use `cross_val_predict` and `sklearn.metrics.confusion_matrix` to print the confusion matrix for the classifier.\n",
        "\n",
        "* Sckit-learn also provides a useful function `sklearn.metrics.classification_report` for evaluating the classifier on a per-class basis. It is a summary of the precision, recall, and F1 score for each class (and support is just the actual class count). Display the classification report for your Naive Bayes classifier."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        },
        "id": "D-mAsC_PlULm",
        "outputId": "726cf765-997c-4278-def3-2a60315736b9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[502  38]\n",
            " [425 186]]\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "'              precision    recall  f1-score   support\\n\\n           0       0.54      0.93      0.68       540\\n           1       0.83      0.30      0.45       611\\n\\n    accuracy                           0.60      1151\\n   macro avg       0.69      0.62      0.56      1151\\nweighted avg       0.69      0.60      0.56      1151\\n'"
            ]
          },
          "execution_count": 52,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# your code goes here\n",
        "predictions = ms.cross_val_predict(clf, data_X, data_Y, cv=10)\n",
        "print(met.confusion_matrix(data_Y, predictions))\n",
        "met.classification_report(data_Y, predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "coxgLzU6lULm"
      },
      "source": [
        "### D. k-Nearest Neighbor (KNN) & Pipelines "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hmm_YJNklULm"
      },
      "source": [
        "For some classification algorithms, scaling of the data is critical (like KNN, SVM, Neural Nets). For other classification algorithms, data scaling is not necessary (like Naive Bayes and Decision Trees). _Take a minute to think about why this is the case!!_ But using scaled data with an algorithm that doesn't explicitly need it to be scaled does not hurt the results of that algorithm."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yeK82mQFlULm"
      },
      "source": [
        "Q10. The distance calculation method is central to the KNN algorithm. By default, `KNeighborsClassifier` uses  Euclidean distance as its metric (but this can be changed). Because of the distance calculations, it is critical to scale the data before running Nearest Neighbor!\n",
        "\n",
        "We discussed why dimensionality reduction may also be needed with KNN because of the curse of dimensionality. So we may want to also perform a dimensionality reduction with PCA before running KNN. PCA should only be performed on scaled data! (Remember that you can also reduce dimensionality by performing feature selection and feature engineering.) \n",
        "\n",
        "An important note about scaling data and dimensionality reduction is that they should only be performed on the **training** data, then you transform the test data into the scaled, PCA space that was found on the training data. (Refer to the concept of [data leakage](https://machinelearningmastery.com/data-leakage-machine-learning/).)\n",
        "\n",
        "So when you are doing cross-validation, the scaling and PCA needs to happen *inside of your CV loop*. This way, it is performed on the training set for the first fold, then the test set is put into that space. On the second fold, it is performed on the trainng set for the second fold, and the test set is put into that space. And so on for the remaining folds. \n",
        "\n",
        "In order to do this with scikit-learn, you must create what's called a `Pipeline` and pass that in to the cross validation. This is a very important concept for Data Mining and Machine Learning, so let's practice it here.\n",
        "\n",
        "Do the following:\n",
        "* Create a `sklearn.preprocessing.StandardScaler` object to standardize the dataset’s features (mean = 0 and variance = 1). (Do not call `fit` on it yet. Just create the `StandardScaler` object.)\n",
        "* Create a `sklearn.decomposition.PCA` object to perform PCA dimensionality reduction. (Do not call `fit` on it yet. Just create the `PCA` object.)\n",
        "* Create a `sklearn.neighbors.KNeighborsClassifier`. The number of neighbors defaults to 5 (k=5). Go ahead and change it to 7. (Do not call `fit` on it yet. Just create the `KNeighborsClassifier` object.)\n",
        "* Create a `sklearn.pipeline.Pipeline` object and set the `steps` to the scaler, the PCA, and the KNN objects that you just created. \n",
        "* Pass the `pipeline` object in to a `cross_val_score` as the estimator, along with the features and the labels, and use a 5-fold-CV. \n",
        "\n",
        "In each fold of the cross validation, the training phase will use _only_ the training data for scaling, PCA, and training the model. Then the testing phase will scale & transform the test data into the PCA space (found on the training data) and run the test data through the trained classifier, to return an accuracy measurement for each fold. Print the average accuracy across all 5 folds. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2tkToU_klULm",
        "outputId": "fd155346-630c-4549-b580-583708efd988"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Scores: [0.62770563 0.64347826 0.63913043 0.58695652 0.5826087 ]\n",
            "Accuracy: 61.59759081498212\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "ss = StandardScaler()\n",
        "pca = PCA()\n",
        "knn_clf = KNeighborsClassifier(n_neighbors=7)\n",
        "pipeline = Pipeline(steps=[('scaler', ss), ('pca', pca), ('knn', knn_clf)])\n",
        "scores = ms.cross_val_score(pipeline, data_X, data_Y, cv=5)\n",
        "print(\"Scores:\", scores)\n",
        "print(\"Accuracy:\", scores.mean()*100)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iDryLcOalULn"
      },
      "source": [
        "Q11. Another important part of KNN is choosing the best number of neighbors (tuning the hyperparameter, k). We can use nested cross validation to do this. Let's try k values from 1-25 to find the best one. \n",
        "\n",
        "We _also_ want to find the best number of dimensions to project down onto using PCA. We can use nested cross validation to do this as well. Let's try from 5-19 dimensions.\n",
        "\n",
        "* Starter code is provided to create the \"parameter grid\" to search. You will need to change this code! Where I have \"knn__n_neighbors\", this indicates that I want to tune the \"n_neighbors\" parameter in the \"knn\" part of the pipeline. When you created your pipeline above, you named the KNN part of the pipeline with a string. You should replace \"knn\" in the param_grid below with whatever you named your KNN part of the pipeline: **<replace_this>__n_neighbors.** Do the same for the PCA part of the pipeline.\n",
        "* Create a `sklearn.model_selection.GridSearchCV` and pass in the pipeline, the param_grid, and set it to a 5-fold-CV.\n",
        "* Now, on that `GridSearchCV` object, call `fit` and pass in the features and labels.\n",
        "* Show the best number of dimensions and best number of neighbors for this dataset by printing the `best_params_` from the `GridSearchCV`.\n",
        "* Also print the accuracy when using this best number of dimensions and neighbors by printing the `best_score_` from the `GridSearchCV`.\n",
        "\n",
        "Be patient, this can take some time to run. It is trying every combination of dimensions from 5-19 with every k from 1-25! A [ * ] next to the cell indicates that it is still running."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "beRX5adMlULn",
        "outputId": "3a2a5bfd-eadf-49d6-a612-552b4beef633"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Best params: {'knn__n_neighbors': 23, 'pca__n_components': 14}\n",
            "Best accuracy: 66.11518915866743\n"
          ]
        }
      ],
      "source": [
        "'''\n",
        "On the \"pca\" part of the pipeline, \n",
        "tune the n_components parameter,\n",
        "by trying the values 1-19.\n",
        "\n",
        "On the \"knn\" part of the pipeline, \n",
        "tune the n_neighbors parameter,\n",
        "by trying the values 1-30.\n",
        "'''\n",
        "param_grid = {\n",
        "    'pca__n_components': list(range(1, 20)),\n",
        "    'knn__n_neighbors': list(range(1, 31))\n",
        "}\n",
        "\n",
        "# your code goes here\n",
        "grid_search = ms.GridSearchCV(pipeline, param_grid, cv=5, scoring='accuracy')\n",
        "grid_search.fit(data_X, data_Y)\n",
        "print(\"Best params:\", grid_search.best_params_)\n",
        "print(\"Best accuracy:\", grid_search.best_score_*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tbp1UY-elULn"
      },
      "source": [
        "Q12. In Q11, we did not hold out a test set. The accuracy reported out is on the _validation_ set. So now we need to wrap the whole process in another cross-validation to perform a nested cross-validation and report the accuarcy of this KNN model on unseen test data. This is the official accuracy you would report on this model.\n",
        "\n",
        "You'll need to pass the `GridSearchCV` into a `cross_val_score`, just as you did with the decision tree. Use a 5-fold-CV for the outer loop. \n",
        "\n",
        "Again, be patient for this one to run. The nested cross-validation loop can take some time. It is doing what it did above in Q11 five times. A [ * ] next to the cell indicates that it is still running. (Just for comparison, mine takes about 2 mins to run and the fan revs up so it sounds like my computer is going to explode. All computers are different, so yours could take shorter or longer...)\n",
        "\n",
        "<img src=\"https://github.com/Tushar1729-k/scikit-learn/blob/main/model_is_training.png?raw=1\" width=\"250\">"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 55,
      "metadata": {
        "id": "npzAg1vClULn"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Scores: [0.59307359 0.68695652 0.63913043 0.59565217 0.64782609]\n",
            "Accuracy: 63.25277620929795\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "scores = ms.cross_val_score(grid_search, data_X, data_Y, cv=5)\n",
        "print(\"Scores:\", scores)\n",
        "print(\"Accuracy:\", scores.mean()*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QpBAz-s3lULn"
      },
      "source": [
        "### E. Support Vector Machines (SVM)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Viwu3osOlULn"
      },
      "source": [
        "Q13. Now put it all together with an SVM. \n",
        "* Create a `pipeline` that includes scaling, PCA, and an `sklearn.svm.SVC`.\n",
        "* Create a parameter grid that tries number of dimensions from 5-19 and SVM kernels `linear`, `rbf` and `poly`.\n",
        "* Create a `GridSearchCV` for the inner CV loop. Use a 5-fold CV.\n",
        "* Run a `cross_val_predict` with a 10-fold CV for the outer loop. \n",
        "* Print out the accuracy and the classification report of using an SVM classifier on this data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 56,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AWaftbg8lULn",
        "outputId": "13300967-6cfd-4a04-f727-d5339379a21d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.66      0.84      0.74       540\n",
            "           1       0.82      0.62      0.71       611\n",
            "\n",
            "    accuracy                           0.73      1151\n",
            "   macro avg       0.74      0.73      0.72      1151\n",
            "weighted avg       0.74      0.73      0.72      1151\n",
            "\n",
            "Accuracy: 0.7254561251086012\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "ss = StandardScaler()\n",
        "pca = PCA()\n",
        "svc = SVC()\n",
        "pipeline = Pipeline(steps=[('scaler', ss), ('pca', pca), ('svc', svc)])\n",
        "params_grid = {\n",
        "    'pca__n_components': list(range(5, 20)),\n",
        "    'svc__kernel': ['linear', 'rbf', 'poly']\n",
        "}\n",
        "grid_search = ms.GridSearchCV(pipeline, params_grid, cv=5, scoring='accuracy')\n",
        "predicts = ms.cross_val_predict(grid_search, data_X, data_Y, cv=10)\n",
        "print(met.classification_report(data_Y, predicts))\n",
        "print(\"Accuracy:\", met.accuracy_score(data_Y, predicts))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nus9IbSjlULn"
      },
      "source": [
        "### F. Neural Networks (NN)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z8jjyxZ_lULn"
      },
      "source": [
        "Q14. Train a multi-layer perceptron with a single hidden layer using `sklearn.neural_network.MLPClassifier`. \n",
        "* Create a pipeline with scaling and a neural net. (No PCA on this one. But scaling is critical to neural nets.)\n",
        "* Use `GridSearchCV` with 5 fold cross validation to find the best hidden layer size and the best activation function. \n",
        "* Try values of `hidden_layer_sizes` ranging from `(30,)` to `(60,)` by increments of 10.\n",
        "* Try activation functions `logistic`, `tanh`, `relu`.\n",
        "* Wrap your `GridSearchCV` in a 5-fold `cross_val_score` and report the accuracy of your neural net.\n",
        "\n",
        "Be patient, as this can take a few minutes to run. You may get ConvergenceWarnings as it runs - that is fine."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SamqDVtilULn",
        "outputId": "39ed5097-0315-43ce-d10f-20a4107c07a3"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy: 73.5855448898927\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "simplefilter(\"ignore\", category=ConvergenceWarning)\n",
        "ss = StandardScaler()\n",
        "nn = MLPClassifier()\n",
        "pipeline = Pipeline(steps=[('scaler', ss), ('nn', nn)])\n",
        "params_grid = {\n",
        "    'nn__hidden_layer_sizes': list((int(item*10),) for item in range(3, 7)),\n",
        "    'nn__activation': ['logistic', 'tanh', 'relu']\n",
        "}\n",
        "grid_search = ms.GridSearchCV(pipeline, params_grid, cv=5, scoring='accuracy')\n",
        "scores = ms.cross_val_score(grid_search, data_X, data_Y, cv=5)\n",
        "print(\"Accuracy:\", scores.mean()*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GhFmnFKulULn"
      },
      "source": [
        "### G. Ensemble Classifiers\n",
        "\n",
        "Ensemble classifiers combine the predictions of multiple base estimators to improve the accuracy of the predictions. One of the key assumptions that ensemble classifiers make is that the base estimators are built independently (so they are diverse)."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M4IU8CeLlULn"
      },
      "source": [
        "**Random Forests**\n",
        "\n",
        "Q15. Use `sklearn.ensemble.RandomForestClassifier` to classify the data. Scaling the data is not necessary for Decision Trees (take a minute to think about why). So, no need for a pipeline here.\n",
        "\n",
        "Use a `GridSearchCV` with a 5-fold CV to tune the hyperparameters to get the best results. \n",
        "* Try `max_depth` ranging from 35-45\n",
        "* Try `min_samples_leaf` of 8, 10, 12\n",
        "* Try `max_features` of `\"sqrt\"` and `\"log2\"`\n",
        "\n",
        "Wrap your GridSearchCV in a cross_val_score with 5-fold CV to report the accuracy of the model.\n",
        "\n",
        "Be patient, this can take a few minutes to run."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JHIHlo6IlULn",
        "outputId": "68afa3fc-962e-47fe-9a1d-d1d690ef3e40"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy: 67.41690193864108\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "rfc = RandomForestClassifier()\n",
        "params_grid = {\n",
        "    'max_depth': list(range(35, 46)),\n",
        "    'min_samples_leaf': [8, 10, 12],\n",
        "    'max_features': ['sqrt', 'log2']\n",
        "}\n",
        "grid_search = ms.GridSearchCV(rfc, params_grid, cv=5, scoring='accuracy')\n",
        "scores = ms.cross_val_score(grid_search, data_X, data_Y, cv=5)\n",
        "print(\"Accuracy:\", scores.mean()*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GcG-Ge2blULn"
      },
      "source": [
        "**AdaBoost**\n",
        "\n",
        "Random Forests are a kind of averaging ensemble classifier, where several estimators are built independently and then to average their predictions (by taking a vote). There is another method of training ensemble classifiers called *boosting*. Here the classifiers are trained sequentially and each time the sampling of the training set depends on the performance of previously generated models.\n",
        "\n",
        "Q16. Evaluate a `sklearn.ensemble.AdaBoostClassifier` classifier on the data. By default, `AdaBoostClassifier` uses decision stumps as the base classifiers (but this can be changed). Use 150 base classifiers to make an `AdaBoostClassifier` and evaluate it's accuracy with a 5-fold-CV."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2nfJqLVglULn",
        "outputId": "d8726fac-5995-4edc-dcf8-80e089feb67d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy: 69.76510446075663\n"
          ]
        }
      ],
      "source": [
        "# your code goes here\n",
        "abc = AdaBoostClassifier(n_estimators=150)\n",
        "scores = ms.cross_val_score(abc, data_X, data_Y, cv=5)\n",
        "print(\"Accuracy:\", scores.mean()*100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xS_jZ7YAlULo"
      },
      "source": [
        "### H. Build your final model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PRxWOI5LlULo"
      },
      "source": [
        "Now you have tested all kinds of classifiers on this data. Some have performed better than others. \n",
        "\n",
        "Q17. We may not want to deploy any of these models in the real world to actually diagnose patients because the accuracies are not high enough. What can we do to improve the accuracy rates? Answer as a comment:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "id": "jc2wiYLKlULo"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'\\nAnswer here as a comment.\\nWe could use a higher sample size to improve accuracy and not overfit \\nWe could improve our feature engineering by trying to collect more data,\\nseeing if adding features could improve our model \\nWe could tune more hyperparameters than we did\\n'"
            ]
          },
          "execution_count": 48,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "'''\n",
        "Answer here as a comment.\n",
        "We could use a higher sample size to improve accuracy and not overfit \n",
        "We could improve our feature engineering by trying to collect more data,\n",
        "seeing if adding features could improve our model \n",
        "We could tune more hyperparameters than we did\n",
        "'''"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WiPakxeUlULo"
      },
      "source": [
        "Q18. Let's say we *did* get to the point where we had a model with very high accuracy and we want to deploy that model and use it for real-world predictions.\n",
        "\n",
        "* Let's say we're going to deploy our SVM classifier.\n",
        "* We need to make one final version of this model, where we use ALL of our available data for training (we do not hold out a test set this time, so no outer cross-validation loop). \n",
        "* We need to tune the parameters of the model on the FULL dataset, so copy the code you entered for Q13, but remove the outer cross validation loop (remove `cross_val_score`). Just run the `GridSearchCV` by calling `fit` on it and passing in the full dataset. This results in the final trained model with the best parameters for the full dataset. You can print out `best_params_` to see what they are.\n",
        "* The accuracy of this model is what you assessed and reported in Q13.\n",
        "\n",
        "\n",
        "* Use the `pickle` package to save your model. We have provided the lines of code for you, just make sure your final model gets passed in to `pickle.dump()`. This will save your model to a file called finalized_model.sav in your current working directory. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "j6W-bkRGlULo",
        "outputId": "29fe064d-a346-43b0-ceb2-f24e6d2db738"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'pca__n_components': 18, 'svc__kernel': 'linear'}\n",
            "Accuracy: 72.54658385093168\n"
          ]
        }
      ],
      "source": [
        "import pickle\n",
        "\n",
        "# your code goes here\n",
        "ss = StandardScaler()\n",
        "pca = PCA()\n",
        "svc = SVC()\n",
        "pipeline = Pipeline(steps=[('scaler', ss), ('pca', pca), ('svc', svc)])\n",
        "params_grid = {\n",
        "    'pca__n_components': list(range(5, 20)),\n",
        "    'svc__kernel': ['linear', 'rbf', 'poly']\n",
        "}\n",
        "grid_search = ms.GridSearchCV(pipeline, params_grid, cv=5, scoring='accuracy')\n",
        "grid_search.fit(data_X, data_Y)\n",
        "# predicts = ms.cross_val_predict(grid_search, data_X, data_Y, cv=10)\n",
        "# met.classification_report(data_Y, predicts)\n",
        "print(grid_search.best_params_)\n",
        "print(\"Accuracy:\", grid_search.best_score_*100)\n",
        "# replace this final_model with your final model\n",
        "final_model = grid_search\n",
        "\n",
        "filename = 'finalized_model.sav'\n",
        "pickle.dump(final_model, open(filename, 'wb'))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hQ2EI9QVlULo"
      },
      "source": [
        "Q19. Now if someone wants to use your trained, saved classifier to classify a new record, they can load the saved model and just call predict on it. \n",
        "* Given this new record, classify it with your saved model and print out either \"Negative for disease\" or \"Positive for disease.\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tuZqQrEPlULo",
        "outputId": "f42a0aa0-d754-4474-e020-ae60128f1629"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Positive for disease\n"
          ]
        }
      ],
      "source": [
        "# some time later...\n",
        "\n",
        "# use this as the new record to classify\n",
        "record = [ 0.05905386, 0.2982129, 0.68613149, 0.75078865, 0.87119216, 0.88615694,\n",
        "  0.93600623, 0.98369184, -0.47426472, -0.57642756, -0.53115361, -0.42789774,\n",
        " -0.21907738, -0.20090532, -0.21496782, -0.2080998, 0.06692373, -2.81681183,\n",
        " -0.7117194 ]\n",
        "\n",
        " \n",
        "# load the model from disk\n",
        "loaded_model = pickle.load(open(filename, 'rb'))\n",
        "\n",
        "# your code goes here\n",
        "prediction = loaded_model.predict([record])[0]\n",
        "print(\"Positive for disease\" if prediction else \"Negative for disease\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "HW3_ScikitLearn.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.7.3 64-bit",
      "metadata": {
        "interpreter": {
          "hash": "5edc29c2ed010d6458d71a83433b383a96a8cbd3efe8531bc90c4b8a5b8bcec9"
        }
      },
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3-final"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}