LogisticRegressionModel(with PCA).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TeamB_LogisticRegression.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j35BLd3kvobm"
      },
      "source": [
        "# Logistic Regression Model (with PCA) - Historical Bitcoin Data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Chf2Arr-vobt"
      },
      "source": [
        "### Importing Essential Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DbNzsLrqrKeZ"
      },
      "source": [
        "import numpy as np \n",
        "import pandas as pd \n",
        "import matplotlib.pyplot as plt \n",
        "%matplotlib inline\n",
        "import seaborn as sns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZLQuAmJ-vobx"
      },
      "source": [
        "### Reading Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ejX6n8l1rTRE"
      },
      "source": [
        "df = pd.read_pickle('./dfe.pkl')    # reading 5 yr cleaned data pickle file"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "It_NwhZqreZe",
        "outputId": "342fb0f4-1d47-4e9e-9af9-e9f01de30f32"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Date</th>\n",
              "      <th>Price</th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Vol.</th>\n",
              "      <th>Change %</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Oct 16, 2015</td>\n",
              "      <td>262.9</td>\n",
              "      <td>254.4</td>\n",
              "      <td>267.3</td>\n",
              "      <td>253.9</td>\n",
              "      <td>95180.0</td>\n",
              "      <td>3.31</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Oct 17, 2015</td>\n",
              "      <td>269.6</td>\n",
              "      <td>262.9</td>\n",
              "      <td>275.4</td>\n",
              "      <td>261.7</td>\n",
              "      <td>113840.0</td>\n",
              "      <td>2.55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Oct 18, 2015</td>\n",
              "      <td>261.7</td>\n",
              "      <td>269.6</td>\n",
              "      <td>272.4</td>\n",
              "      <td>259.5</td>\n",
              "      <td>52270.0</td>\n",
              "      <td>-2.93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Oct 19, 2015</td>\n",
              "      <td>263.8</td>\n",
              "      <td>261.7</td>\n",
              "      <td>266.0</td>\n",
              "      <td>259.9</td>\n",
              "      <td>58170.0</td>\n",
              "      <td>0.82</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Oct 20, 2015</td>\n",
              "      <td>269.8</td>\n",
              "      <td>263.8</td>\n",
              "      <td>272.1</td>\n",
              "      <td>262.7</td>\n",
              "      <td>75400.0</td>\n",
              "      <td>2.25</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "           Date  Price   Open   High    Low      Vol.  Change %\n",
              "0  Oct 16, 2015  262.9  254.4  267.3  253.9   95180.0      3.31\n",
              "1  Oct 17, 2015  269.6  262.9  275.4  261.7  113840.0      2.55\n",
              "2  Oct 18, 2015  261.7  269.6  272.4  259.5   52270.0     -2.93\n",
              "3  Oct 19, 2015  263.8  261.7  266.0  259.9   58170.0      0.82\n",
              "4  Oct 20, 2015  269.8  263.8  272.1  262.7   75400.0      2.25"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pha4343W3U4I",
        "outputId": "b833612d-e9cb-4209-f1ca-6c89b8d944de"
      },
      "source": [
        "df.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(2193, 7)"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NNE1j5UQ36fh",
        "outputId": "f78b4adf-dd20-4ec6-e5b8-732246d51bba"
      },
      "source": [
        "print(\"Number of data points:\", df.shape[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Number of data points: 2193\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LFXrbcHONLcT"
      },
      "source": [
        "df[\"Date\"] = pd.to_datetime(df[\"Date\"], infer_datetime_format=True)\n",
        "df = df.set_index(\"Date\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "fTTokLyaEPQ5",
        "outputId": "cd3a49d3-73f6-4eee-bdc7-40afd01d1881"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Price</th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Vol.</th>\n",
              "      <th>Change %</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Date</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2015-10-16</th>\n",
              "      <td>262.9</td>\n",
              "      <td>254.4</td>\n",
              "      <td>267.3</td>\n",
              "      <td>253.9</td>\n",
              "      <td>95180.0</td>\n",
              "      <td>3.31</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-17</th>\n",
              "      <td>269.6</td>\n",
              "      <td>262.9</td>\n",
              "      <td>275.4</td>\n",
              "      <td>261.7</td>\n",
              "      <td>113840.0</td>\n",
              "      <td>2.55</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-18</th>\n",
              "      <td>261.7</td>\n",
              "      <td>269.6</td>\n",
              "      <td>272.4</td>\n",
              "      <td>259.5</td>\n",
              "      <td>52270.0</td>\n",
              "      <td>-2.93</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-19</th>\n",
              "      <td>263.8</td>\n",
              "      <td>261.7</td>\n",
              "      <td>266.0</td>\n",
              "      <td>259.9</td>\n",
              "      <td>58170.0</td>\n",
              "      <td>0.82</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-20</th>\n",
              "      <td>269.8</td>\n",
              "      <td>263.8</td>\n",
              "      <td>272.1</td>\n",
              "      <td>262.7</td>\n",
              "      <td>75400.0</td>\n",
              "      <td>2.25</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            Price   Open   High    Low      Vol.  Change %\n",
              "Date                                                      \n",
              "2015-10-16  262.9  254.4  267.3  253.9   95180.0      3.31\n",
              "2015-10-17  269.6  262.9  275.4  261.7  113840.0      2.55\n",
              "2015-10-18  261.7  269.6  272.4  259.5   52270.0     -2.93\n",
              "2015-10-19  263.8  261.7  266.0  259.9   58170.0      0.82\n",
              "2015-10-20  269.8  263.8  272.1  262.7   75400.0      2.25"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "f3r4FwoNvob7"
      },
      "source": [
        "### Encoding Change % Column in 0/1"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3u7CU_0yaQrW"
      },
      "source": [
        "def partition(x):\n",
        "  if x<0:\n",
        "    return 1\n",
        "  return 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qqTv5rjSc7qJ"
      },
      "source": [
        "bit_pred=df['Change %']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AqaIqGGadLMa"
      },
      "source": [
        "up_down=bit_pred.map(partition)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IAOkhOvTdYHY"
      },
      "source": [
        "df['Change %']=up_down"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "id": "fsUxaFCNdjPS",
        "outputId": "d76a805b-d6fd-423d-f97c-323b1b6eb651"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Price</th>\n",
              "      <th>Open</th>\n",
              "      <th>High</th>\n",
              "      <th>Low</th>\n",
              "      <th>Vol.</th>\n",
              "      <th>Change %</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>Date</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>2015-10-16</th>\n",
              "      <td>262.9</td>\n",
              "      <td>254.4</td>\n",
              "      <td>267.3</td>\n",
              "      <td>253.9</td>\n",
              "      <td>95180.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-17</th>\n",
              "      <td>269.6</td>\n",
              "      <td>262.9</td>\n",
              "      <td>275.4</td>\n",
              "      <td>261.7</td>\n",
              "      <td>113840.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-18</th>\n",
              "      <td>261.7</td>\n",
              "      <td>269.6</td>\n",
              "      <td>272.4</td>\n",
              "      <td>259.5</td>\n",
              "      <td>52270.0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-19</th>\n",
              "      <td>263.8</td>\n",
              "      <td>261.7</td>\n",
              "      <td>266.0</td>\n",
              "      <td>259.9</td>\n",
              "      <td>58170.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2015-10-20</th>\n",
              "      <td>269.8</td>\n",
              "      <td>263.8</td>\n",
              "      <td>272.1</td>\n",
              "      <td>262.7</td>\n",
              "      <td>75400.0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "            Price   Open   High    Low      Vol.  Change %\n",
              "Date                                                      \n",
              "2015-10-16  262.9  254.4  267.3  253.9   95180.0         0\n",
              "2015-10-17  269.6  262.9  275.4  261.7  113840.0         0\n",
              "2015-10-18  261.7  269.6  272.4  259.5   52270.0         1\n",
              "2015-10-19  263.8  261.7  266.0  259.9   58170.0         0\n",
              "2015-10-20  269.8  263.8  272.1  262.7   75400.0         0"
            ]
          },
          "execution_count": 12,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QbziMxHcdmtg",
        "outputId": "664eb93e-4b44-4432-8610-0895a88e2e1b"
      },
      "source": [
        "df['Change %'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    1208\n",
              "1     985\n",
              "Name: Change %, dtype: int64"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wh2UwynVds5u"
      },
      "source": [
        "y=df['Change %']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CQT3qhUXfnFA",
        "outputId": "f53ff1d6-1ab7-4754-be51-9bdee2a99a57"
      },
      "source": [
        "df.drop(['Change %'], axis=1, inplace=True)\n",
        "print(df.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(2193, 5)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6oAGh0gPvocF"
      },
      "source": [
        "### Scaling Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f9H8Cz2WfuQF"
      },
      "source": [
        "from sklearn.preprocessing import StandardScaler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PVpN_7lujtU6",
        "outputId": "8751263f-3a3f-49a5-cd04-ce2c74ed7c6d"
      },
      "source": [
        "scaler=StandardScaler()\n",
        "scaler.fit(df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "StandardScaler()"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3UR7fEd0kF-Z",
        "outputId": "02cc404d-2fa4-49e3-b4b8-cfb2b37a5065"
      },
      "source": [
        "scaled_data=scaler.transform(df)\n",
        "scaled_data"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[-0.75071195, -0.75134418, -0.75001256, -0.75202752, -0.31790091],\n",
              "       [-0.75024567, -0.75075109, -0.74946467, -0.75146385, -0.30352852],\n",
              "       [-0.75079546, -0.75028359, -0.74966759, -0.75162284, -0.35095124],\n",
              "       ...,\n",
              "       [ 3.22191766,  3.23396813,  3.18937757,  3.33910891, -0.34512064],\n",
              "       [ 3.52303019,  3.2324191 ,  3.48607372,  3.33968703, -0.29764401],\n",
              "       [ 3.46656159,  3.53392016,  3.44795094,  3.57722962, -0.35275356]])"
            ]
          },
          "execution_count": 18,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ehUYUbeQvocJ"
      },
      "source": [
        "### Performing Principal Component Analysis"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ZFAqLeTkLhx"
      },
      "source": [
        "from sklearn.decomposition import PCA"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4oIKA4KpkW2H",
        "outputId": "d0076241-94e1-4b22-a878-c6081f9f90e8"
      },
      "source": [
        "pca=PCA(n_components=2)\n",
        "pca.fit(scaled_data)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "PCA(n_components=2)"
            ]
          },
          "execution_count": 20,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRZ-cin2kb0U"
      },
      "source": [
        "x_pca=pca.transform(scaled_data)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dbHW1SLvkkFr",
        "outputId": "babfbbae-8f4d-47bb-c831-1dded6fe1fc5"
      },
      "source": [
        "print('Shape of scaled data:', scaled_data.shape)\n",
        "print('Shape of PCA data:', x_pca.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Shape of scaled data: (2193, 5)\n",
            "Shape of PCA data: (2193, 2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sA7Ag7IQvocN"
      },
      "source": [
        "### Train-Test Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sfqdPSBymhoq"
      },
      "source": [
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dRhP3f91mPY9",
        "outputId": "b5494b8d-17a8-4eb8-fbaa-c0f0ab74adab"
      },
      "source": [
        "X_train, X_test, y_train, y_test = train_test_split(x_pca, y, stratify = y, test_size = 0.3)\n",
        "print('X-Train Shape:', X_train.shape)\n",
        "print('Y-Train Shape:', y_train.shape)\n",
        "print('X-Test Shape:', X_test.shape)\n",
        "print('Y-Test Shape:', y_test.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "X-Train Shape: (1535, 2)\n",
            "Y-Train Shape: (1535,)\n",
            "X-Test Shape: (658, 2)\n",
            "Y-Test Shape: (658,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F0rBHd32vocP",
        "outputId": "2a7f99aa-bfae-4491-8cb5-59a8b332be54"
      },
      "source": [
        "X_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[-1.45636245e+00, -4.30066464e-01],\n",
              "       [ 3.99700670e+00,  1.78196717e-03],\n",
              "       [-1.44731227e+00, -3.76024444e-01],\n",
              "       ...,\n",
              "       [-2.65544712e-01,  1.52737694e-01],\n",
              "       [-1.43256104e+00, -3.44801291e-01],\n",
              "       [-4.18229370e-01, -3.59679866e-01]])"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3cxd3tiIvocQ",
        "outputId": "53ccccf4-6c97-4be8-b0d2-554ae5742101"
      },
      "source": [
        "y_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Date\n",
              "2016-01-12    1\n",
              "2021-01-08    0\n",
              "2016-05-31    1\n",
              "2019-06-16    0\n",
              "2018-04-21    0\n",
              "             ..\n",
              "2020-09-29    0\n",
              "2019-02-11    1\n",
              "2019-06-18    1\n",
              "2016-07-07    1\n",
              "2017-11-19    0\n",
              "Name: Change %, Length: 1535, dtype: int64"
            ]
          },
          "execution_count": 26,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jkYuxidavocR",
        "outputId": "8270bca8-fa03-43bc-b242-5a693fda5a6d"
      },
      "source": [
        "X_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[-0.2167594 , -0.05823246],\n",
              "       [-1.01677173, -0.35402218],\n",
              "       [-1.06030993, -0.08116417],\n",
              "       ...,\n",
              "       [ 0.1311542 , -0.31220849],\n",
              "       [-0.07619402,  0.1657587 ],\n",
              "       [-0.18105675,  0.00570893]])"
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TA7SQfKEvocR",
        "outputId": "89ef2dd8-b548-4a6c-8d1b-af3ab739216c"
      },
      "source": [
        "y_test"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Date\n",
              "2020-06-17    1\n",
              "2017-09-22    1\n",
              "2019-02-07    1\n",
              "2018-02-20    0\n",
              "2017-06-15    1\n",
              "             ..\n",
              "2021-07-01    1\n",
              "2019-05-27    0\n",
              "2020-10-20    0\n",
              "2019-07-19    1\n",
              "2020-06-05    1\n",
              "Name: Change %, Length: 658, dtype: int64"
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c-nHOMFMvocS"
      },
      "source": [
        "### Performing Hyperparameter Tuning Using Grid SearchCV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "du4wQZdyvocT"
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.pipeline import Pipeline\n",
        "pipeline_lr = Pipeline([('scalar1',StandardScaler()),\n",
        "                     ('pca1',PCA(n_components=2)),\n",
        "                     ('lr_classifier',LogisticRegression(random_state=0))])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YsWHlO5ZvocU"
      },
      "source": [
        "from sklearn.tree import DecisionTreeClassifier\n",
        "pipeline_dt=Pipeline([('scalar2',StandardScaler()),\n",
        "                     ('pca2',PCA(n_components=2)),\n",
        "                     ('dt_classifier',DecisionTreeClassifier())])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WtOWk55LvocU"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "pipeline_randomforest=Pipeline([('scalar3',StandardScaler()),\n",
        "                     ('pca3',PCA(n_components=2)),\n",
        "                     ('rf_classifier',RandomForestClassifier())])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qYzJORUUvocV"
      },
      "source": [
        "pipelines = [pipeline_lr, pipeline_dt, pipeline_randomforest]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yd1jCwLRvocW"
      },
      "source": [
        "best_accuracy=0.0\n",
        "best_classifier=0\n",
        "best_pipeline=\"\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D8UfIFolvocW"
      },
      "source": [
        "# Dictionary of pipelines and classifier types for ease of reference\n",
        "pipe_dict = {0: 'Logistic Regression', 1: 'Decision Tree', 2: 'RandomForest'}\n",
        "\n",
        "# Fit the pipelines\n",
        "for pipe in pipelines:\n",
        "    pipe.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_fleKeyEvocX",
        "outputId": "cdaf3175-cebb-41c7-c8eb-7cbfbeb7f48b"
      },
      "source": [
        "for i,model in enumerate(pipelines):\n",
        "    print(\"{} Test Accuracy: {}\".format(pipe_dict[i], model.score(X_test,y_test)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Logistic Regression Test Accuracy: 0.5547112462006079\n",
            "Decision Tree Test Accuracy: 0.47112462006079026\n",
            "RandomForest Test Accuracy: 0.48024316109422494\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xC4znwkpvocY"
      },
      "source": [
        "* Logistic Regression seems to be a better Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5skx6_AlvocY"
      },
      "source": [
        "from sklearn.model_selection import GridSearchCV"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n4c6MTsLvocZ",
        "outputId": "856634df-93c0-4f55-e37b-7ad1682cccd8"
      },
      "source": [
        "# define models and parameters\n",
        "model=LogisticRegression()\n",
        "solvers = ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']\n",
        "penalty = ['l1','l2','elasticnet', 'none']\n",
        "c = np.logspace(0, 4, 10)\n",
        "\n",
        "# define grid search\n",
        "grid = dict(solver=solvers, penalty=penalty, C=c)\n",
        "gridsearch = GridSearchCV(model, grid, cv=5, verbose=0, n_jobs=-1)\n",
        "best_model = gridsearch.fit(X_train,y_train)\n",
        "\n",
        "# summarize results\n",
        "means = best_model.cv_results_['mean_test_score']\n",
        "stds = best_model.cv_results_['std_test_score']\n",
        "params = best_model.cv_results_['params']\n",
        "\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"accuracy: %f (%f) using: %r\" % (mean, stdev, param))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1.0, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1.0, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1.0, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1.0, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.538111 (0.020332) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.537459 (0.021607) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'saga'}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\harsh\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:918: UserWarning: One or more of the test scores are non-finite: [       nan        nan 0.53745928        nan 0.53811075 0.53811075\n",
            " 0.53811075 0.53745928 0.53811075 0.53811075        nan        nan\n",
            "        nan        nan        nan 0.53745928 0.53745928        nan\n",
            " 0.53745928 0.53745928        nan        nan 0.53745928        nan\n",
            " 0.53811075 0.53745928 0.53745928 0.53745928 0.53745928 0.53745928\n",
            "        nan        nan        nan        nan        nan 0.53745928\n",
            " 0.53745928        nan 0.53745928 0.53745928        nan        nan\n",
            " 0.53745928        nan 0.53811075 0.53745928 0.53745928 0.53745928\n",
            " 0.53745928 0.53745928        nan        nan        nan        nan\n",
            "        nan 0.53745928 0.53745928        nan 0.53745928 0.53745928\n",
            "        nan        nan 0.53745928        nan 0.53811075 0.53745928\n",
            " 0.53745928 0.53745928 0.53745928 0.53745928        nan        nan\n",
            "        nan        nan        nan 0.53745928 0.53745928        nan\n",
            " 0.53745928 0.53745928        nan        nan 0.53745928        nan\n",
            " 0.53745928 0.53745928 0.53745928 0.53745928 0.53745928 0.53745928\n",
            "        nan        nan        nan        nan        nan 0.53745928\n",
            " 0.53745928        nan 0.53745928 0.53745928        nan        nan\n",
            " 0.53745928        nan 0.53745928 0.53745928 0.53745928 0.53745928\n",
            " 0.53745928 0.53745928        nan        nan        nan        nan\n",
            "        nan 0.53745928 0.53745928        nan 0.53745928 0.53745928\n",
            "        nan        nan 0.53745928        nan 0.53745928 0.53745928\n",
            " 0.53745928 0.53745928 0.53745928 0.53745928        nan        nan\n",
            "        nan        nan        nan 0.53745928 0.53745928        nan\n",
            " 0.53745928 0.53745928        nan        nan 0.53745928        nan\n",
            " 0.53745928 0.53745928 0.53745928 0.53745928 0.53745928 0.53745928\n",
            "        nan        nan        nan        nan        nan 0.53745928\n",
            " 0.53745928        nan 0.53745928 0.53745928        nan        nan\n",
            " 0.53745928        nan 0.53745928 0.53745928 0.53745928 0.53745928\n",
            " 0.53745928 0.53745928        nan        nan        nan        nan\n",
            "        nan 0.53745928 0.53745928        nan 0.53745928 0.53745928\n",
            "        nan        nan 0.53745928        nan 0.53745928 0.53745928\n",
            " 0.53745928 0.53745928 0.53745928 0.53745928        nan        nan\n",
            "        nan        nan        nan 0.53745928 0.53745928        nan\n",
            " 0.53745928 0.53745928]\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ERyyz3mvoca",
        "outputId": "1ca1106a-f8a6-43e7-86d2-6d077ceeef9a"
      },
      "source": [
        "print(\"The mean accuracy of the model is:\", best_model.score(X_test,y_test))\n",
        "print(\"The model best paramters are:\", best_model.best_params_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The mean accuracy of the model is: 0.5531914893617021\n",
            "The model best paramters are: {'C': 1.0, 'penalty': 'l1', 'solver': 'saga'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y4x0UihXvocb"
      },
      "source": [
        "### Model Creation and Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PzEc28m_nEw1",
        "outputId": "eea127ac-77ff-4085-d81c-6a6ade968710"
      },
      "source": [
        "from sklearn.linear_model import LogisticRegression\n",
        "classifier = LogisticRegression(C=1, penalty='l1', solver='liblinear')\n",
        "classifier.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "LogisticRegression(C=1, penalty='l1', solver='liblinear')"
            ]
          },
          "execution_count": 39,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W6yWb-lxqnda"
      },
      "source": [
        "y_pred_train = classifier.predict(X_train)\n",
        "y_pred_test = classifier.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HxyJlffsvocd"
      },
      "source": [
        "### Confusion Matrix / Accuracy / Precision"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MwVhbWwWqxwb",
        "outputId": "f158f99e-a397-486c-b7c0-0bde12344e2a"
      },
      "source": [
        "from sklearn import metrics\n",
        "from sklearn.metrics import confusion_matrix\n",
        "cm = confusion_matrix(y_test, y_pred_test)\n",
        "  \n",
        "print (\"Confusion Matrix : \\n\", cm)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Confusion Matrix : \n",
            " [[362   0]\n",
            " [293   3]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0lnYWfCVvoce",
        "outputId": "22f1abca-1be3-4eb1-d8bd-2700fc98e6dc"
      },
      "source": [
        "print(metrics.classification_report(y_test, y_pred_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.55      1.00      0.71       362\n",
            "           1       1.00      0.01      0.02       296\n",
            "\n",
            "    accuracy                           0.55       658\n",
            "   macro avg       0.78      0.51      0.37       658\n",
            "weighted avg       0.75      0.55      0.40       658\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ozqbh1xBq8Fc",
        "outputId": "9e31aa5e-a8f3-4510-fd90-0fe3dc817361"
      },
      "source": [
        "from sklearn.metrics import accuracy_score, precision_score\n",
        "print(\"Train Accuracy:\", accuracy_score(y_train,y_pred_train))\n",
        "print (\"Test Accuracy : \", accuracy_score(y_test, y_pred_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Accuracy: 0.5517915309446254\n",
            "Test Accuracy :  0.5547112462006079\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "BG3SOPy5vocf",
        "outputId": "102a377d-d48a-4eb3-f0c5-8702ad09173c"
      },
      "source": [
        "print(\"Train Precision Score:\", precision_score(y_train, y_pred_train))\n",
        "print(\"Test Precision SCore:\", precision_score(y_test, y_pred_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Precision Score: 0.6\n",
            "Test Precision SCore: 1.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bLLp5rKnvocg"
      },
      "source": [
        "## Implementing without PCA"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rTXMv30_voch"
      },
      "source": [
        "### Train-Test Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pyYS0Luuvoci",
        "outputId": "3be119c5-66fd-48a6-b67a-92f264afbc94"
      },
      "source": [
        "xtrain, xtest, ytrain, ytest = train_test_split(scaled_data, y, stratify = y, test_size = 0.3)\n",
        "print(xtrain.shape, xtest.shape, ytrain.shape, ytest.shape)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1535, 5) (658, 5) (1535,) (658,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "51rHj9g-voci",
        "outputId": "6148772f-a834-477c-97f8-9e07c9f44e99"
      },
      "source": [
        "xtrain"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[-0.72662545, -0.72642028, -0.72664917, -0.72654685, -0.37196296],\n",
              "       [-0.44906354, -0.46275862, -0.45370872, -0.45600802, -0.31357225],\n",
              "       [-0.52046703, -0.51528581, -0.52047094, -0.51414529,  0.02844589],\n",
              "       ...,\n",
              "       [ 0.14000129,  0.14029234,  0.12678435,  0.15406834, -0.32229889],\n",
              "       [-0.1098622 , -0.10856291, -0.12651975, -0.09325894, -0.11010207],\n",
              "       [-0.22762923, -0.23282656, -0.23473948, -0.22185467,  0.19642413]])"
            ]
          },
          "execution_count": 51,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oRtjd-wTvocj",
        "outputId": "4b5d7d2c-134e-4b96-8dca-1ba328d3dbed"
      },
      "source": [
        "ytrain"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Date\n",
              "2016-10-04    1\n",
              "2017-08-29    0\n",
              "2019-01-15    1\n",
              "2015-11-01    0\n",
              "2021-03-23    0\n",
              "             ..\n",
              "2018-08-18    1\n",
              "2015-11-18    0\n",
              "2020-10-26    0\n",
              "2020-06-13    0\n",
              "2019-06-05    0\n",
              "Name: Change %, Length: 1535, dtype: int64"
            ]
          },
          "execution_count": 52,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6dSdU1Ovvock",
        "outputId": "86e5d68f-1c2d-4358-f65d-73d2c1dee27a"
      },
      "source": [
        "xtest"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[-0.73892272, -0.73771   , -0.73645719, -0.74120221, -0.22835462],\n",
              "       [-0.6850291 , -0.68506419, -0.68591542, -0.6840333 , -0.36039419],\n",
              "       [-0.65164483, -0.6421521 , -0.64481641, -0.65177416, -0.30699453],\n",
              "       ...,\n",
              "       [-0.36819527, -0.37295024, -0.37609004, -0.36741825, -0.3439499 ],\n",
              "       [-0.1747308 , -0.14765826, -0.163195  , -0.16493869, -0.24034701],\n",
              "       [-0.3309903 , -0.32806349, -0.33574185, -0.32308361,  2.15052962]])"
            ]
          },
          "execution_count": 53,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2JAIVnGEvock",
        "outputId": "0ddf2799-8160-46ef-f901-45054278d99b"
      },
      "source": [
        "ytrain"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Date\n",
              "2016-10-04    1\n",
              "2017-08-29    0\n",
              "2019-01-15    1\n",
              "2015-11-01    0\n",
              "2021-03-23    0\n",
              "             ..\n",
              "2018-08-18    1\n",
              "2015-11-18    0\n",
              "2020-10-26    0\n",
              "2020-06-13    0\n",
              "2019-06-05    0\n",
              "Name: Change %, Length: 1535, dtype: int64"
            ]
          },
          "execution_count": 54,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CgCtchHDvocl"
      },
      "source": [
        "### Performing Hyperparameter Tuning Using Grid SearchCV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kcX3ewVuvocl",
        "outputId": "54607e69-f7aa-4a48-e2b4-f428c4c6ba37"
      },
      "source": [
        "# define models and parameters\n",
        "model = LogisticRegression()\n",
        "solvers = ['newton-cg', 'lbfgs', 'liblinear', 'sag', 'saga']\n",
        "penalty = ['l1','l2','elasticnet', 'none']\n",
        "c = np.logspace(0, 4, 10)\n",
        "\n",
        "# define grid search\n",
        "grid = dict(solver=solvers, penalty=penalty, C=c)\n",
        "grid_search = GridSearchCV(estimator=model, param_grid=grid, n_jobs=-1)\n",
        "grid_result = grid_search.fit(xtrain, ytrain)\n",
        "\n",
        "# summarize results\n",
        "means = grid_result.cv_results_['mean_test_score']\n",
        "stds = grid_result.cv_results_['std_test_score']\n",
        "params = grid_result.cv_results_['params']\n",
        "\n",
        "for mean, stdev, param in zip(means, stds, params):\n",
        "    print(\"accuracy: %f (%f) using: %r\" % (mean, stdev, param))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.844951 (0.013758) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.637785 (0.016787) using: {'C': 1.0, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.637785 (0.015608) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.637785 (0.015608) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.637785 (0.015608) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.635179 (0.016609) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.622150 (0.013351) using: {'C': 1.0, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 1.0, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 1.0, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1.0, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 1.0, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 1.0, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.888599 (0.015878) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.640391 (0.017187) using: {'C': 2.7825594022071245, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.692508 (0.017063) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.692508 (0.017063) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.692508 (0.017063) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.658632 (0.011359) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.637134 (0.016300) using: {'C': 2.7825594022071245, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 2.7825594022071245, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.901629 (0.017650) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.015334) using: {'C': 7.742636826811269, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.746580 (0.010784) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.746580 (0.010784) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.746580 (0.010784) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.670358 (0.013255) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.639088 (0.015878) using: {'C': 7.742636826811269, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 7.742636826811269, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.912052 (0.013509) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.641694 (0.015554) using: {'C': 21.544346900318832, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.793485 (0.010823) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.793485 (0.010823) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.793485 (0.010823) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.680130 (0.016143) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.641042 (0.015195) using: {'C': 21.544346900318832, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 21.544346900318832, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.905537 (0.024549) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 59.94842503189409, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.838436 (0.016558) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.838436 (0.016558) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.838436 (0.016558) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.682736 (0.014362) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.641694 (0.015554) using: {'C': 59.94842503189409, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 59.94842503189409, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.916612 (0.019060) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 166.81005372000593, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.867752 (0.013127) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.867752 (0.013127) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.867752 (0.013127) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 166.81005372000593, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 166.81005372000593, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.914007 (0.010625) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 464.15888336127773, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.888599 (0.012931) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.888599 (0.012931) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.888599 (0.012931) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 464.15888336127773, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 464.15888336127773, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.902280 (0.010705) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 1291.5496650148827, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.908143 (0.015195) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.908143 (0.015195) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.908143 (0.015195) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 1291.5496650148827, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 1291.5496650148827, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.907492 (0.018610) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 3593.813663804626, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.926384 (0.017553) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.926384 (0.017553) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.926384 (0.017553) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 3593.813663804626, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 3593.813663804626, 'penalty': 'none', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'lbfgs'}\n",
            "accuracy: 0.910098 (0.018723) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 10000.0, 'penalty': 'l1', 'solver': 'saga'}\n",
            "accuracy: 0.940065 (0.018380) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'newton-cg'}\n",
            "accuracy: 0.940065 (0.018380) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "accuracy: 0.940065 (0.018380) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 10000.0, 'penalty': 'l2', 'solver': 'saga'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'newton-cg'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'liblinear'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'sag'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'elasticnet', 'solver': 'saga'}\n",
            "accuracy: 0.993485 (0.005451) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'newton-cg'}\n",
            "accuracy: 0.994137 (0.006956) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'lbfgs'}\n",
            "accuracy: nan (nan) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'liblinear'}\n",
            "accuracy: 0.684039 (0.014998) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'sag'}\n",
            "accuracy: 0.642345 (0.016011) using: {'C': 10000.0, 'penalty': 'none', 'solver': 'saga'}\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "C:\\Users\\harsh\\anaconda3\\lib\\site-packages\\sklearn\\model_selection\\_search.py:918: UserWarning: One or more of the test scores are non-finite: [       nan        nan 0.84495114        nan 0.63778502 0.63778502\n",
            " 0.63778502 0.63778502 0.63517915 0.62214984        nan        nan\n",
            "        nan        nan        nan 0.99348534 0.99413681        nan\n",
            " 0.68403909 0.64234528        nan        nan 0.88859935        nan\n",
            " 0.64039088 0.69250814 0.69250814 0.69250814 0.65863192 0.63713355\n",
            "        nan        nan        nan        nan        nan 0.99348534\n",
            " 0.99413681        nan 0.68403909 0.64234528        nan        nan\n",
            " 0.90162866        nan 0.64234528 0.7465798  0.7465798  0.7465798\n",
            " 0.67035831 0.63908795        nan        nan        nan        nan\n",
            "        nan 0.99348534 0.99413681        nan 0.68403909 0.64234528\n",
            "        nan        nan 0.91205212        nan 0.64169381 0.79348534\n",
            " 0.79348534 0.79348534 0.68013029 0.64104235        nan        nan\n",
            "        nan        nan        nan 0.99348534 0.99413681        nan\n",
            " 0.68403909 0.64234528        nan        nan 0.90553746        nan\n",
            " 0.64234528 0.83843648 0.83843648 0.83843648 0.68273616 0.64169381\n",
            "        nan        nan        nan        nan        nan 0.99348534\n",
            " 0.99413681        nan 0.68403909 0.64234528        nan        nan\n",
            " 0.91661238        nan 0.64234528 0.86775244 0.86775244 0.86775244\n",
            " 0.68403909 0.64234528        nan        nan        nan        nan\n",
            "        nan 0.99348534 0.99413681        nan 0.68403909 0.64234528\n",
            "        nan        nan 0.91400651        nan 0.64234528 0.88859935\n",
            " 0.88859935 0.88859935 0.68403909 0.64234528        nan        nan\n",
            "        nan        nan        nan 0.99348534 0.99413681        nan\n",
            " 0.68403909 0.64234528        nan        nan 0.90228013        nan\n",
            " 0.64234528 0.90814332 0.90814332 0.90814332 0.68403909 0.64234528\n",
            "        nan        nan        nan        nan        nan 0.99348534\n",
            " 0.99413681        nan 0.68403909 0.64234528        nan        nan\n",
            " 0.90749186        nan 0.64234528 0.92638436 0.92638436 0.92638436\n",
            " 0.68403909 0.64234528        nan        nan        nan        nan\n",
            "        nan 0.99348534 0.99413681        nan 0.68403909 0.64234528\n",
            "        nan        nan 0.91009772        nan 0.64234528 0.94006515\n",
            " 0.94006515 0.94006515 0.68403909 0.64234528        nan        nan\n",
            "        nan        nan        nan 0.99348534 0.99413681        nan\n",
            " 0.68403909 0.64234528]\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e18tszCJvocm",
        "outputId": "59caf735-9fd8-4416-a2f1-ac0d33dcc82d"
      },
      "source": [
        "print(\"The mean accuracy of the model is:\", grid_result.score(xtest,ytest))\n",
        "print(\"The model best paramters are:\", grid_result.best_params_)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "The mean accuracy of the model is: 0.9954407294832827\n",
            "The model best paramters are: {'C': 1.0, 'penalty': 'none', 'solver': 'lbfgs'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RTQpbi3bvocn"
      },
      "source": [
        "### Model Creation and Prediction"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i7sQ2pTqvocn",
        "outputId": "0eb6675d-3ff6-407c-934d-aaaf71d200aa"
      },
      "source": [
        "classifier = LogisticRegression(C=1, penalty= 'none', solver= 'lbfgs')\n",
        "classifier.fit(xtrain, ytrain)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "data": {
            "text/plain": [
              "LogisticRegression(C=1, penalty='none')"
            ]
          },
          "execution_count": 57,
          "metadata": {},
          "output_type": "execute_result"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-mgZ6WkTvoco"
      },
      "source": [
        "ypred_train = classifier.predict(xtrain)\n",
        "ypred_test = classifier.predict(xtest)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yG92Miuuvoco"
      },
      "source": [
        "### Confusion Matrix / Accuracy / Precision"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VpXlmX8rvocp",
        "outputId": "207d5608-20d3-4c7a-c41b-bc6f0149ad09"
      },
      "source": [
        "cm = confusion_matrix(ytest, ypred_test)\n",
        "print (\"Confusion Matrix : \\n\", cm)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Confusion Matrix : \n",
            " [[359   3]\n",
            " [  0 296]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zzbCcyg3vocq",
        "outputId": "6bad0a33-9719-414a-9d8b-7baf9373ec99"
      },
      "source": [
        "print(metrics.classification_report(ytest, ypred_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       1.00      0.99      1.00       362\n",
            "           1       0.99      1.00      0.99       296\n",
            "\n",
            "    accuracy                           1.00       658\n",
            "   macro avg       0.99      1.00      1.00       658\n",
            "weighted avg       1.00      1.00      1.00       658\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "94jJuIpHvocq",
        "outputId": "1698d237-9ad6-4108-8618-79e686098f36"
      },
      "source": [
        "print (\"Train Accuracy : \", accuracy_score(ytrain, ypred_train))\n",
        "print (\"Test Accuracy : \", accuracy_score(ytest, ypred_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Accuracy :  0.9980456026058632\n",
            "Test Accuracy :  0.9954407294832827\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vQO3gh9mvocr",
        "outputId": "66a40465-e224-4964-a50c-8bff576995fb"
      },
      "source": [
        "print(\"Train Precision Score:\", precision_score(ytrain, ypred_train))\n",
        "print(\"Test Precision Score:\", precision_score(ytest, ypred_test))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train Precision Score: 0.9971014492753624\n",
            "Test Precision Score: 0.9899665551839465\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1aV-XltJvocr"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}