Project 2 - Classification - MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5472f5b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2f0f5ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_openml\n",
    "\n",
    "mnist = fetch_openml('mnist_784', as_frame = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a323b7ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['5', '0', '4', ..., '4', '5', '6'], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "06cd73ed",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist.data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "358fe336",
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mnist.data, mnist.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "45df6fb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ed8febd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "22270a47",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAIqElEQVR4nO3cPajO/x/H8S+J3HRKncJwGQjF7FhZzuKmM5lQBqIsZiGlQ3JzNoPF7JQjRFHMWM5wKEblMFCYTrn5/ha9pn/9vb+c6xw8Hvur72e46nl9ls+Ctm3bBgCaplk41wcAYP4QBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACAWzfUB4E82MzNT3hw+fLjTtyYmJsqbffv2lTcHDx4sb4aGhsob5ic3BQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYBY0LZtO9eHgD/V2bNny5uTJ0/Owkl+n4GBgfJmfHy8vBkeHi5vmH1uCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgDhQTz4BUuXLi1vZmZmOn1r165d5c2OHTvKm7GxsfLmzZs35c2rV6/Km6ZpmvXr13fa8XPcFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQDCg3jww71798qbLo/U9Xq98qZpmubhw4flzcaNG8ubFy9elDdHjx4tb0ZGRsqbpmma48ePd9rxc9wUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAGLRXB8A5ovp6enypst7ksPDw+VN03R73K6LzZs3lzddHuubnJwsb5h9bgoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhFdS4Yf79+/35Tv79+/vy3f6afHixeXN0NDQLJyEX+WmAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABAexIMfXr9+Xd4MDg6WN71er7zpampqqrw5c+ZMefP27dvy5s6dO+VN0zTNypUrO+34OW4KAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCAOFBPPgFX758KW+6PLzXNE1z48aN8mZsbKy8effuXXnTxYcPHzrtPIg3u9wUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAMKDePyVvn//3pfNp0+fypuRkZHypmma5uPHj512VQMDA+XNiRMnypu1a9eWN8w+NwUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAYkHbtu1cHwJ+t2fPnpU3Q0NDs3CS32fJkiXlzZ49e8qb69evlzfLli0rb5if3BQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAYtFcHwD+n6mpqfLm3Llzs3CS32PdunWddqdPny5vDhw40Olb/LvcFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBiQdu27Vwfgn/D06dPO+127txZ3rx//77Tt/rh8ePHnXbbt2//vQeB/8FNAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACAWzfUB+DONj4+XN8eOHev0rfn8uF0XCxf6L8b85dcJQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEB7E+8t8/fq1vDl16lR5c/78+fKmbdvypmmaZmBgoLw5cuRIefPgwYPyZnJysryB+cxNAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYDwSuo8NT093Wl35cqV8ubixYudvlU1ODjYaTc6OlreHDp0qLx59OhReQN/GzcFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgPAgXh98+/atvLl06VKnb12+fLnTrmr16tXlza1btzp9a9u2beVNlwcF3759W97A38ZNAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACA8iNcHV69eLW/69bBd0zTN4OBgeTM5OVnerFq1qrzp6smTJ+XNmzdvypter1febN68ubyBfnFTACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAIgFbdu2c32Iv92aNWvKm3fv3nX61vLly8ubu3fvljdbt24tb6anp8ubpmma0dHR8ubmzZvlzefPn8ub27dvlze7d+8ub6Bf3BQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAwoN4fdDlAbQuj9R1tWnTpvLm5cuXs3CS32fFihXlzd69e8uba9eulTcLF/ovxvzl1wlAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBALJrrA/wLLly4UN70er1O33r+/Hl58/r1607f6peJiYnyZsuWLeXNhg0byhv427gpABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAMSCtm3buT4EAPODmwIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEP8BKMX+AjDzqZoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def digit_display(data_set, data_index):\n",
    "    digit = data_set.data[data_index]\n",
    "    image = digit.reshape(28, 28)\n",
    "    plt.imshow(image, cmap = \"binary\")\n",
    "    plt.axis(\"off\")\n",
    "    print(data_set.target[data_index])\n",
    "# some_digit = X[31313]\n",
    "# some_digit_image = some_digit.reshape(28, 28)\n",
    "\n",
    "# plt.imshow(some_digit_image, cmap = matplotlib.cm.binary, interpolation = 'nearest')\n",
    "# plt.axis('off')\n",
    "# plt.show()\n",
    "digit_display(mnist, 31313)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "5bbe5408",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAIqElEQVR4nO3cPajO/x/H8S+J3HRKncJwGQjF7FhZzuKmM5lQBqIsZiGlQ3JzNoPF7JQjRFHMWM5wKEblMFCYTrn5/ha9pn/9vb+c6xw8Hvur72e46nl9ls+Ctm3bBgCaplk41wcAYP4QBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACAWzfUB4E82MzNT3hw+fLjTtyYmJsqbffv2lTcHDx4sb4aGhsob5ic3BQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYBY0LZtO9eHgD/V2bNny5uTJ0/Owkl+n4GBgfJmfHy8vBkeHi5vmH1uCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgDhQTz4BUuXLi1vZmZmOn1r165d5c2OHTvKm7GxsfLmzZs35c2rV6/Km6ZpmvXr13fa8XPcFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQDCg3jww71798qbLo/U9Xq98qZpmubhw4flzcaNG8ubFy9elDdHjx4tb0ZGRsqbpmma48ePd9rxc9wUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAGLRXB8A5ovp6enypst7ksPDw+VN03R73K6LzZs3lzddHuubnJwsb5h9bgoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhFdS4Yf79+/35Tv79+/vy3f6afHixeXN0NDQLJyEX+WmAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABAexIMfXr9+Xd4MDg6WN71er7zpampqqrw5c+ZMefP27dvy5s6dO+VN0zTNypUrO+34OW4KAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCAOFBPPgFX758KW+6PLzXNE1z48aN8mZsbKy8effuXXnTxYcPHzrtPIg3u9wUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAMKDePyVvn//3pfNp0+fypuRkZHypmma5uPHj512VQMDA+XNiRMnypu1a9eWN8w+NwUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAYkHbtu1cHwJ+t2fPnpU3Q0NDs3CS32fJkiXlzZ49e8qb69evlzfLli0rb5if3BQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAYtFcHwD+n6mpqfLm3Llzs3CS32PdunWddqdPny5vDhw40Olb/LvcFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBiQdu27Vwfgn/D06dPO+127txZ3rx//77Tt/rh8ePHnXbbt2//vQeB/8FNAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACAWzfUB+DONj4+XN8eOHev0rfn8uF0XCxf6L8b85dcJQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEB7E+8t8/fq1vDl16lR5c/78+fKmbdvypmmaZmBgoLw5cuRIefPgwYPyZnJysryB+cxNAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACBEAYDwSuo8NT093Wl35cqV8ubixYudvlU1ODjYaTc6OlreHDp0qLx59OhReQN/GzcFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgPAgXh98+/atvLl06VKnb12+fLnTrmr16tXlza1btzp9a9u2beVNlwcF3759W97A38ZNAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAAhRACA8iNcHV69eLW/69bBd0zTN4OBgeTM5OVnerFq1qrzp6smTJ+XNmzdvypter1febN68ubyBfnFTACBEAYAQBQBCFAAIUQAgRAGAEAUAQhQACFEAIEQBgBAFAEIUAIgFbdu2c32Iv92aNWvKm3fv3nX61vLly8ubu3fvljdbt24tb6anp8ubpmma0dHR8ubmzZvlzefPn8ub27dvlze7d+8ub6Bf3BQACFEAIEQBgBAFAEIUAAhRACBEAYAQBQBCFAAIUQAgRAGAEAUAwoN4fdDlAbQuj9R1tWnTpvLm5cuXs3CS32fFihXlzd69e8uba9eulTcLF/ovxvzl1wlAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBALJrrA/wLLly4UN70er1O33r+/Hl58/r1607f6peJiYnyZsuWLeXNhg0byhv427gpABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAMSCtm3buT4EAPODmwIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEKIAQIgCACEKAIQoABCiAECIAgAhCgCEKAAQogBAiAIAIQoAhCgAEP8BKMX+AjDzqZoAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_digit(image_data):\n",
    "    image = image_data.reshape(28, 28)\n",
    "    plt.imshow(image, cmap = \"binary\")\n",
    "    plt.axis(\"off\")\n",
    "    \n",
    "some_digit = X[31313]\n",
    "plot_digit(some_digit)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a759d236",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = X[:60000], X[60000:], y[:60000], y[60000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "59593b1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffle_index = np.random.permutation(60000)\n",
    "X_train, y_train= X[shuffle_index], y[shuffle_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a21673c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-1 {color: black;background-color: white;}#sk-container-id-1 pre{padding: 0;}#sk-container-id-1 div.sk-toggleable {background-color: white;}#sk-container-id-1 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-1 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-1 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-1 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-1 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-1 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-1 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-1 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-1 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-1 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-1 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-1 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-1 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-1 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-1 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-1 div.sk-item {position: relative;z-index: 1;}#sk-container-id-1 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-1 div.sk-item::before, #sk-container-id-1 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-1 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-1 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-1 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-1 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-1 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-1 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-1 div.sk-label-container {text-align: center;}#sk-container-id-1 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-1 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-1\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>SGDClassifier(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-1\" type=\"checkbox\" checked><label for=\"sk-estimator-id-1\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SGDClassifier</label><div class=\"sk-toggleable__content\"><pre>SGDClassifier(random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "SGDClassifier(random_state=42)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Starting with binary classifier(3 or not)\n",
    "\n",
    "y_train_3 = (y_train == \"3\")\n",
    "y_test_3 = (y_test == \"3\")\n",
    "\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "\n",
    "sgd_classifier = SGDClassifier(random_state = 42)\n",
    "sgd_classifier.fit(X_train, y_train_3)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "02027f65",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[array([  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          8., 162., 243., 151.,  54.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0., 124., 254., 254., 254., 241.,\n",
       "         70.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0., 246., 254., 179., 114., 226., 252.,  35.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,  74., 253., 225.,  38.,   0.,\n",
       "         42., 193., 113.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0., 100., 254., 155.,   0.,   0.,   0.,   6.,  54.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0., 172., 254.,  62.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0., 203., 235.,  33.,   0.,  44., 171., 196.,  17.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 203., 240.,  63.,\n",
       "        119., 230., 254., 254.,  36.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   2., 204., 254., 238., 254., 254., 241., 118.,   6.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  53., 254., 254.,\n",
       "        254., 248., 170.,  10.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,  43., 174., 254., 254., 220.,  94.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,  54., 235., 254., 254.,\n",
       "        254.,  64.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,  85., 233., 254., 254., 254., 254.,  62.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   4., 134., 253., 254., 241., 137.,\n",
       "        206., 254.,  62.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  26.,\n",
       "        139., 254., 254., 235., 101.,   0., 203., 254.,  62.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   3., 140., 254., 255., 230.,  55.,   0.,\n",
       "         26., 228., 254.,  62.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 108.,\n",
       "        254., 254., 235.,  23.,   0.,   0.,  81., 254., 226.,  20.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0., 231., 254., 244.,  64.,  11.,  37.,\n",
       "        159., 241., 254.,  77.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "        179., 254., 254., 216., 254., 254., 254., 243., 156.,   2.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,  41., 150., 193., 223., 254.,\n",
       "        254.,  80.,  37.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "          0.,   0.,   0.])]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[X[31313]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ef5277fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_classifier.predict([X[31313]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f372e481",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.9439, 0.9329, 0.9556])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Measuring Accuracy\n",
    "\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "score = cross_val_score(sgd_classifier, X_train, y_train_3, cv = 3, scoring = \"accuracy\")\n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2f2a807a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.8989 , 0.89515, 0.8994 ])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Base Estimato\n",
    "\n",
    "from sklearn.base import BaseEstimator\n",
    "\n",
    "class Never3Classifier(BaseEstimator):\n",
    "    \n",
    "    def fit(self, X, y = None):\n",
    "        pass\n",
    "    \n",
    "    def predict(self, X):\n",
    "        return np.zeros((len(X), 1), dtype = bool)\n",
    "    \n",
    "never_3_classifier = Never3Classifier()\n",
    "\n",
    "cross_val_score(never_3_classifier, X_train, y_train_3, cv = 3, scoring = \"accuracy\")\n",
    "\n",
    "# Gives about 90% accuracy because 10% of the dataset is 3\n",
    "# So accuracy is not preferred\n",
    "# Solution: Confusion Matrix\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6821d8d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[51264,  2605],\n",
       "       [  747,  5384]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_predict\n",
    "\n",
    "train_predictions = cross_val_predict(sgd_classifier, X_train, y_train_3, cv = 3)\n",
    "\n",
    "# train_predictions\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_train_3, train_predictions)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "8f97dc11",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[53869,     0],\n",
       "       [ 6131,     0]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Confusion matrix of Never3Classifier\n",
    "train_nostradamus = cross_val_predict(never_3_classifier, X_train, y_train_3, cv = 3)\n",
    "\n",
    "confusion_matrix(y_train_3, train_nostradamus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4105bd3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precision : 0.673926649142571\n",
      "Recall : 0.8781601696297504\n",
      "F1 : 0.7626062322946175\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "\n",
    "def display_scores(prediction):\n",
    "    print(\"Precision :\", precision_score(y_train_3, prediction))\n",
    "    print(\"Recall :\", recall_score(y_train_3, prediction))\n",
    "    print(\"F1 :\", f1_score(y_train_3, prediction))\n",
    "\n",
    "display_scores(train_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ccb44f6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_scores = cross_val_predict(sgd_classifier, X_train, y_train_3, cv = 3, method = \"decision_function\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d99f37d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAG2CAYAAABViX0rAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABZwElEQVR4nO3deVxU5f4H8M8MywCy74sgILibC264ZmJumd66N7NS9Hbz3tKytM3KpbpJt8y6mW3eXOpWmlbWT02va5riviSCqAiCyCKy7zDz/P44zcDIIgMznBn4vHvNa4Yzzznn+wAOn855znMUQggBIiIiIjOilLsAIiIiotsxoBAREZHZYUAhIiIis8OAQkRERGaHAYWIiIjMDgMKERERmR0GFCIiIjI7DChERERkdhhQiIiIyOwwoBAREZHZMTigHDx4EJMnT4a/vz8UCgW2bt16x3UOHDiA/v37Q6VSISwsDOvXr29GqURERNReGBxQSkpK0KdPH6xevbpJ7ZOTkzFp0iSMHj0aZ8+exbPPPou//e1v2LVrl8HFEhERUfugaMnNAhUKBX788UdMnTq1wTYvvfQStm/fjri4ON2yhx9+GPn5+di5c2dzd01ERERtmLWpdxAbG4uoqCi9ZePGjcOzzz7b4DoVFRWoqKjQfa3RaJCbmwsPDw8oFApTlUpERERGJIRAUVER/P39oVQadtLG5AElMzMTPj4+est8fHxQWFiIsrIy2Nvb11knJiYGr7/+uqlLIyIiolaQlpaGjh07GrSOyQNKcyxatAgLFizQfV1QUICgoCCkpaXB2dnZaPtJyUtBubrcoHWae0ZMwPD1mrOv5uynre7L3H9Wzd1XM7/tBu9LCIGiyiJohKbe9Wv3ufZ7DS2XYx2N0KC4shhV6ipooIFGSI9qTTUKywtRqa6EWqilrysLUVldiWpNNao11SiuKkZxRTHyyvNQVl0GtUYNtVBDo9FALdQoqiiq8z1rKoVCAReVCzwcPBDkEoQIvwgMCxyGEZ1GwMbKptnbpdaVkQF89x2wZIn+8rFjgS1bgPx84MYNoL4DBz4+gJub9Lq4GLh+veH9eHsD7u7S65ISIC2t4bZeXoCHh/S6rAy4dq3hth4eUntTKiwsRGBgIJycnAxe1+QBxdfXF1lZWXrLsrKy4OzsXO/REwBQqVRQqVR1ljs7Oxs1oNzlfJfRtkVE7YtGaFCprkRpVSlyy3JRqa5EblkuyqrKUFZdhtyyXJRUluBawTVcyb2C9KJ0XC+8jhtFNyAgkI985JfmI6k0Cfsz9mPF6RUAgE4unRDqFor7utyHIJcgDA0cCn8nf5l7S1VVwEsvSa+jo4E+fYDFi4EPP6xp4+EB2NgAgwcDzs7SIyjoztt2dgb8m/gjdnYG/Pya3va2Exiyac7wDJMHlMjISOzYsUNv2e7duxEZGWnqXRMRmYxSoYSdtR3srO3gbu/e5PWqNdXILM7ErdJbSC9KR1x2HE5nnMaOyztQVFmEawXXcK3gGvan7Net493BG76OvnC1c0Vv7964J+QeTOk6BVZKK1N0rc0TArh1C7C2BlxdpWVXrkjLR48G0tOB4cOBQ4eAwkJgxAjg999r1n//fWDNGmDAAMDJCSgqAh58ENi0CbCyAtRqWbrV5hh8FU9xcTGuXLkCAOjXrx9WrlyJ0aNHw93dHUFBQVi0aBHS09Px5ZdfApAuM+7Vqxfmzp2Lv/71r9i3bx+eeeYZbN++HePGjWvSPgsLC+Hi4oKCggKjHkEhIjIXGqFBSn4KruZdxZ6re5CQk4AzGWeQVlj/8XxnlTMe7P4gHrvrMdwTck8rV2tZkpOB0NC6y+PigO7dgS+/BGbP1n/vrruAX38F7r0XOHFC/z1ra+DCBaBLF9PV3Fa05O+3wQHlwIEDGD16dJ3l0dHRWL9+PWbNmoWUlBQcOHBAb53nnnsO8fHx6NixIxYvXoxZs2Y1eZ8MKETUXmUWZyK1IBW5ZbnILM7E3uS9+D7+e5RVl+naONk6IdAlEAP9ByLULRRDOg7BiKARsLep/zR6W1ZZCdQeIVBSAvzyC/DnP9dtGxUF7NwJ/O1vwO3zh374IfD009J4jyFDpLEks2dLR04KC2vGj1DjWjWgyKEpHdRoNKisrGzlyqglbGxsYGXFQ9REhsopzcHupN34Lv47/HTxp3oHQNta2eKB7g/gzdFvIsw9TIYqW1d2dv3jLfr3B378EejUCbC1BaZPB1JTpSMkTz8NdO4sne756SfpaEpcHDBuXNPGjtCdtfuAUllZieTkZGg0Ghmqo5ZwdXWFr68v57chaqa8sjykF6Xj0q1LOJt5Fmcyz2DXlV2o0lQBAOyt7fFgjwcxNnQspvea3mavEqrvI8TfHzh/vuYKGGp97TqgCCGQmpqKqqqqZk0EQ/IQQqC0tBTZ2dlwdXWFX1OHpRPRHQkhsOPyDvzz0D9x9PpR3XI/Rz9seWgLhgYOlbE643r9demqmVGjAAcHadm//w0884y8dZGkXQeUqqoqXLlyBf7+/nBxcZGpQmquW7duITs7G126dOHpHiIjU2vU2HZpGzZe2IiNcRsBADZKG3xx/xd49K5HoVRY9v/QdegAlJYC48cDGzcC/BNgfloSUCz7txOA+o/ruWxtbWWuhJrD4Y//5amqqpK5EqK2x0pphSndpuDbB79F5sJMdPPshipNFWZunQmnGCe8svcVVKotc+zeoEFSOAGkga78E9D2WHxA0eIYBsvEnxtR6/Bx9MGxvx3DzD4zAQClVaWI+S0G/T7rh4PXDspcnWFGjtS/9PfUKaCBeT/JgrWZgEJERI1zVjljw9QNKHmlBO+OfRcqKxXib8Zj1PpRmLJxSoum728tBw5IE6hpVVRIV+pQ28OA0o4oFAps3brV6G2JyLI42Djg+aHP4/LTl3Fv53sBAD8n/gz/lf6ITYuVubrG+ftLY08AacZXntppuxhQZDJr1iwoFAooFArY2toiLCwMb7zxBqqrq022z4yMDEyYMMHobYnIMgW6BGLXY7vw8cSPAQDFlcUYunYo5vzfHLM7mqI9jRMUJI05KS5u+v1ryDIxoMho/PjxyMjIwOXLl7Fw4UIsW7YM7777bp12xpqAztfXt96bMLa0LRFZticHPoljfzuG+7veDwBYc3oNuq/ujos5F2WurMaAAUB5uRRUhg+vOYpCbRcDioxUKhV8fX3RqVMnPPnkk4iKisLPP/+MWbNmYerUqXjrrbfg7++Prl27AgDS0tLw0EMPwdXVFe7u7pgyZQpSUlL0trl27Vr07NkTKpUKfn5+mDdvnu692qdtKisrMW/ePPj5+cHOzg6dOnVCTExMvW0B4Pz587jnnntgb28PDw8PzJkzB8XFxbr3tTWvWLECfn5+8PDwwNy5c3l1DpGFGBQwCD89/BO+eeAb2FvbI70oHd1Xd8f6s+vlLg23btW8PnVKvjqodbXZgFJS0vCjvLzpbcvKmtbWGOzt7XVHS/bu3YvExETs3r0b27ZtQ1VVFcaNGwcnJyccOnQIhw8fhqOjI8aPH69b55NPPsHcuXMxZ84cnD9/Hj///DPCwuqf4vrDDz/Ezz//jO+++w6JiYn4+uuvERwcXG/bkpISjBs3Dm5ubjhx4gQ2b96MPXv26IUfANi/fz+SkpKwf/9+bNiwAevXr8f6229wQURmbXrv6TjxxAmorKQjqLN/mo3Vx1fLVo8QwAMP1Hx928cOtWXCAhQUFAgAoqCgoM57ZWVlIj4+XpSVlektl36t639MnKi/DQeHhtuOGqXf1tOz/naGio6OFlOmTBFCCKHRaMTu3buFSqUSzz//vIiOjhY+Pj6ioqJC1/6rr74SXbt2FRqNRresoqJC2Nvbi127dgkhhPD39xevvvpqg/sEIH788UchhBBPP/20uOeee/S211Dbzz//XLi5uYni4mLd+9u3bxdKpVJkZmbq+tOpUydRXV2ta/OXv/xFTJs2rdHvQ0M/PyKSV3lVuZjw3wkCyyCwDOJA8gFZ6li1quZzdvNmWUqgFmjs7/edtNkjKJZg27ZtcHR0hJ2dHSZMmIBp06Zh2bJlAIDevXvrTT537tw5XLlyBU5OTnB0dISjoyPc3d1RXl6OpKQkZGdn48aNGxgzZkyT9j1r1iycPXsWXbt2xTPPPIP//e9/DbZNSEhAnz590KHWSd9hw4ZBo9EgMTFRt6xnz556s8H6+fkhOzu7qd8OIjIjKmsVtj68FQP9BwIAJn87GVXq1j9l+/TTNa8ffLDVd08yspa7AFOpNTyijttnVG/sb+jtt/a5bchHi4wePRqffPIJbG1t4e/vD2vrmh9Hh9tGgBUXFyMiIgJff/11ne14eXkZfA+i/v37Izk5Gb/88gv27NmDhx56CFFRUdiyZUvzOgPp7sS1KRQK3sCRyILZWtli+yPb4b/SH0WVRZizbQ7WTVnXavuv/XF37Fj9NwSktqvNHkHp0KHhh51d09vePjthQ+2aV2MHhIWFISgoSC+c1Kd///64fPkyvL29ERYWpvdwcXGBk5MTgoODsXfv3ibv39nZGdOmTcOaNWuwadMmfP/998jNza3Trnv37jh37hxKag22OXz4MJRKpW4ALxG1TV4dvLBs1DIAwPqz67Hn6p5W2/ef/lTzetCgVtstmYk2G1DamkcffRSenp6YMmUKDh06hOTkZBw4cADPPPMMrl+/DgBYtmwZ3nvvPXz44Ye4fPkyTp8+jVWrVtW7vZUrV+Lbb7/FxYsXcenSJWzevBm+vr5wdXWtd992dnaIjo5GXFwc9u/fj6effhozZsyAj4+PKbtNRGbg1ZGvYkKYNC/SxK8n4ufEn02+T7VaukhBOwKF2h8GFAvh4OCAgwcPIigoCA888AC6d++Oxx9/HOXl5bo7REZHR+ODDz7Axx9/jJ49e+K+++7D5cuX692ek5MT3nnnHQwYMAADBw5ESkoKduzYUe+pIgcHB+zatQu5ubkYOHAg/vznP2PMmDH46KOPTNpnIjIf66asQ2e3zqjSVOGhzQ/hat5Vk+7vsceA+fOlqeypfVIIYf7ZtLHbNZeXlyM5ORkhISGwu/3cDZk9/vyILEdZVRkiPo9AQk4Cpnabih+n/WiS/Wg0NWMFX38dWLLEJLuhVtDY3+874REUIiJqEnsbe3z5py8BAFsvbsWvKb+aZD9Hj9a8fuEFk+yCLAADChERNdkA/wF4vN/jAIBndj4DtUZt9H388IP0/MgjdS9UoPaDAYWIiAzy5ug34WTrhN+zfjf6VT1CAN9/L72uPYMstT8MKEREZBA/Jz881PMhAMD8nfOhEcab7+jMGWm+KXt7YPx4o22WLBADChERGWzZ3ctgZ22HxFuJ2HB2g9G2qz16MmEC71jc3jGgEBGRwTo6d8SSkdLlNc/vfh5lVWV3WKNpSkulyTQ5rT0xoBARUbPMGzQPXg5eyC3LxftH3zfKNt9/H8jJ4fgTYkAhIqJmclI5YfmY5QCA5YeWI7es7q0ymqO+W5JQ+8OAQkREzfbXfn9FV4+uKKkqweenPm/RtgYPliZmM//pQ6k1MKC0YwqFAlu3bgUApKSkQKFQ4OzZs7LWRESWRalQYkHkAgDAvw7/C6VVpc3azvXrwPHjwLJlQFqaEQski8WAIpNZs2ZBoVBAoVDAxsYGISEhePHFF1FeXi53aUREBnm096PwcvBCfnk+1pxa06xt1L6kOCjISIWRRWNAkdH48eORkZGBq1ev4v3338dnn32GpUuXyl0WEZFBOth2wIvDXgQArD6xGpXqSoO3wVBCt2NAkZFKpYKvry8CAwMxdepUREVFYffu3QAAjUaDmJgYhISEwN7eHn369MGWLVv01r9w4QLuu+8+ODs7w8nJCSNGjEBSUhIA4MSJExg7diw8PT3h4uKCUaNG4fTp063eRyJqH/7a768AgMu5l7ElfssdWuvTaIATJ6TXv/1m7MrIUlnLXYCxCSGafQ60pRxsHKBQKJq1blxcHI4cOYJOnToBAGJiYvDf//4Xn376KcLDw3Hw4EE89thj8PLywqhRo5Ceno6RI0fi7rvvxr59++Ds7IzDhw+juroaAFBUVITo6GisWrUKQgi89957mDhxIi5fvgwnJyej9ZmICADc7d0xd+BcrD6xGssOLMP0XtOb/Hl44YJ0abGDAzBwoIkLJYvR5gJKaVUpHGMcZdl38aJidLBt+tSH27Ztg6OjI6qrq1FRUQGlUomPPvoIFRUVWL58Ofbs2YPIyEgAQGhoKH777Td89tlnGDVqFFavXg0XFxds3LgRNjY2AIAuXbrotn3PPffo7evzzz+Hq6srfv31V9x3331G6C0Rkb5XR7yKz099jsu5l3Hw2kGMCh7VpPW++kp6HjIEsLU1YYFkUXiKR0ajR4/G2bNncezYMURHR2P27Nl48MEHceXKFZSWlmLs2LFwdHTUPb788kvdKZyzZ89ixIgRunByu6ysLDzxxBMIDw+Hi4sLnJ2dUVxcjNTU1NbsIhG1I7Xv0bP+3Pomr5eeLj0PGmSCoshitbkjKA42DiheVCzbvg3RoUMHhIWFAQDWrl2LPn364IsvvkCvXr0AANu3b0dAQIDeOiqVCgBgf4d7kEdHR+PWrVv497//jU6dOkGlUiEyMhKVlYYPXiMiaqoZd83A1+e/xlfnvsKKsSvg4eBxx3U+/hjo3x/4619boUCyGG0uoCgUCoNOs5gLpVKJV155BQsWLMClS5egUqmQmpqKUaPqP0R61113YcOGDaiqqqr3KMrhw4fx8ccfY+LEiQCAtLQ05OTkmLQPRERjO49FN89uuJhzEZ+c/ASvjXztjuu4uAALF7ZCcWRReIrHjPzlL3+BlZUVPvvsMzz//PN47rnnsGHDBiQlJeH06dNYtWoVNmyQ7ho6b948FBYW4uGHH8bJkydx+fJlfPXVV0hMTAQAhIeH46uvvkJCQgKOHTuGRx999I5HXYiIWkqpUOLJAU8CAL489+Ud2z/0EGBvD1y5YurKyNIwoJgRa2trzJs3D++88w4WLVqExYsXIyYmBt27d8f48eOxfft2hISEAAA8PDywb98+FBcXY9SoUYiIiMCaNWt0R1O++OIL5OXloX///pgxYwaeeeYZeHt7y9k9ImonHu71MGyUNricexm/pvzaaNvNm4HycmD//lYqjiyGQgjzv+tBYWEhXFxcUFBQAGdnZ733ysvLkZycjJCQENjx7lIWhz8/orZpxo8z8N/f/4uhgUNx+K+H621TWQn8MawOKSnAH7MsUBvS2N/vO+ERFCIiMrp/jv4nrBRWOJJ2BGcyztTb5vr1mtecSZZux4BCRERG18m1EyaGS4P0d1/dXW+bixel5169gGbOcUltGAMKERGZxJCOQwAApzJO1fu+NqB0795aFZElYUAhIiKTuDv4bgDAlvgtKKooqvP+559Lz56erVgUWYw2E1AsYKwv1YM/N6K2K7JjJIJcgqARGmyM21jnfXd36dnNrZULI4tg8QHFysoKADhDqoUqLZVu7NjQlP1EZLkUCgX+EfEPAPVPfX/kiHT1zgsvtG5dZBksfiZZa2trODg44ObNm7CxsYFSafGZq10QQqC0tBTZ2dlwdXXVBU0ialui+0bjtf2v4UjaEcTfjEcPrx567/PSYmqIxQcUhUIBPz8/JCcn49q1a3KXQwZydXWFr6+v3GUQkYn4O/ljYvhEbLu0DVvit2DJqCUAgLIywM6OV+9Qwyw+oACAra0twsPDeZrHwtjY2PDICVE7MLXrVGy7tA07Lu/QBZQXXwQ2bADeegt4+mmZCySz1CYCCiDdbI8zkRIRmZ8xoWMAACdvnERxZTEcbR0RFwcUFQEGTi5K7QgHbBARkUkFuwbD38kfaqHW3ZsnLk56r1cvGQsjs8aAQkREJje161QAwOb4zcjOBnJypPEnnKSNGsKAQkREJjey00gAwJnMM9ixQ1qmVAIODjIWRWatzYxBISIi86Wd9j7+Zjy+PJYLwF03URtRfXgEhYiITK6TaycEuQShWlON5KrDAIDJk2UuiswaAwoREbWKiWHS3Y39Rv4ChQKYOVPmgsisMaAQEVGruK/LfQCA6w7boFYLjBolc0Fk1hhQiIioVdwTcg9UViqkFabhTOYZucshM8eAQkRErcLexh4jO0YBAL49t0XmasjcMaAQEVGr8SkaBwD4bM//ZK6EzB0DChERtZqTX08FABQ7nkVxZbG8xZBZY0AhIqJW42EdCBQEQijUOJ5+XO5yyIwxoBARUatJTweQOgwAcDj1sLzFkFljQCEiolZRVQWkpgJI+yOgpDGgUMOaFVBWr16N4OBg2NnZYfDgwTh+vPHDdB988AG6du0Ke3t7BAYG4rnnnkN5eXmzCiYiIsuUmgpoNIBtlhRQYq/HQiM0MldF5srggLJp0yYsWLAAS5cuxenTp9GnTx+MGzcO2dnZ9bb/5ptv8PLLL2Pp0qVISEjAF198gU2bNuGVV15pcfFERGQ5kpOl586OvWClsEJhRSFuFN2QtygyWwYHlJUrV+KJJ57A7Nmz0aNHD3z66adwcHDA2rVr621/5MgRDBs2DI888giCg4Nx7733Yvr06Xc86kJERG1Lly7ARx8BC561QVfPrgCAMxmcsI3qZ1BAqaysxKlTpxAVFVWzAaUSUVFRiI2NrXedoUOH4tSpU7pAcvXqVezYsQMTJ05scD8VFRUoLCzUexARkWULCgLmzgX+9jdggP8AAMCRtCMyV0XmyqCAkpOTA7VaDR8fH73lPj4+yMzMrHedRx55BG+88QaGDx8OGxsbdO7cGXfffXejp3hiYmLg4uKiewQGBhpSJhERmSkhpOeoEOl/dPel7JOxGjJnJr+K58CBA1i+fDk+/vhjnD59Gj/88AO2b9+ON998s8F1Fi1ahIKCAt0jLS3N1GUSEZGJDRgAKJVAZiYwpOMQAMDvWb+jUl0pc2VkjqwNaezp6QkrKytkZWXpLc/KyoKvr2+96yxevBgzZszA3/72NwBA7969UVJSgjlz5uDVV1+FUlk3I6lUKqhUKkNKIyIiM3fqlPScnAwMGRIGVztX5Jfn40L2BfTz6ydvcWR2DDqCYmtri4iICOzdu1e3TKPRYO/evYiMjKx3ndLS0johxMrKCgAgtMf6iIioTSsoqHnduzegUCgQ4RcBADhx44RMVZE5M/gUz4IFC7BmzRps2LABCQkJePLJJ1FSUoLZs2cDAGbOnIlFixbp2k+ePBmffPIJNm7ciOTkZOzevRuLFy/G5MmTdUGFiIjatqQk6dnLC3B0lF7385WOmnDCNqqPQad4AGDatGm4efMmlixZgszMTPTt2xc7d+7UDZxNTU3VO2Ly2muvQaFQ4LXXXkN6ejq8vLwwefJkvPXWW8brBRERmbVLl6Tnrl1rlkUGRgKxwOmM0/IURWZNISzgPEthYSFcXFxQUFAAZ2dnucshIiIDvfEGsHQp8Ne/Al98IS1LzktG6IehUCqUKHmlBHbWdvIWSUbXkr/fvBcPERGZnPYUT+fONcuCXYPh6eAJjdDwzsZUBwMKERGZXGKi9BweXrNMoVAgsqN0gcWx68dkqIrMGQMKERGZXEwM8J//AEOH6i/v49MHAJB4K1GGqsicGTxIloiIyFCjR0uP24V7SIdUUvJTWrcgMns8gkJERLIJcQ0BAFzNuypzJWRuGFCIiMik0tOBL78E6runbCfXTgCAtMI0VGuqW7kyMmcMKEREZFJHjwLR0cDChXXf6+jcEfbW9qjWVCM5L7n1iyOzxYBCREQmpb3fa8eOdd9TKpQIdQsFwHEopI8BhYiITEobUAID638/2DUYAHDp1qXWKYgsAgMKERGZ1J0CSm/v3gCAc1nnWqkisgQMKEREZFLXr0vPDQYUHymgXLh5oZUqIkvAgEJERCbV1CMo57POQyM0rVQVmTsGFCIiMpnqaiAjQ3pd3yBZAOju1R02ShsUVRbheuH11iuOzBoDChERmdS2bcDnnwM+PvW/b620RqCLdHgltSC1FSsjc8ap7omIyGSsrYHx4+/cLtA5EFfzrjKgkA6PoBARkezC3MMA8FJjqsEjKEREZDInTgAJCUC/fkDv3g236+HVAwBwPvt8K1VG5o5HUIiIyGQ2b5amuV+7tvF2fXz6AAB+z/q9FaoiS8CAQkREJqO9gsffv/F23b26AwCS85JRUV1h4qrIEjCgEBGRydy6JT17ejbezs/RD062TlALNZLykkxfGJk9BhQiIjKZ/Hzp2dW18XYKhYIDZUkPAwoREZlMZqb07OV157bagBJ/M96EFZGlYEAhIiKT0Ghq7sMTFHTn9torea7mXTVhVWQpGFCIiMgkcnKAqipAoQD8/O7cXnsEhWNQCOA8KEREZCJOTsDOnVJQsbG5c/vObp0BAEm5DCjEgEJERCZibw+MG9f09p1cOwEA0ovSUaWugo1VE1INtVk8xUNERGbB19EXzipnaIQGCTkJcpdDMmNAISIik9i7F/jwQ+DUqaa1VyqU6OXdCwBwIfuCCSsjS8CAQkREJrF5MzB/PvDzz01fZ6D/QADAb6m/magqshQMKEREZBLaae6bcgWPVj/ffgCAxFuJJqiILAkDChERmURqqvQcGNj0dcI9wgEAcdlxJqiILAkDChERmYT2CErHjk1fp7d3bwBAVkkWbpbcNEFVZCkYUIiIyOg0GuDmH/miKdPcazmpnBDsGgwAvJKnnWNAISIio8vNlUIKcOc7Gd9OO+X92cyzxi2KLAoDChERGV1WlvTs5gbY2hq27oigEQCAPVf3GLkqsiScSZaIiIwuJASIjQWKiw1fd2SnkQCAUxlNnECF2iQGFCIiMjoHB2DIkOat282zGwDgRtENFFcWw9HW0YiVkaXgKR4iIjIr7vbu8O7gDQD4Pet3mashuTCgEBGR0W3fDvz738C5c81bXzsO5deUX41YFVkSBhQiIjK6r78Gnn0W2LeveesPChgEAIi7yQnb2isGFCIiMrrsbOnZ27t563d26wwAuHTrkpEqIkvDgEJEREbX0oBS+67Gao3aSFWRJWFAISIio2tpQAlzD4OdtR3KqstwOfey8Qoji8GAQkRERlV7mvvmBhQrpRW6eHQBACTnJRupMrIkDChERGRULZnmvrYglyAAQFphmhGqIkvDgEJEREalPb3j5gbY2DR/O6GuoQCA+JvxRqiKLA1nkiUiIqMKDgaOHAFKS1u2nd4+vQFwsrb2igGFiIiMysEBiIxs+Xa0U94n53MMSnvEUzxERGSWQlxDAABpBWmo1lTLXA21NgYUIiIyqo0bgQ8+AC5ebNl2/Jz8YGdtB7VQIyU/xRilkQVhQCEiIqPasAF47jlpHEpLKBVK9PDqAYDjUNojBhQiIjKqzEzp2de35dvq49MHAANKe8SAQkRERpWRIT37+bV8W109ugIAZ5NthxhQiIjIaNTqmllkjXEEJdwjHABw+RYDSnvDgEJEREZz86Y0i6xCAXh5tXx7Ye5hAKQjKEKIlm+QLAYDChERGU1urvTs6gpYG2GmLW1AyS/PR25Zbss3SBaDAYWIiIzm1i3puSX34KnNwcYBAU4BADgOpb1hQCEiIqPp1w84ehRYt8542+Q4lPaJAYWIiIzG0REYPBgYNsx42wx3/yOg8AhKu8KAQkREZo0BpX3izQKJiMhofvxRmuJ+7FhgwADjbFM7UPZq3lXjbJAsAgMKEREZzTffAFu2SHc0NlZA6eTaCQBwLf+acTZIFoGneIiIyGiysqRnY8wiqxXqFiptuyQLRRVFxtswmbVmBZTVq1cjODgYdnZ2GDx4MI4fP95o+/z8fMydOxd+fn5QqVTo0qULduzY0ayCiYjIfOXlSc8eHsbbpqudK5xVzgCAG0U3jLdhMmsGB5RNmzZhwYIFWLp0KU6fPo0+ffpg3LhxyM7Orrd9ZWUlxo4di5SUFGzZsgWJiYlYs2YNAgICWlw8ERGZF+0098YMKADg5ygdkmFAaT8MHoOycuVKPPHEE5g9ezYA4NNPP8X27duxdu1avPzyy3Xar127Frm5uThy5AhsbGwAAMHBwS2rmoiIzE51NaD9f1VjnuIBAH8nfyTeSmRAaUcMOoJSWVmJU6dOISoqqmYDSiWioqIQGxtb7zo///wzIiMjMXfuXPj4+KBXr15Yvnw51Gp1g/upqKhAYWGh3oOIiMzbzZuAEIBSabyZZLX8nKTEk1mcadwNk9kyKKDk5ORArVbDx8dHb7mPjw8yM+v/pbl69Sq2bNkCtVqNHTt2YPHixXjvvffwz3/+s8H9xMTEwMXFRfcIDAw0pEwiIpKBdoCslxdgZWXcbfs7+gMArhXwSp72wuSXGWs0Gnh7e+Pzzz+HlZUVIiIikJ6ejnfffRdLly6td51FixZhwYIFuq8LCwsZUoiIzFy3bsDvvwMlJcbfdlfPrgCApLwk42+czJJBAcXT0xNWVlbI0sbkP2RlZcHX17fedfz8/GBjYwOrWnG6e/fuyMzMRGVlJWxtbeuso1KpoFKpDCmNiIhkZmcH9O5tmm0HOkv/k5qSn2KaHZDZMegUj62tLSIiIrB3717dMo1Gg7179yIyMrLedYYNG4YrV65Ao9Holl26dAl+fn71hhMiIqLbdfHoAgBIyk2CWtPwGEZqOwy+zHjBggVYs2YNNmzYgISEBDz55JMoKSnRXdUzc+ZMLFq0SNf+ySefRG5uLubPn49Lly5h+/btWL58OebOnWu8XhARkex++AF46y3gDlNjNUuQSxDsre1Roa7gPXnaCYPHoEybNg03b97EkiVLkJmZib59+2Lnzp26gbOpqalQKmtyT2BgIHbt2oXnnnsOd911FwICAjB//ny89NJLxusFERHJbssW4NtvpVM9gwYZd9tWSiuEuYfhfPZ5XMu/hm6e3Yy7AzI7zRokO2/ePMybN6/e9w4cOFBnWWRkJI4ePdqcXRERkYXQXsxp7DlQtIJcgnA++zzHobQTvBcPEREZhTagNHDNRIuFu4cDAC7dumSaHZBZYUAhIiKjMHVA0Q6U5RiU9oEBhYiIWqyiouZGgbfN5Wk04R7SERQGlPaBAYWIiFpMew8eGxvA3d00+9Ce4knKTUK1pto0OyGzwYBCREQtpg0oXl6AQmGafQS6BEJlpUKVpgqpBamm2QmZDQYUIiJqsZ49pWnuf/zRdPtQKpQIcw8DAFy+xdM8bR0DChERtZh2mntjz39yO45DaT8YUIiIyGLwUuP2w+R3MyYiorbvm2+ACxeAyZOBIUNMtx9tQOERlLaPAYWIiFrsxx+lqe59fU0bUHRzoXAMSpvHUzxERNRipp6kTUs7BiUlPwWV6krT7oxkxYBCREQtlpUlPZtqkjYtP0c/dLDpALVQIzkv2bQ7I1kxoBARUYu1VkBRKBQ1lxpzHEqbxoBCREQtUlkJFBZKrz09Tb8/jkNpHxhQiIioRW7dkp6VSsDV1fT746XG7QMDChERtYh2gKy3N2BlZfr9cbK29oGXGRMRUYv06gUkJdWc5jE1zoXSPjCgEBFRi9jYAKGhrbe/zu6dAQDXC6+joroCKmtV6+2cWg1P8RARkUXx6eADNzs3aIQGCTkJcpdDJsKAQkRELfKf/wBPPQXs29c6+1MoFOju1R0AkJiT2Do7pVbHgEJERC2yfTvwySdAQisezNDOhXIl90rr7ZRaFQMKERG1SEaG9BwQ0Hr7DHOTAsrVvKutt1NqVQwoRETUIseOSc8eHq23z1A3aVRuUl5S6+2UWhUDChERNZsQNa9NPc19bdoreRhQ2i4GFCIiarb8/JrXQUGtt9/OblJASS9MR3l1eevtmFoNAwoRETWb9iaBLi6AnV3r7dfTwRNOtk4QELyrcRvFgEJERM2WkyM9e3u37n4VCgVP87RxDChERNRsw4cDpaXAoUOtv2/taZ6kXAaUtohT3RMRUYvY20uP1qYLKDyC0ibxCAoREVkknuJp2xhQiIio2VatAmbNAnbvbv198xRP28aAQkREzbZnD7BhA5AkQ0bQHkFJzk+GWqNu/QLIpBhQiIio2bTT3Pv5tf6+A50DYaWwQqW6EpnFma1fAJkUAwoRETXbjRvSsxwBxUppBa8OXgCArJKs1i+ATIoBhYiImqW6uuYISmCgPDX4OvoCADKKMuQpgEyGAYWIiJrl1i1AowEUitafqE0r0FlKRmmFafIUQCbDgEJERM2SnS09u7sDVlby1ODv5A+AR1DaIgYUIiJqFu00976+8tXg00G6hXJ2SbZ8RZBJcCZZIiJqltGjgbIyoLBQvhq0Y1BuFN+QrwgyCQYUIiJqNju71r2L8e0CnAMAgJcZt0E8xUNERBbLu4M0OpcBpe1hQCEiomZZvhyIjgYOH5avho7OHQEAN4puQCM08hVCRseAQkREzbJzJ/Dll8D16/LV4OvoCwUUqNZUI6c0R75CyOgYUIiIqFnknEVWy1ppDR9H6Uqe9MJ0+Qoho2NAISKiZpHzPjy1aedCSS9iQGlLGFCIiMhgJSVAaan02sdH3lq0AeVGES81bksYUIiIyGDaoycODoCTk7y1aKe7T8lPkbcQMioGFCIiMph2YGzHjtK9eOTU2a0zAOBq3lV5CyGjYkAhIiKDZWVJzwEB8tYBAMGuwQB4BKWt4UyyRERksGnTgKlTa8ahyEkbUFILUuUthIyKAYWIiJpFpZIecgtyCQIAZBRnoKK6AiprMyiKWoyneIiIyKJ5OnhCZSWFkoziDJmrIWNhQCEiIoPNmgVERQHx8XJXAigUCt1NA68XyjitLRkVAwoRERlswwZg7155p7mvLdQtFABw6dYlmSshY2FAISIig6jVNa/Dw+Wro7auHl0BMKC0JQwoRERkkMzMmteBgfLVUVuYexgA4EruFZkrIWNhQCEiIoNcu1bz2tpMrgXVnuJJzk+WuRIyFgYUIiIySHa29GwOk7RpaQMKZ5NtOxhQiIjIICkp0nNkpKxl6AlxDQEA5JfnI788X95iyCgYUIiIyCBFRYCtLRASInclNTrYdoBPB+m2yjyK0jYwoBARkUEWLwbKyoDXX5e7En0hblJiSs7jOJS2gAGFiIgMplQC9vZyV6HP38kfAGeTbSsYUIiIqE0IcJJG7aYVpMlcCRlDswLK6tWrERwcDDs7OwwePBjHjx9v0nobN26EQqHA1KlTm7NbIiKSWX4+MHAg8NBD+hO2mYNOLp0AANcKrt2hJVkCgwPKpk2bsGDBAixduhSnT59Gnz59MG7cOGRrrztrQEpKCp5//nmMGDGi2cUSEZG8UlKAkyeBAwcAKyu5q9EX7BoMgAGlrTA4oKxcuRJPPPEEZs+ejR49euDTTz+Fg4MD1q5d2+A6arUajz76KF5//XWEhoa2qGAiIpKP9hLj4GA5q6hfJ9c/jqDkM6C0BQYFlMrKSpw6dQpRUVE1G1AqERUVhdjY2AbXe+ONN+Dt7Y3HH3+8SfupqKhAYWGh3oOIiORn1gHlj1M8GcUZqKiukLkaaimDAkpOTg7UajV8fHz0lvv4+CCz9s0Zavntt9/wxRdfYM2aNU3eT0xMDFxcXHSPQHO52QMRUTunPVhuaytvHfXxdPCEvbV0aVFaIQfKWjqTXsVTVFSEGTNmYM2aNfD09GzyeosWLUJBQYHukZbGXzQiInNQWio9OzvLW0d9FAoFT/O0IQbd5snT0xNWVlbIysrSW56VlQVfX9867ZOSkpCSkoLJkyfrlmk0GmnH1tZITExE586d66ynUqmgUqkMKY2IiFqBq6v0PGGCrGU0qJNLJ1zMuciBsm2AQUdQbG1tERERgb179+qWaTQa7N27F5H13JShW7duOH/+PM6ePat73H///Rg9ejTOnj3LUzdERBbG3l46vWOOY1CAWpca8wiKxTP4RtkLFixAdHQ0BgwYgEGDBuGDDz5ASUkJZs+eDQCYOXMmAgICEBMTAzs7O/Tq1Utvfdc/4vfty4mIyPwdOiTNf6JQyF1J/bSneFIKUuQthFrM4IAybdo03Lx5E0uWLEFmZib69u2LnTt36gbOpqamQqnkBLVERG2Vuc1/UhuPoLQdCiGEkLuIOyksLISLiwsKCgrgbI4js4iIyCwcTj2M4euGI9g1GMnzedNAubXk7zcPdRARUZO89x4QEQF88onclTRMe4rneuF1qDVmNhc/GYQBhYiImiQuDjh9GsjNlbuShvk5+sFaaY1qTTVuFN2QuxxqAQYUIiJqkitXpGdzvmOJldIKgc7SFaIp+SnyFkMtwoBCRERNcumS9Nyli7x13In2NE9yPsegWDIGFCIiuqPCQkB70/qwMHlruZMenj0AABeyL8hcCbUEAwoREd2R9vSOpyfg4iJvLXfSzbMbAOBy7mWZK6GWYEAhIqI7uvzH3/rwcHnraIowd+kQDwOKZWNAISKiO9JogG7dgD595K7kzrQBJSk3CRYw1Rc1wOCZZImIqP2ZPl16WIIglyAooEBZdRlult6EdwdvuUuiZuARFCIialNU1ir4O/kD4KXGlowBhYiIGiWE9LAkwa7BABhQLBkDChERNSojA3B1BYYNk8aiWAIGFMvHgEJERI2Kj5fmQcnJASzlZvXagMK7GlsuC/lVIyIiucTHS8/du8tbhyF0R1AKUmStg5qPAYWIiBp14Y8JWXv2lLcOQ/AUj+VjQCEiokbFxUnPvXvLW4chagcUzoVimRhQiIioQUIAR45Iry3pCEqgcyAUUKC0qhQ5pTlyl0PNwIBCREQNunq15nXXrvLVYSjOhWL5GFCIiKhBlZU1r21t5aujOTq5dgLAgGKpGFCIiKhB3btb5kRtAAfKWjoGFCIiapOCXYIBMKBYKgYUIiKqlxBASoplHj0BOBeKpWNAISKieqWmAiEhgJcXUF0tdzWG4ykey8aAQkRE9Tp2THoOCgKsreWtpTk4F4plY0AhIqJ6HT0qPUdGyltHcwW5BOnmQrlZelPucshADChERFQv7RGUIUPkraO5VNYqdHTuCAC4kntF5mrIUAwoRERUh1oNnD8vve7XT95aWqKze2cAQFJuksyVkKEYUIiIqI4LF4CiIsDR0bLuYny7zm5/BJQ8BhRLw4BCRER1HD4sPQ8ZAlhZyVtLSzCgWC4LHJdNRESmNmQIsGiRZd1/pz4hbiEAgGv512SuhAzFgEJERHX062fZY0+0tJcaX7p1Sd5CyGA8xUNERG1WL+9eAICskizkl+fLWwwZhAGFiIj0nD8P/PILkJ8vdyUt52jrCF9HXwC81NjSMKAQEZGeNWuAiROBxYvlrsQ4unh0AcDTPJaGAYWIiPT8+qv0PHKkvHUYSxd3KaAk5iTKXAkZggGFiIh0MjOB33+XXreZgKI9gpLLIyiWhAGFiIh0/vtf6blHD8DHR95ajCXcIxwAx6BYGgYUIiLSWb9eeu7dW9YyjCrMPQwAcPnWZd7V2IIwoBAREQBACCA+Xnr9yCPy1mJM4e7hUCqUKKgoQFZJltzlUBMxoBAREQAgIUEKKQ4OwLhxcldjPCprFQKcAgAAyXnJMldDTcWAQkREAKRxJykpwKZNgEoldzXGpZ3yPiU/Rd5CqMkYUIiISKdTJ+C+++Suwvi0U94zoFgOBhQiImrzgpyDADCgWBIGFCIiwjvvAJMmAbt3y12JaYS6hQIAUgpS5C2EmowBhYiI8M03wI4dwPXrcldiGkEu0hGUa/nXZK6EmooBhYionUtKAs6dA6ysgPvvl7sa09DOhXI17yqqNdUyV0NNwYBCRNTO/fCD9Hz33YCHh6ylmEygSyBUVipUaaqQVpAmdznUBAwoRETt3HffSc8PPCBvHaakVCh141A45b1lYEAhImrHzpwBTp4EbGyAP/9Z7mpMq4dXDwDAmcwzMldCTcGAQkTUjm3YID3/6U+At7e8tZhaf7/+AIDz2edlroSawlruAoiISD7DhklHUGbOlLsS0+vtLd0B8VzmOZkroaZgQCEiasf+8hfp0R5u8tvZvTMAIK2Qg2QtAU/xEBERFAq5KzC9QOdAAEB+eT6KKopkrobuhAGFiKgdOnkSeOMNIC9P7kpaj5PKCa52rgCk+VDIvDGgEBG1Q8uXA0uXAi+9JHclrauPTx8AwOmM0zJXQnfCgEJE1M5cvAhs3Sq9fu45WUtpddqAcuHmBZkroTthQCEiamfefVcaFDtlCtC9u9zVtC7tXCgJOQkyV0J3woBCRNSOpKcDX30lvW5vp3cAIMQtBABvGmgJGFCIiNqRd98FqqqAESOAyEi5q2l9wa7BAIBrBdcg2sO11RaMAYWIqJ3IyAA++0x6vXixvLXIJcglCABQXFmMW2W3ZK6GGsOAQkTUTlhbA9OnS7PHRkXJXY087Kzt4O/kDwBIyU+RtxhqFAMKEVE74eUFrF0L7N3bPiZma0hnN2lG2Ys5F2WuhBrDgEJE1MZpNPpT2atU8tViDvr59gMAnEg/IXMl1JhmBZTVq1cjODgYdnZ2GDx4MI4fP95g2zVr1mDEiBFwc3ODm5sboqKiGm1PRETGtWYNMGYMkMArawEAgwIGAQBO3GBAMWcGB5RNmzZhwYIFWLp0KU6fPo0+ffpg3LhxyM7Orrf9gQMHMH36dOzfvx+xsbEIDAzEvffei/T09BYXT0REjbt5E1i0CNi/H/jf/+SuxjxoA8rpjNOoVFfKXA01RCEMvM5q8ODBGDhwID766CMAgEajQWBgIJ5++mm8/PLLd1xfrVbDzc0NH330EWY28f7ehYWFcHFxQUFBAZydnQ0pl4ioXXvsMeDrr4G+fYETJ6SBsu2dEAIub7ugqLII8U/Fo7tXO5utrhW15O+3QUdQKisrcerUKUTVGv6tVCoRFRWF2NjYJm2jtLQUVVVVcHd3b7BNRUUFCgsL9R5ERGSYH3+UwolSCXz6KcOJlkKhQBePLgCA37N+l7kaaohBASUnJwdqtRo+Pj56y318fJCZmdmkbbz00kvw9/fXCzm3i4mJgYuLi+4RGBhoSJlERO3e9evA3/8uvX7xRWDwYHnrMTdDOg4BAJy8cVLmSqghrXoVz9tvv42NGzfixx9/hJ2dXYPtFi1ahIKCAt0jLS2tFaskIrJsVVXAtGnS+JM+fYBly+SuyPxor+Q5mcGAYq4MOuDn6ekJKysrZGVl6S3PysqCr69vo+uuWLECb7/9Nvbs2YO77rqr0bYqlQqq9n4dHBFRM+XmAuXlgLMz8P33vKy4PpGB0jz/R68fRZW6CjZWNjJXRLcz6AiKra0tIiIisHfvXt0yjUaDvXv3IrKRmzq88847ePPNN7Fz504MGDCg+dUSEdEd+fgAR44A+/YBnTvLXY156ubZDe727iivLudpHjNl8CmeBQsWYM2aNdiwYQMSEhLw5JNPoqSkBLNnzwYAzJw5E4sWLdK1/9e//oXFixdj7dq1CA4ORmZmJjIzM1FcXGy8XhARESoqal6rVEBEhHy1mDulQolhgcMAAKcyTslcDdXH4IAybdo0rFixAkuWLEHfvn1x9uxZ7Ny5UzdwNjU1FRkZGbr2n3zyCSorK/HnP/8Zfn5+useKFSuM1wsionauoAAYMABYsgSorpa7GsvQw6sHAE55b66addHZvHnzMG/evHrfO3DggN7XKSkpzdkFERE1kVoNPPIIEBcH5OUB8+YB3t5yV2X+unl2AwDE34yXuRKqD+/FQ0RkwYQA5s8HduwA7OykuU8YTppGd0+eGydQpa6SuRq6HQMKEZEFW7IEWL1aujvxhg3AwIFyV2Q5evv0hru9O4ori3E647Tc5dBtGFCIiCyQEMA//yk9AGDVKuChh+StydIoFUqMCBoBADh47aDM1dDtGFCIiCzQiRPA4sXS63/9C5g7V956LJX2Sp5j6cdkroRuxzszEBFZoEGDgPXrpUnZnntO7mosV3+//gCA4+nHZa6EbseAQkRkIYqKpKt0goKkr6Oj5a2nLRgUMAhWCiukFaYhMScRXT27yl0S/YGneIiILMD168CIEcC990ohhYzDSeWEu4PvBgDsS94nbzGkhwGFiMjMxcZKk7CdOyeFE94/1bgiO0q3ajlx44TMlVBtDChERGZKCGDNGmDUKCArC+jdGzh2DLjD/VbJQAMDpGuzOQ7FvDCgEBGZoZwc4C9/AebMAaqqgAcflG4AGBwsd2Vtz0B/KaDE34xHUUWRzNWQFgMKEZEZWrIE+P57wNoaiIkBvvsOcHSUu6q2yc/JDx2dO0JAcMI2M8KAQkRkJmrfjXjxYmD4cODoUeDllwElP61NalDAIAA8zWNO+CtPRCSz5GTggQeAhx+uWebnBxw6BEREyFdXe6IdKHso9ZDMlZAW50EhIpJJUZE0C+yKFdLREysr4PJlIDxc7sranyEdhwAAT/GYER5BISJqZTk5wBtvAKGhwFtvSeFkzBjg7FmGE7n09e0LG6UN0ovSkZSbJHc5BAYUIqJWdfw40KkTsHSpFFTCwoAffgB27wZ69ZK7uvbL0dYRgzsOBsAbB5oLBhQiIhPSaKQxJlp9+wLu7kD//sA33wAJCcCf/gQoFLKVSH/Q3tmY41DMA8egEBGZwLVr0s381q0D1GoppFhbA7a20pU5/v4MJeZmeNBwAAwo5oIBhYjISNLSgP/7P2nOkl9/rVnu6gokJgI9e0pfBwTIUh7dgfZKniu5V5BTmgNPB0+ZK2rfeIqHiMgIPvpIusvw3LlSOFEogNGjga+/Bm7cqAknZL7c7N0Q5h4GAPgh4QeZqyEGFCKiJqqulga5vvsuMGkS8NNPNe/17y+FkqFDgXfekU7x7NsHPPIIYG8vX81kmMd6PwYA+F/S/2SuhHiKh4ioASUlwK5dwIkT0h2FT5wASktr3g8NBaZMkV4PGgRkZAA+PvLUSsYxOmQ0lv26DAevHYRao4aV0kruktotBhQiavdu3QIuXADOn5fGh0ydKi0vLpZu0lebqyswciRw993AuHE1y62tGU7agiEdh8BF5YKbpTdxLP0YhgYOlbukdosBhYjaFbVaOkWTnCzN2pqQAGRm1rw/aVJNQPHxAcaOlcaWREZKj27deF+ctszWyhaTukzCN+e/wXcXvmNAkZFCCCHkLuJOCgsL4eLigoKCAjg7O8tdDhGZod9/B65f13+kpUljQbp3rxkvIoR0FKSwUH/9Tp2A3r2BUaOA559v9fLJjPyc+DOmbJwCNzs33HrxFhS8HrzZWvL3m0dQiMhsqNXS/Wi0du2SZlvNy6t55OYC2dlASAiwenVN21GjgPz8+rfr4FDzWqEA5s2T9hMWBnTtCvToATg5maRLZIHGdR4HK4UV8srzkFaYhiCXILlLapcYUIjojoQAqqqke8aUlkqnOLy8at7fs0caUFpWJj1KS2ted+wIzJ5d03bmTGnMR1GR9CgsrHkeMgQ4cKCm7YwZwM2b9dfUu7f+13fdJW0jIAAIDJT2GxgoPUJC9Nu+9VaLvh3UxqmsVYjwj8Dx9OPYnbQbj/d/XO6S2qV2H1Dy8oDKSukDWKv2az+/mte3bgHl5Q237dixZmbInBzpA7u+doB0Tlt7Hjs7W/qAbqhtcLA0AE/bVvt/idp2tduHhkozVWrb5uQ03LZz55rLH7OypEdDbcPDgQ4darZ7/XrD9YaHA9ojednZQEpKw227dAHc3KTXN28CV640XEOXLjV/FHNypLEDjbXV/uxyc4Fz5xrvW9Af/4OUlwecPNl429BQ6XV+vjTfhVotTWeu0ei/7t1bmtZcu92NG2vev/154EDpZnHa7a5YUXd72tdDhwIPPyy1LS4Gnnmm4bYjRgDz50ttKyqAyZOloFHfY+xYYNWqmj67uNS8p1br/9wmTQK2bav5+v77pTBSn5Ej9QPKzp0Nh468PP2vhw8HCgqk3xHtw90d8PaWgkdttSdGI2qpiWETcTz9OHZc2cGAIhdhAQoKCgQAUVBQYPRtjxkjhPRxXPdha6vf9v77G24LCFFVVdP24Ycbb5ufX9P28ccbb3vjRk3bZ55pvO3lyzVtX3658bZnz9a0ffPNxtseOVLTduXKxtvu3l3T9tNPG2+7dWtN26++arztN9/UtP3hh8bbrllT03bnzsbbfvBBTdtDhxpvu3x5TdtTpxpv++qrNW0TEhpv+9xzNW3T0hpvO2dOTdtbtxpv+8gjNW0rKhpvO2WK0GNt3XDb8eP1244YIcTgwUKMHi3ExIlCPPigEDNmSLWuXKnfdv16Ib74QohNm6SfzeHDQpw/L0RKihB5eYLILJxIPyGwDMJxuaMoryqXuxyL1ZK/3+3+CIpCUfd+GNqvbx+pb20N2NjU3xaQPrq1bG0BO7u6beoba2VnBzg6Nq2tvb00wO/2NvXV3KED4OnZcNva5/odHQFfX/191d5/7X47OupP1X37dlWqmvecnKTBhw21rT02wMlJOqrTUFvt90j7umvXhtu6uOi3vX0Wz9p98/Coed2hA9CnT8Ntvb316x0yRPqeW1nVfQ4Lq2nr7CxdrqpU1t9+4ED97T79dMPbHTCgpq2DA/D223W3q30dHl7T1toa+Oor6WdZ36P2KRtAOkJ1exuVSvp9vf1386ABN3+Njm56WyK59Pfrj47OHXG98Dp+ufILpnabKndJ7Q6v4iEiIqrH3//v7/j89Od4PvJ5vHvvu3KXY5Fa8vebV/MTERHVY1TwKADArqRdMlfSPjGgEBER1ePezvdCAQXOZ59HemG63OW0OwwoRERE9fB08MSggEEAgP+79H8yV9P+MKAQERE14IHuDwAA1p5ZCwsYstmmMKAQERE14LG7HoOtlS1O3DiB7Ze3y11Ou8KAQkRE1AB/J39E95GujX//6PsyV9O+MKAQERE1YtHwRVBAgX3J+3A176rc5bQbDChERESNCHELwT0h9wAAntr+lMzVtB8MKERERHfw/rj3oYACu5J2ITEnUe5y2gUGFCIiojvo7dMb48LGAQA2XdgkczXtAwMKERFREzzcU7qN+HcXvpO5kvaBAYWIiKgJpnSbAhulDS7cvIC47Di5y2nzGFCIiIiawNXOFRPDJwIAPjnxiczVtH0MKERERE00b9A8AMDnpz/nYFkTY0AhIiJqoqjQKEwMn4hqTTX+sf0fnP7ehBhQiIiIDPDBuA9gpbDCgZQD+PLcl3KX02YxoBARERkg3CMccwfOBQDM+mkW9ifvl7mitokBhYiIyEDvjH0Hw4OGAwAe/O5BToFvAgwoREREBlJZq/DDQz/A08ETeeV5mPHjDKg1arnLalMYUIiIiJrBq4MXjj5+FE62TjiSdgT/OvwvuUtqUxhQiIiImqmze2e8O/ZdAMDi/YtxIfuCzBW1HQwoRERELTAnYg4iO0ZCIzR4dtez0AiN3CW1CQwoRERELaBQKPDF/V/AztoOe67uwYu7X5S7pDaBAYWIiKiFunt1x+f3fQ4AeC/2PSw/tFzmiiwfAwoREZERzOgzA/+KkgbKvrrvVbwf+z5nmm0BBhQiIiIjeXHYi3hxqHSKZ8H/FmDSN5NwJfeKzFVZJgYUIiIiI1o+ZjkWRi4EAPxy5RdEfhGJhJsJMldleRhQiIiIjMhKaYUV967A7hm70dOrJ3JKczDxm4m4Xnhd7tIsCgMKERGRCUSFRuGXR39BgFMAUvJT0GN1D3xx+gtUqivlLs0iMKAQERGZSKBLIH6d9SsG+g9EUWUR/vZ/f8Ndn9yFpNwkuUszewwoREREJtTZvTN+nfUrloxcAmeVMxJvJWLEuhHYdmkbr/JpBAMKERGRidnb2OP10a/j9JzTCHENQUZxBiZ/Oxmj1o/CDwk/yF2eWWpWQFm9ejWCg4NhZ2eHwYMH4/jx442237x5M7p16wY7Ozv07t0bO3bsaFaxRERElqyze2eceOKE7lLkQ6mH8OB3D6LrR12x9sxaJNxM4FGVPyiEgd+JTZs2YebMmfj0008xePBgfPDBB9i8eTMSExPh7e1dp/2RI0cwcuRIxMTE4L777sM333yDf/3rXzh9+jR69erVpH0WFhbCxcUFBQUFcHZ2NqRcIiIis/R71u9Yfmg5Nl3YpLc8wCkAEf4R6OvTF5GBkejh1QNOtk6wt7GHykoFhUIhU8WGa8nfb4MDyuDBgzFw4EB89NFHAACNRoPAwEA8/fTTePnll+u0nzZtGkpKSrBt2zbdsiFDhqBv37749NNPm7RPBhQiImqrzmScwdaLW7E/ZT9O3DiB8uryBtsqoICdtR0cbBxgb2MPe2t73bN2mYvKBV4OXrC1soW10hrWSmtYKa10rxt6TAyfCO8OdQ80tERL/n5bG9K4srISp06dwqJFi3TLlEoloqKiEBsbW+86sbGxWLBggd6ycePGYevWrQ3up6KiAhUVFbqvCwoKAEgdJSIiaks6d+iMhRELsTBiIUqrSnHqxilcuHkBpzJO4UzGGSTlJunukCwgUPbHf8a2e8ZuDOo4yKjb1P7dbs5pK4MCSk5ODtRqNXx8fPSW+/j44OLFi/Wuk5mZWW/7zMzMBvcTExOD119/vc7ywMBAQ8olIiKiJhr79liTbbuoqAguLi4GrWNQQGktixYt0jvqotFokJubCw8Pj1Y991ZYWIjAwECkpaW1yVNL7J9lY/8sG/tn2di/phFCoKioCP7+/gava1BA8fT0hJWVFbKysvSWZ2VlwdfXt951fH19DWoPACqVCiqVSm+Zq6urIaUalbOzc5v8BdRi/ywb+2fZ2D/Lxv7dmaFHTrQMuszY1tYWERER2Lt3r26ZRqPB3r17ERkZWe86kZGReu0BYPfu3Q22JyIiIjL4FM+CBQsQHR2NAQMGYNCgQfjggw9QUlKC2bNnAwBmzpyJgIAAxMTEAADmz5+PUaNG4b333sOkSZOwceNGnDx5Ep9//rlxe0JERERthsEBZdq0abh58yaWLFmCzMxM9O3bFzt37tQNhE1NTYVSWXNgZujQofjmm2/w2muv4ZVXXkF4eDi2bt3a5DlQ5KRSqbB06dI6p5vaCvbPsrF/lo39s2zsn+kZPA8KERERkanxXjxERERkdhhQiIiIyOwwoBAREZHZYUAhIiIis9MmA8pbb72FoUOHwsHBod4J3s6dO4fp06cjMDAQ9vb26N69O/7973/XaXfgwAH0798fKpUKYWFhWL9+fZ02q1evRnBwMOzs7DB48GAcP35c7/3y8nLMnTsXHh4ecHR0xIMPPlhn4rrU1FRMmjQJDg4O8Pb2xgsvvIDq6upm96+p2zTX/t3u0qVLmDJlCjw9PeHs7Izhw4dj//79Ztvf5ti+fTsGDx4Me3t7uLm5YerUqW2qf4B0j62+fftCoVDg7Nmzeu/9/vvvGDFiBOzs7BAYGIh33nmnzvqbN29Gt27dYGdnh969e2PHjh167wshsGTJEvj5+cHe3h5RUVG4fPmyXpvc3Fw8+uijcHZ2hqurKx5//HEUFxc3u08pKSl4/PHHERISAnt7e3Tu3BlLly5FZWVlm+hfS9zpd621xcTEYODAgXBycoK3tzemTp2KxMREvTbG+jwzxr/Flnr77behUCjw7LPPWm7/RBu0ZMkSsXLlSrFgwQLh4uJS5/0vvvhCPPPMM+LAgQMiKSlJfPXVV8Le3l6sWrVK1+bq1avCwcFBLFiwQMTHx4tVq1YJKysrsXPnTl2bjRs3CltbW7F27Vpx4cIF8cQTTwhXV1eRlZWla/OPf/xDBAYGir1794qTJ0+KIUOGiKFDh+rer66uFr169RJRUVHizJkzYseOHcLT01MsWrSo2f1ryjbNuX+3Cw8PFxMnThTnzp0Tly5dEk899ZRwcHAQGRkZZtff5tiyZYtwc3MTn3zyiUhMTBQXLlwQmzZt0r1v6f3TeuaZZ8SECRMEAHHmzBnd8oKCAuHj4yMeffRRERcXJ7799lthb28vPvvsM12bw4cPCysrK/HOO++I+Ph48dprrwkbGxtx/vx5XZu3335buLi4iK1bt4pz586J+++/X4SEhIiysjJdm/Hjx4s+ffqIo0ePikOHDomwsDAxffr0Zvfpl19+EbNmzRK7du0SSUlJ4qeffhLe3t5i4cKFbaJ/zdWU37XWNm7cOLFu3ToRFxcnzp49KyZOnCiCgoJEcXGxro0xPs+M9W+xJY4fPy6Cg4PFXXfdJebPn2+x/WuTAUVr3bp19f4Br89TTz0lRo8erfv6xRdfFD179tRrM23aNDFu3Djd14MGDRJz587Vfa1Wq4W/v7+IiYkRQgiRn58vbGxsxObNm3VtEhISBAARGxsrhBBix44dQqlUiszMTF2bTz75RDg7O4uKiopm9a8p27SE/gkhxM2bNwUAcfDgQd2ywsJCAUDs3r3b7PprqKqqKhEQECD+85//NNjGkvtXuw/dunUTFy5cqBNQPv74Y+Hm5qb3+/DSSy+Jrl276r5+6KGHxKRJk/S2OXjwYPH3v/9dCCGERqMRvr6+4t1339W9n5+fL1Qqlfj222+FEELEx8cLAOLEiRO6Nr/88otQKBQiPT29Rf2r7Z133hEhISFttn9NcaffNXOQnZ0tAIhff/1VCGG8zzNj/FtsiaKiIhEeHi52794tRo0apQsolti/NnmKpzkKCgrg7u6u+zo2NhZRUVF6bcaNG4fY2FgAQGVlJU6dOqXXRqlUIioqStfm1KlTqKqq0mvTrVs3BAUF6drExsaid+/eend8HjduHAoLC3HhwoVm9aUp27SU/nl4eKBr16748ssvUVJSgurqanz22Wfw9vZGRESE2fXXUKdPn0Z6ejqUSiX69esHPz8/TJgwAXFxcbo2ltw/QLr31hNPPIGvvvoKDg4Odd6PjY3FyJEjYWtrq1d7YmIi8vLymtS/5ORkZGZm6rVxcXHB4MGD9X4XXV1dMWDAAF2bqKgoKJVKHDt2rNn9u119nyVtqX930pTfNXNQUFAAALqflbE+z4zxb7El5s6di0mTJtWpwRL7x4AC4MiRI9i0aRPmzJmjW5aZman3QwIAHx8fFBYWoqysDDk5OVCr1fW2yczM1G3D1ta2zjiR29vUtw3te83RlG1aSv8UCgX27NmDM2fOwMnJCXZ2dli5ciV27twJNzc3s+uvoa5evQoAWLZsGV577TVs27YNbm5uuPvuu5Gbm2vx/RNCYNasWfjHP/6h94eztpb0r/b7tddrqI23t7fe+9bW1nB3d292/2535coVrFq1Cn//+991y9pS/5qiKb9rctNoNHj22WcxbNgw3azmxvo8M8a/xebauHEjTp8+rbvVTG2W2D+LCSgvv/wyFApFo4+LFy8avN24uDhMmTIFS5cuxb333muCypumdv86dOgAAJg4cWKL+2fObu9ffT9PIQTmzp0Lb29vHDp0CMePH8fUqVMxefJkZGRkyN2FBjX191Wj0QAAXn31VTz44IOIiIjAunXroFAosHnzZpl70bCm9m/VqlUoKirCokWL5C7ZIM35vElPT8f48ePxl7/8BU888YRMlVNTzJ07F3Fxcdi4caPcpRhNWloa5s+fj6+//hp2dnZyl2MUBt+LRy4LFy7ErFmzGm0TGhpq0Dbj4+MxZswYzJkzB6+99pree76+vnVGN2dlZcHZ2Rn29vawsrKClZVVvW18fX1126isrER+fr5ear29zfHjx/X6d/36dYwdOxbff/89evToYXD/tNu8vS7te3L0r75abu/f7UJDQ7Fv3z5s27YNeXl5ult+f/zxx9i9ezc2bNiAl19+2az6q9XU31dtyKr9fVCpVAgNDUVqaqpuv5bav3379iE2NrbO/TwGDBiARx99FBs2bGiw9qb0r/b72mV+fn56bfr27atrk52drbeN6upq5ObmNrt/Wjdu3MDo0aMxdOjQOjdCNcf+mZKnp+cdf9fkNG/ePGzbtg0HDx5Ex44ddctb+nlmzH+LzXHq1ClkZ2ejf//+umVqtRoHDx7ERx99hF27dlle/wwasWJhGhskGxcXJ7y9vcULL7xQ7/svvvii6NWrl96y6dOn1xkING/ePN3XarVaBAQE1Bl0uGXLFl2bixcv1jsoqfbo5s8++0w4OzuL8vLyZvWvKdu0hP4JIcTPP/8slEqlKCoq0lvepUsX8dZbb5ldfw1VUFAgVCqV3iDZyspK4e3trbvKw5L7d+3aNXH+/HndY9euXQKA2LJli0hLSxNC1Awirays1K23aNGiOoNI77vvPr1tR0ZG1hlEumLFCt372u/t7YNIT548qWuza9euFg8ivX79uggPDxcPP/ywqK6urvO+pfevOe70uyYHjUYj5s6dK/z9/cWlS5fqvG+szzNj/FtsjsLCQr1/a+fPnxcDBgwQjz32mDh//rxF9q9NBpRr166JM2fOiNdff104OjqKM2fOiDNnzuj+yJ0/f154eXmJxx57TGRkZOge2dnZum1oL6V64YUXREJCgli9enW9l1KpVCqxfv16ER8fL+bMmSNcXV31RkD/4x//EEFBQWLfvn3i5MmTIjIyUkRGRure117Wde+994qzZ8+KnTt3Ci8vr0Yvw71T/5qyTXPuX203b94UHh4e4oEHHhBnz54ViYmJ4vnnnxc2Njbi7NmzZtff5pg/f74ICAgQu3btEhcvXhSPP/648Pb2Frm5uW2if7UlJyfXuYonPz9f+Pj4iBkzZoi4uDixceNG4eDgUOcyXGtra7FixQqRkJAgli5dWu9luK6uruKnn34Sv//+u5gyZUq9l+H269dPHDt2TPz2228iPDy8RZfhXr9+XYSFhYkxY8aI69ev632etIX+NVdTftda25NPPilcXFzEgQMH9H5OpaWlujbG+Dwz1r9FY6h9FY8l9q9NBpTo6GgBoM5j//79Qgghli5dWu/7nTp10tvO/v37Rd++fYWtra0IDQ0V69atq7OvVatWiaCgIGFraysGDRokjh49qvd+WVmZeOqpp4Sbm5twcHAQf/rTn/Q+vIQQIiUlRUyYMEHY29sLT09PsXDhQlFVVdXs/jV1m+bav9udOHFC3HvvvcLd3V04OTmJIUOGiB07dhi8j9bqr6EqKyvFwoULhbe3t3BychJRUVEiLi6uzfSvtvoCihBCnDt3TgwfPlyoVCoREBAg3n777Trrfvfdd6JLly7C1tZW9OzZU2zfvl3vfY1GIxYvXix8fHyESqUSY8aMEYmJiXptbt26JaZPny4cHR2Fs7OzmD17dp2jc4ZYt25dvf8Wbz84ban9a4k7/a61toZ+TrX/nRjr88wY/xaN4faAYmn9UwghhGEnhYiIiIhMy2Ku4iEiIqL2gwGFiIiIzA4DChEREZkdBhQiIiIyOwwoREREZHYYUIiIiMjsMKAQERGR2WFAISJZpaSkQKFQ4OzZsy3aTnBwMD744ING2ygUCmzdurVF+yGi1sGAQkR67nQX32XLlsldIhG1AxZzN2Miah3aOywDwKZNm7BkyRIkJibqljk6OupeCyGgVqthbc2PEiIyLh5BISI9vr6+uoeLiwsUCoXu64sXL8LJyQm//PILIiIioFKp8Ntvv0Gj0SAmJgYhISGwt7dHnz59sGXLFt028/Ly8Oijj8LLywv29vYIDw/HunXr9PZ79epVjB49Gg4ODujTpw9iY2P13v/+++/Rs2dPqFQqBAcH47333mu0H5cvX8bIkSNhZ2eHHj16YPfu3cb7JhGRyfF/e4jIYC+//DJWrFiB0NBQuLm5ISYmBv/973/x6aefIjw8HAcPHsRjjz0GLy8vjBo1CosXL0Z8fDx++eUXeHp64sqVKygrK9Pb5quvvooVK1YgPDwcr776KqZPn44rV67A2toap06dwkMPPYRly5Zh2rRpOHLkCJ566il4eHhg1qxZderTaDR44IEH4OPjg2PHjqGgoADPPvts63xziMgoGFCIyGBvvPEGxo4dCwCoqKjA8uXLsWfPHkRGRgIAQkND8dtvv+Gzzz7DqFGjkJqain79+mHAgAEApAGtt3v++ecxadIkAMDrr7+Onj174sqVK+jWrRtWrlyJMWPGYPHixQCALl26ID4+Hu+++269AWXPnj24ePEidu3aBX9/fwDA8uXLMWHCBGN/K4jIRHiKh4gMpg0aAHDlyhWUlpZi7NixcHR01D2+/PJLJCUlAQCefPJJbNy4EX379sWLL76II0eO1NnmXXfdpXvt5+cHAMjOzgYAJCQkYNiwYXrthw0bhsuXL0OtVtfZVkJCAgIDA3XhBIAuPBGRZeARFCIyWIcOHXSvi4uLAQDbt29HQECAXjuVSgUAmDBhAq5du4YdO3Zg9+7dGDNmDObOnYsVK1bo2trY2OheKxQKANKpGiJqnxhQiKhFevToAZVKhdTUVIwaNarBdl5eXoiOjkZ0dDRGjBiBF154QS+gNKZ79+44fPiw3rLDhw+jS5cusLKyqrd9WloaMjIydEdjjh49akCviEhuDChE1CJOTk54/vnn8dxzz0Gj0WD48OEoKCjA4cOH4ezsjOjoaCxZsgQRERHo2bMnKioqsG3bNnTv3r3J+1i4cCEGDhyIN998E9OmTUNsbCw++ugjfPzxx/W2j4qKQpcuXRAdHY13330XhYWFePXVV43VZSJqBQwoRNRib775Jry8vBATE4OrV6/C1dUV/fv3xyuvvAIAsLW1xaJFi5CSkgJ7e3uMGDECGzdubPL2+/fvj++++w5LlizBm2++CT8/P7zxxhv1DpAFAKVSiR9//BGPP/44Bg0ahODgYHz44YcYP368MbpLRK1AIYQQchdBREREVBuv4iEiIiKzw4BCREREZocBhYiIiMwOAwoRERGZHQYUIiIiMjsMKERERGR2GFCIiIjI7DCgEBERkdlhQCEiIiKzw4BCREREZocBhYiIiMwOAwoRERGZnf8HsrID61+WjZYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "precisions, recalls, thresholds = precision_recall_curve(y_train_3, y_scores)\n",
    "\n",
    "\n",
    "def plot_precision_recall_treshold(precisions, recalls, thresholds):\n",
    "    plt.plot(thresholds, precisions[:-1], \"b--\", label = \"Precision\")\n",
    "    plt.plot(thresholds, recalls[:-1], \"g-\", label = \"Recall\")\n",
    "    plt.xlabel(\"Treshold\")\n",
    "    plt.legend(loc = \"upper left\")\n",
    "    plt.ylim([0,1])\n",
    "    plt.show()\n",
    "    \n",
    "    \n",
    "plot_precision_recall_treshold(precisions, recalls, thresholds)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "68ba7d09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi4AAAGiCAYAAADA0E3hAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAyBklEQVR4nO3de3wU9b3/8XcSyAaQBDASLgYjWAW5KpcY0IelRnOqJ8rxWFO0gKioQL2lKkTEKCggWmpVChW1YgsFodJjhQNqlJ9V8VAC9KhcvACCHBOISAJBEpLM749vkyESIBt297uz+3o+HvuYz8zO5RMGzNu5xjiO4wgAAMADYm03AAAA0FgEFwAA4BkEFwAA4BkEFwAA4BkEFwAA4BkEFwAA4BkEFwAA4BkEFwAA4BkEFwAA4BkEFwAA4Bl+B5f33ntP2dnZ6tSpk2JiYvTXv/71pMusXr1aF154oXw+n8455xy9/PLLTWgVAABEO7+DS3l5ufr27avZs2c3av7t27frqquu0tChQ7Vx40bdc889uvXWW7Vq1Sq/mwUAANEt5lReshgTE6Nly5Zp2LBhx51nwoQJWr58uT755JO6aT//+c+1f/9+rVy5sqmbBgAAUahZsDewZs0aZWZm1puWlZWle+6557jLVFRUqKKiom68pqZG+/bt0+mnn66YmJhgtQoAAALIcRwdOHBAnTp1UmxsYC6rDXpwKSoqUkpKSr1pKSkpKisr0/fff68WLVocs8z06dP16KOPBrs1AAAQArt27dKZZ54ZkHUFPbg0RV5ennJzc+vGS0tL1aVLF+3atUuJiYknXX7dOumyyxq/vbQ0ac8e6Y47pAcekBrIUvWUl0v790udOkkcAAIAoGFlZWVKTU1V69atA7bOoAeXDh06qLi4uN604uJiJSYmNni0RZJ8Pp98Pt8x0xMTExsVXNLTpd69pb17pW7dzKdLF6l9e/M54wzpr3+Vnn3WzL9jhxnOmmU+l18uPfig9MUX0vbt0u7d5vN//2eGpaXutiZPlkaNklJTpW++kQ4ckM4/X2rsEbGKCunbb6WSEvfTpo2UmWnW9d130r59kuNIF1zQ+PUCABAuAnmZR9CDS0ZGhlasWFFv2ltvvaWMjIygbbNVK+l///dkfUkXXSTFxUnV1dKNNx7dn/k0xtSp5vNDo0ZJ48cfG3r27KkfUg4caPzPVWv9eqmqygSa2mDzw+Hnn0uXXCJ17y4NHWrCVlmZOzy6/uG00lJp2zbphhtMfwcPSps2mfW0bWvGDx6U/vlP6aGHpLPPdqfVfkpLpSFDzNGp2FipRw9T134OHjTD3r3Nn395uXTokDvcv98Ezh//WDpyxJ3e0Ke01PycMTHS6adL553nHglzHI6KAUAk8fuuooMHD+qLL76QJF1wwQWaNWuWhg4dqnbt2qlLly7Ky8vT7t279corr0gyt0P36tVL48eP180336x33nlHd911l5YvX66srKxGbbOsrExJSUkqLS1t1BGXptixQ9q5U7rySnMU5OyzpXPOkbp2lc4805wW6tzZDJOSpIcflv7rv0z4OFVxceYXbnKyCQhH8/lMP/BPu3Ym1Bw+bMavuUaqrDQB6JNPpH//d+n7783n0CG3rh0/dMgcCbvwQlNv2WKOxNXOU1goDRwo9elj/l6ccYY5qnfwoPn70bq12VbnziY41S53+LAZ7t9vTlHW/t1q2dLmnxYABEcwfn/7HVxWr16toUOHHjN91KhRevnll3XTTTdpx44dWr16db1l7r33Xm3atElnnnmmJk+erJtuuqnR2wxFcKlVXW3+L71ZI45FOY706afm6EfnzlJCgjRvnvSrX5nva4NO585u3aGD+SWXnOx+kpLcU0A1NeYU1WmnmaMbLVqYaYWF5qjQ55+737Vrd+ywXTsTpv7+d2ntWhOKkpOlxESznYaGP5z24Yfm52/d2nyKiqTNm6WOHc22W7c28yxbZk6RnXZa/c/f/mZ+3rZtTUiQzPRWrczntNPcI2Jt2phpLVu6w4MHjz1iFhtrvvvhZ+tWc1SoUydzVMvLWrUyIbVPHxNuNm82R83++U9zdPC778w8bdqY06E+n9Szp9kvCQnSj35k/k5WVLhBqaLC/J07fNh82rc3fycAIBTCIrjYEMrgEijBOkVRU+O961ya+mdRe3SiZUspPr5x6ygulr7+2g02q1eb4NW6tRlft84ExxYtzKdly4brr782fbdoYQLEaaeZOiHBXPdUUGB62rnThKzevU1///u/JswdOGCOqrRu7a4zIcEMP/vMHMUJhpgY0/fJpKW5IXHnTnMasGdP6c03pRdekM46y/1zAoCmIrh4KLgAJ1P7L+9//scEtC+/dI+yFReb7/ftM99VVZkjcWvXmlNQ331ngl18vDkFFky1R3g6dzYBp/ZUWlmZOcp16aWmxw4dTJ/JyeYIkdcCNoDAI7gQXIAG7d1rjp7UHt1p0UJq3twEie+/N0d7vvtO2rDBDJOSzDz//d/mVFSvXuaOukC77DL3uqGtW0199dVu+Nm3z1wIXlkp3X676WncOHN9GRdVA95HcCG4ACFx+LD0+uvS4sXmtNdbb0kXX+yeTouPl5YsMaeTKivN/F27mtNqgXTFFeZIz+7d5uLnNm1MGDt40Gz7/vvNUSoA4YngQnABwt7Bg9Izz5g75U47zYSdmBhzdOf8893rj1q0MEdb3nvPnCb74R11jXXeeea6oiuvNNeAxcSY8dRU8318vDmNlZLChclAqBFcCC5AxNu/3zwcsrTUhJ+dO81RnzPOMHe7ff+91MiX0zeob19zIfKQIeaana5dG3/xNwD/EFwILgD+parKnMp6+WVzQfCRI+ZurwMHzG3jbdtKX33l3zoTEsyFyHFx5s6yc84x4Wb8eO6wApqC4EJwAdAEVVXSmjXSggXSP/5hnj7tr2HDpP/4D/O8nPPOM9MSE83dU9XV5mJoAPURXAguAAKo9kLfnTvNrd1FReZ01K5d0rvvNv7VH7XS0806e/SQbr7ZHK0BohnBheACIMS++848q+bjj81dTfv3N37Z2FhzwfBPf2pCTceO5qjNGWcEq1sgvBBcCC4ALDtyxH2C9b595nqYrVvN2+G3bzd3Sb3xxsnX06+feXbORReZdVZUmFd2cOcTIgnBheACwCO2bTN3R33+ubkde+lSc6fUyfToYd4DdvbZ5m4nwMsILgQXAB63caM0atSxLxI9kY4dzRvp77gjaG0BQRGM39+8TQQAQqhfP/OaBccxF/KWlppn09S+Sb0h33wjjR1rnjXTooV5ivDEieb0EhBtCC4AYElcnLmlOiHBPBTPccynstJcL/P735uLeY92+LC56+mJJ8xyXbpIDzxgXsAJRANOFQGAB5SXm9NLq1ZJb78tffBBw/MtXixlZpoLfQHbuMaF4AIAksxt2lOmSH//u1RYePz5zjrL3Lm0cKG5EwoIJa5xAQBIMq80+M1vpHXrzOmlyZMbnu+rr8xRmLg4KT/fv+fQAOGI4AIAEWDKFBNgamrM9S4LF0r33HPsPG3bmot8n3/eSpvAKeNUEQBEuHnzpNtua/i7VavMNTGcRkIwcKoIAOC3MWPcozF5efW/y8oyp5F+8hNOI8EbCC4AECViYqRp00yISU+v/92777qnkT76yE5/QGMQXAAgCn30kQkwCxce+11GhgkwL74Y+r6AkyG4AEAUGz7cBJj9+6Xevet/d+utJsDk55un/ALhgOACAFBSknnAneNIf/hD/e+mTJGaNzch5uOP7fQH1CK4AADquekmcyHvffcd+12fPibA9Oxp5jtwINTdIdoRXAAAx4iJkZ580hyB2bpVGjy4/vebNknz55t3Ld1/v3kQ3qFDdnpFdOE5LgCARqmqMiHl22+lP/7x+PMdPiz5fKHrC+GL57gAAKxp1sy8ZuCVV8yRmA8+kFq2PHa+hARzxGbJktD3iMhHcAEANMngweat1bUPtxs6tP7311/vnnL6+GMzH3CqCC4AgFMWEyO9845UViZNmlT/uwceMBf1xsaaozZFRXZ6RGQguAAAAqZ1a+mxx8z1MBMmHPt9dbXUsaN5Nsy+faHvD95HcAEABFxcnDRjhjk95DjmGTFHmzJFOv106de/ttMfvIvgAgAIut69TYCZPVs65xx3+n33mdNMU6eaO5W++85ej/AGbocGAITcH/8ojRzZ8Hc5OdLvf2+e5gtv43ZoAEBEGDHCPO/liivMU3hjYtzvFi+W2rQx0zZu5G4k1EdwAQBY4fNJq1ZJn3xibqdes+bYeS64wNyN1LattGNHyFtEGCK4AADCwkUXmaMrR45IaWn1v9u/Xzr7bGnuXBudIZwQXAAAYaVZM2n7dnPr9A+f+TJ2rPTqq3b6QngguAAAwlJsrJSSYo7CvP22Oz0nxzzoDtGJ4AIACHuXXSZNnuyOJyVJy5bZ6wf2EFwAAJ4wZYrUo4c7fu210pw59vqBHQQXAIBnbNokffSROz5uHKeNog3BBQDgKenp0nvvueNJSQ2/FwmRieACAPCcSy6RHnrIHZ850zyw7p57rLWEECG4AAA8aepU6Z//rD/tt7+VzjzTTj8IDYILAMCz+vSRKiul//5vd9ru3ebN1IhMBBcAgKc1by7927+Zdx/VysuTrr/eXk8IHoILACAi+HzS3//uji9Zwu3SkYjgAgCIGBdfbG6ZrjVunLnmhRc0Rg6CCwAgovToIRUWuuO7d5sXNB48aK8nBA7BBQAQcS680LykMSPDnda6tbRli72eEBgEFwBARIqNlT78UPrZz9xpPXpIK1bY6wmnjuACAIhor74q9e/vjl91FQ+q8zKCCwAg4q1bJ734ojv+299yu7RXEVwAAFHh5pul7dvd8SVLpKIie/2gaQguAICokZYmHTnijnfsaK0VNBHBBQAQVZo1k376U3c8K0tyHHv9wD8EFwBA1Pnb39z6zTelhQvt9QL/EFwAAFEnLk76+mt3/Be/kEpK7PWDxiO4AACiUufOUm6uO37GGfZ6QeMRXAAAUeupp6SzznLHly+31wsah+ACAIhaMTH1b5H+93+vf9cRwg/BBQAQ1WJipL/8xR2Pj7fXC06uScFl9uzZSktLU0JCgtLT07V27doTzv/000/rvPPOU4sWLZSamqp7771Xhw8fblLDAAAE2rXXmvcY1Tr67dIIL34Hl8WLFys3N1f5+flav369+vbtq6ysLO3Zs6fB+RcuXKiJEycqPz9fmzdv1osvvqjFixfrwQcfPOXmAQAIlA0b3HrAAHt94MT8Di6zZs3SmDFjNHr0aJ1//vmaO3euWrZsqZdeeqnB+T/88EMNGTJEN9xwg9LS0nTFFVdo+PDhJz1KAwBAKPl80vDh7vjRb5VG+PAruFRWVqqwsFCZmZnuCmJjlZmZqTVr1jS4zODBg1VYWFgXVLZt26YVK1boyiuvPO52KioqVFZWVu8DAECwLVjg1kuXSocO2esFDWvmz8wlJSWqrq5WSkpKvekpKSnasmVLg8vccMMNKikp0cUXXyzHcVRVVaU77rjjhKeKpk+frkcffdSf1gAAOGUxMVJVlXktgCS1asXrAMJN0O8qWr16taZNm6bf/e53Wr9+vV577TUtX75cU6dOPe4yeXl5Ki0trfvs2rUr2G0CACDJPFX36HcZLVpkrxccy68jLsnJyYqLi1NxcXG96cXFxerQoUODy0yePFkjRozQrbfeKknq3bu3ysvLddttt2nSpEmKjT02O/l8Pvl8Pn9aAwAgYF57TWrRwtTDh0vXXy818OsKFvi1G+Lj49W/f38VFBTUTaupqVFBQYEyMjIaXObQoUPHhJO4uDhJksPxNwBAGEpIkJ5/3h2Pi+OUUbjwOz/m5uZq3rx5mj9/vjZv3qyxY8eqvLxco0ePliSNHDlSeXl5dfNnZ2drzpw5WrRokbZv36633npLkydPVnZ2dl2AAQAg3IwZU388NlY6eNBOL3D5dapIknJycrR37149/PDDKioqUr9+/bRy5cq6C3Z37txZ7wjLQw89pJiYGD300EPavXu3zjjjDGVnZ+vxxx8P3E8BAEAQVFeboy21OneWSkvt9QMpxvHA+ZqysjIlJSWptLRUiYmJttsBAESZnBzp1VdNvXatNHCg3X68Ihi/v7nUCACAk/jzn9160CB7fYDgAgDAScXGSrNmueNvvWWvl2hHcAEAoBHuusutr7iCu4xsIbgAANAIcXHSffe545wysoPgAgBAI82c6dbr1klvvGGvl2hFcAEAoJFiYqSiInc8O9teL9GK4AIAgB9SUqQ//ckd/8tf7PUSjQguAAD46cYb3fq66+z1EY0ILgAANMHRgWX/fmttRB2CCwAATTB/vlu3bWuvj2hDcAEAoAlatpQuvtgd/+47e71EE4ILAABNtGqVWxcU2OsjmhBcAABoopYtpfh4U//sZ3Z7iRYEFwAATsHRT9PlNQDBR3ABAOAUTJjg1qNH2+sjWhBcAAA4BYmJ7umi+fM56hJsBBcAAE5RYaFbr15trY2oQHABAOAU9erl1sOGWWsjKhBcAAAIgDvuMMOyMrt9RDqCCwAAATB2rFtXVtrrI9IRXAAACICePd36N7+x10ekI7gAABAAcXFSu3amnjjRbi+RjOACAECA3H+/W+/eba+PSEZwAQAgQHJz3frMM+31EckILgAABEjtg+hqVVfb6SOSEVwAAAig779368REe31EKoILAAABlJAg9e1r6kOHeAVAoBFcAAAIsA8/dOsuXez1EYkILgAABFjLllJMjKm//lp69VW7/UQSggsAAEHw9ddunZPDKaNAIbgAABAEnTpJCxe648uW2eslkhBcAAAIkuHD3froIzBoOoILAABBlJNjhnffbbePSEFwAQAgiNq2td1BZCG4AAAQRLff7tabNtnrI1IQXAAACKI+fdz6P//TXh+RguACAEAQxcZKl19u6i1b7PYSCQguAAAE2RNPuPWhQ/b6iAQEFwAAgqxfP7d+/XVrbUQEggsAAEFW+/h/Sdq2zV4fkYDgAgBACFx5pRlOmmS3D68juAAAEAI/+pFbV1fb68PrCC4AAITAzJluzXuLmo7gAgBACMTHu/WBA/b68DqCCwAAITJsmBnefLPVNjyN4AIAQIjUPohO4vH/TUVwAQAgRMaNc+uePe314WUEFwAAQuimm9z6yy+tteFZBBcAAEJozhy3/uUv7fXhVQQXAABCKCFBGjrU1CtX2u3FiwguAACEWF6eW1dW2uvDiwguAACEWGamWz/2mL0+vIjgAgBAiB390sWpU+314UUEFwAALHjgATNs395uH15DcAEAwIJf/MIM9+yRHMduL15CcAEAwIKuXd36mWfs9eE1BBcAACxo1cqt77nHWhueQ3ABAMCSP/zBrWtq7PXhJQQXAAAsuf56t/7kE3t9eAnBBQAAS1q2dOu+fe314SUEFwAALLr/frc+eNBeH15BcAEAwKIZM9z63Xft9eEVBBcAACyKjTUfSZo1y24vXtCk4DJ79mylpaUpISFB6enpWrt27Qnn379/v8aPH6+OHTvK5/Pp3HPP1YoVK5rUMAAAkaZ7dzN8/327fXiB38Fl8eLFys3NVX5+vtavX6++ffsqKytLe/bsaXD+yspKXX755dqxY4eWLl2qrVu3at68eercufMpNw8AQCS46y4zrKqSNm+220u4i3Ec/x40nJ6eroEDB+q5556TJNXU1Cg1NVV33nmnJk6ceMz8c+fO1ZNPPqktW7aoefPmTWqyrKxMSUlJKi0tVWJiYpPWAQBAuDp8WGrRwtQzZ9a/YNfLgvH7268jLpWVlSosLFTmUe/jjo2NVWZmptasWdPgMq+//royMjI0fvx4paSkqFevXpo2bZqqq6uPu52KigqVlZXV+wAAEKkSEqTaX60LF9rtJdz5FVxKSkpUXV2tlJSUetNTUlJUVFTU4DLbtm3T0qVLVV1drRUrVmjy5Mn69a9/rccee+y425k+fbqSkpLqPqmpqf60CQCA55x+uhlu3Gi1jbAX9LuKampq1L59ez3//PPq37+/cnJyNGnSJM2dO/e4y+Tl5am0tLTus2vXrmC3CQCAVT//uVtv326vj3DXzJ+Zk5OTFRcXp+Li4nrTi4uL1aFDhwaX6dixo5o3b664uLi6aT169FBRUZEqKysVHx9/zDI+n08+n8+f1gAA8LTsbLfevl06+2x7vYQzv464xMfHq3///iooKKibVlNTo4KCAmVkZDS4zJAhQ/TFF1+o5qi3R3322Wfq2LFjg6EFAIBoFBcnDRpk6ilT7PYSzvw+VZSbm6t58+Zp/vz52rx5s8aOHavy8nKNHj1akjRy5Ejl5eXVzT927Fjt27dPd999tz777DMtX75c06ZN0/jx4wP3UwAAEAFq70VZt85uH+HMr1NFkpSTk6O9e/fq4YcfVlFRkfr166eVK1fWXbC7c+dOxca6eSg1NVWrVq3Svffeqz59+qhz5866++67NWHChMD9FAAARIC77pLGjZPKy213Er78fo6LDTzHBQAQDbZvl7p2NfUjj0j5+VbbOWXWn+MCAACC5+gLch95RAr/QwuhR3ABACCM/M//uPUHH9jrI1wRXAAACCO1dxZJXOvSEIILAABh5qKLzHDvXrt9hCOCCwAAYaZ1azMcMcJuH+GI4AIAQJhp2dKtKyrs9RGOCC4AAISZBQvc+uh3GIHgAgBA2GnVyq3/67/s9RGOCC4AAISh3//eDHmWS30EFwAAwtDVV7v1Ue82jnoEFwAAwlCHDm49Zoy9PsINwQUAgDD105+aYbdudvsIJwQXAADCVE6OGb79tt0+wgnBBQCAMHX0Sxdrauz1EU4ILgAAhKmj31u0bZu9PsIJwQUAgDCVkODWN9xgr49wQnABACCM9eplhv/4h90+wgXBBQCAMPb4425dWmqvj3BBcAEAIIxlZ7v1nDn2+ggXBBcAAMJYTIxbE1wILgAAhL2lS81w506posJuL7YRXAAACHPXXOPW77xjr49wQHABACDMNWvm1r/8pb0+wgHBBQAAD/jFL8ww2h9ER3ABAMADHnjArf/v/+z1YRvBBQAAD+jd260LC+31YRvBBQAAj+jZ0wwXLbLbh00EFwAAPKKszAwXLrTbh00EFwAAPOK++2x3YB/BBQAAj7juOrf+9FN7fdhEcAEAwCM6dXLrP/3JXh82EVwAAPCQgQPN8Jln7PZhC8EFAAAPGTDADA8dstuHLQQXAAA8ZPRot66qsteHLQQXAAA85Pzz3fqrr+z1YQvBBQAAD2nVyq2j8ToXggsAAB4zeLAZElwAAEDYmzjRrb/80l4fNhBcAADwmOxst1692lobVhBcAADwoHbtzPCFF+z2EWoEFwAAPGjQIDP86CO7fYQawQUAAA+69lq3rqy010eoEVwAAPCgW25x63/8w14foUZwAQDAg2JjpZYtTT1njt1eQongAgCARzVvboYLFtjtI5QILgAAeNTkyW4dLS9dJLgAAOBRt9/u1n/8o70+QongAgCAR512mluXltrrI5QILgAAeNjPfmaG+/bZ7SNUCC4AAHhY+/Zm+MQTdvsIFYILAAAedvrptjsILYILAAAedvfdbn322fb6CBWCCwAAHlb7skVJ2rFDchxrrYQEwQUAAI8rKXHrggJ7fYQCwQUAAI87/XSpa1dT/+pXdnsJNoILAAARYPRoM9y7124fwUZwAQAgAgwZYobffGO3j2AjuAAAEAHOPdet//53e30EG8EFAIAI0LmzW9eeNopEBBcAACLEZZeZ4Zdf2u0jmAguAABEiGefdevPP7fXRzARXAAAiBA9erj1ggX2+ggmggsAABHkkkvM8NFH7fYRLE0KLrNnz1ZaWpoSEhKUnp6utWvXNmq5RYsWKSYmRsOGDWvKZgEAwEn87GduXV5ur49g8Tu4LF68WLm5ucrPz9f69evVt29fZWVlac+ePSdcbseOHbrvvvt0SW0UBAAAAXfbbW799NPW2ggav4PLrFmzNGbMGI0ePVrnn3++5s6dq5YtW+qll1467jLV1dW68cYb9eijj6pr7TOJT6CiokJlZWX1PgAA4OR8PqlNG1O//bbVVoLCr+BSWVmpwsJCZWZmuiuIjVVmZqbWrFlz3OWmTJmi9u3b65ZbbmnUdqZPn66kpKS6T2pqqj9tAgAQ1caMMcPVq622ERR+BZeSkhJVV1crJSWl3vSUlBQVFRU1uMz777+vF198UfPmzWv0dvLy8lRaWlr32bVrlz9tAgAQ1a6/3q3XrbPXRzA0C+bKDxw4oBEjRmjevHlKTk5u9HI+n08+ny+InQEAELkGDHDryy6TSkvt9RJofgWX5ORkxcXFqbi4uN704uJidejQ4Zj5v/zyS+3YsUPZ2dl102pqasyGmzXT1q1b1a1bt6b0DQAATuCyy6SCAinSLhP161RRfHy8+vfvr4KCgrppNTU1KigoUEZGxjHzd+/eXR9//LE2btxY97n66qs1dOhQbdy4kWtXAAAIktmz3fpfxwwigt+ninJzczVq1CgNGDBAgwYN0tNPP63y8nKN/tcbnUaOHKnOnTtr+vTpSkhIUK9eveot3+Zflzr/cDoAAAictDS3fuUV6aabbHUSWH4Hl5ycHO3du1cPP/ywioqK1K9fP61cubLugt2dO3cqNpYH8gIAYNPRl4q++27kBJcYx3Ec202cTFlZmZKSklRaWqrExETb7QAA4Am/+IV5Z1F2tvT666HffjB+f3NoBACACHX++Wb4//6f3T4CieACAECE6tTJDCPpziKCCwAAEeqnP3Xrffvs9RFIBBcAACLU0Q+6/8Ej2DyL4AIAQARr3twMX3nFbh+BQnABACCCxcWZYfjfQ9w4BBcAACJYbq4ZPvGE3T4CheACAEAEa9HCdgeBRXABACCC3XyzWy9bZq+PQCG4AAAQwWqf5SJJ115rr49AIbgAABDhXnzRrXftstdHIBBcAACIcEefLpo1y14fgUBwAQAgCiQlmeGbb9rt41QRXAAAiAJjxpjhpk12+zhVBBcAAKLA5Ze79bff2uvjVBFcAACIAkOGuPVTT9nr41QRXAAAiAKtWrlHXWbMkMrK7PbTVAQXAACixCOPuHW3btbaOCUEFwAAosTgwe5Rl5ISu700FcEFAIAoMmeOWy9YYK+PpiK4AAAQRY4+RbRqlb0+morgAgBAlBk50gz/+Ee7fTQFwQUAgCgzYIBbey28xDiO49hu4mTKysqUlJSk0tJSJSYm2m4HAABPq6qSmjd3x4OVBILx+5sjLgAARJlmzaQpU9zxqip7vfiL4AIAQBSaMMGtvfQKAIILAABRKD5e8vlMPXeu3V78QXABACBKVVSY4dFP1A13BBcAAKLU5Mm2O/AfwQUAgCg1bpxbHzpkrw9/EFwAAIhSZ5zh1gQXAAAQ1uLi3Dojw14f/iC4AAAAffGF7Q4ah+ACAEAUe/VVt/bC81wILgAARLHrrnPrUaPs9dFYBBcAAKJYTIzUqpWpN22y20tjEFwAAIhyU6ea4fbtdvtoDIILAABRLj3draur7fXRGAQXAACi3KBBbv3EE/b6aAyCCwAAUa5ZM7f+05/s9dEYBBcAAKBf/coMN2+228fJEFwAAIDuuMOt9+yx18fJEFwAAIDOOcett2yx18fJEFwAAEA906fb7uD4CC4AAECS1LOnGa5cabePEyG4AAAASdKDD7r1tm32+jgRggsAAJAkXX21W3frZq+PEyG4AAAASdJpp9U/6vLVV/Z6OR6CCwAAqPP442595532+jgeggsAAKgnJcUM//Y3u300hOACAADquf9+t96/31obDSK4AACAesaNc+vf/MZeHw0huAAAgHpatHDr2bPt9dEQggsAADhG7UW6334r7d5tt5ejEVwAAMAx7rrLrc88014fP0RwAQAAxzjtNOk//9N2F8ciuAAAgAZNnerWFRX2+jgawQUAADSoe3e3fucde30cjeACAAAaFBMj9ehh6qOf7WITwQUAABxX7VN0P/1Uqq6224tEcAEAACfw8stu3ayZ5DjWWpFEcAEAACdw1ln17y4aNsxaK5IILgAA4CSWLnXr11+3e8qoScFl9uzZSktLU0JCgtLT07V27drjzjtv3jxdcsklatu2rdq2bavMzMwTzg8AAMLPJ5+49fr19vrwO7gsXrxYubm5ys/P1/r169W3b19lZWVpz549Dc6/evVqDR8+XO+++67WrFmj1NRUXXHFFdodTs8PBgAAJ9Szp1s/+aS9PmIcx7/LbNLT0zVw4EA999xzkqSamhqlpqbqzjvv1MSJE0+6fHV1tdq2bavnnntOI0eObHCeiooKVRz1pJuysjKlpqaqtLRUiYmJ/rQLAAAC5LLL3Oe5NCY9lJWVKSkpKaC/v/064lJZWanCwkJlZma6K4iNVWZmptasWdOodRw6dEhHjhxRu3btjjvP9OnTlZSUVPdJTU31p00AABAE//Efbm3rdJFfwaWkpETV1dVKqb2p+19SUlJUVFTUqHVMmDBBnTp1qhd+figvL0+lpaV1n127dvnTJgAACILbbnPr/v3t9BDSu4pmzJihRYsWadmyZUpISDjufD6fT4mJifU+AADArvh4aeFCd3zz5tD34FdwSU5OVlxcnIqLi+tNLy4uVocOHU647FNPPaUZM2bozTffVJ8+ffzvFAAAWDd8uFv/63LXkPIruMTHx6t///4qKCiom1ZTU6OCggJlZGQcd7mZM2dq6tSpWrlypQYMGND0bgEAgHV33mmGv/td6Lft96mi3NxczZs3T/Pnz9fmzZs1duxYlZeXa/To0ZKkkSNHKi8vr27+J554QpMnT9ZLL72ktLQ0FRUVqaioSAcPHgzcTwEAAELm5pvduqwstNtu5u8COTk52rt3rx5++GEVFRWpX79+WrlyZd0Fuzt37lRsrJuH5syZo8rKSl133XX11pOfn69HHnnk1LoHAAAh16+fW3/6qXSCky4B5/dzXGwIxn3gAACg6ZKSzNGW4cPrX7B7NOvPcQEAAJAkn88M//nP0G6X4AIAAPw2dqwZbtoU2u0SXAAAgN+OfgDdm2+GbrsEFwAA4LfsbLfOygrddgkuAADAbzEx0u9/744vWhSa7RJcAABAk/zrEW6S6j9RN5gILgAAoEmaN5f+8hd3fM+e4G+T4AIAAJrs2mvdeuvW4G+P4AIAAE5JaqoZvvJK8LdFcAEAAKfkzDPN8IUXgr8tggsAADglDzzg1h99FNxtEVwAAMApueYatx45MrjbIrgAAIBTEhMjjRhh6s8/l776KnjbIrgAAIBTNmuWW6elBW87BBcAAHDKkpOlCy5wx2tqgrMdggsAAAiIoy/MXbIkONsguAAAgICIj3frn/88ONsguAAAgIC5+263rqwM/PoJLgAAIGCmTnXrTz8N/PoJLgAAIGBat5bOOsvUCxYEfv0EFwAAEFC17y5atizw6ya4AACAgPrxj82wpCTw6ya4AACAgMrODt66CS4AACCgBg4M3roJLgAAIKBiYoK3boILAAAIuKFDg7NeggsAAAi4wYODs16CCwAACLjmzYOzXoILAAAIuLZtg7NeggsAAAi4yy+XxowJ/HoJLgAAIOB69JCeeirw6yW4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAzyC4AAAAz2hScJk9e7bS0tKUkJCg9PR0rV279oTzL1myRN27d1dCQoJ69+6tFStWNKlZAAAQ3fwOLosXL1Zubq7y8/O1fv169e3bV1lZWdqzZ0+D83/44YcaPny4brnlFm3YsEHDhg3TsGHD9Mknn5xy8wAAILrEOI7j+LNAenq6Bg4cqOeee06SVFNTo9TUVN15552aOHHiMfPn5OSovLxcb7zxRt20iy66SP369dPcuXMb3EZFRYUqKirqxktLS9WlSxft2rVLiYmJ/rQLAAAsKSsrU2pqqvbv36+kpKSArLOZPzNXVlaqsLBQeXl5ddNiY2OVmZmpNWvWNLjMmjVrlJubW29aVlaW/vrXvx53O9OnT9ejjz56zPTU1FR/2gUAAGHg22+/tRNcSkpKVF1drZSUlHrTU1JStGXLlgaXKSoqanD+oqKi424nLy+vXtjZv3+/zjrrLO3cuTNgPziapjY9c/TLPvZF+GBfhBf2R/ioPWPSrl27gK3Tr+ASKj6fTz6f75jpSUlJ/CUME4mJieyLMMG+CB/si/DC/ggfsbGBu4nZrzUlJycrLi5OxcXF9aYXFxerQ4cODS7ToUMHv+YHAAA4Hr+CS3x8vPr376+CgoK6aTU1NSooKFBGRkaDy2RkZNSbX5Leeuut484PAABwPH6fKsrNzdWoUaM0YMAADRo0SE8//bTKy8s1evRoSdLIkSPVuXNnTZ8+XZJ0991369JLL9Wvf/1rXXXVVVq0aJHWrVun559/vtHb9Pl8ys/Pb/D0EUKLfRE+2Bfhg30RXtgf4SMY+8Lv26El6bnnntOTTz6poqIi9evXT88884zS09MlST/+8Y+Vlpaml19+uW7+JUuW6KGHHtKOHTv0ox/9SDNnztSVV14ZsB8CAABEhyYFFwAAABt4VxEAAPAMggsAAPAMggsAAPAMggsAAPCMsAkus2fPVlpamhISEpSenq61a9eecP4lS5aoe/fuSkhIUO/evbVixYoQdRr5/NkX8+bN0yWXXKK2bduqbdu2yszMPOm+Q+P5+++i1qJFixQTE6Nhw4YFt8Eo4u++2L9/v8aPH6+OHTvK5/Pp3HPP5b9TAeLvvnj66ad13nnnqUWLFkpNTdW9996rw4cPh6jbyPXee+8pOztbnTp1UkxMzAnfQVhr9erVuvDCC+Xz+XTOOefUuwO50ZwwsGjRIic+Pt556aWXnE8//dQZM2aM06ZNG6e4uLjB+T/44AMnLi7OmTlzprNp0ybnoYcecpo3b+58/PHHIe488vi7L2644QZn9uzZzoYNG5zNmzc7N910k5OUlOR8/fXXIe488vi7L2pt377d6dy5s3PJJZc411xzTWiajXD+7ouKigpnwIABzpVXXum8//77zvbt253Vq1c7GzduDHHnkcfffbFgwQLH5/M5CxYscLZv3+6sWrXK6dixo3PvvfeGuPPIs2LFCmfSpEnOa6+95khyli1bdsL5t23b5rRs2dLJzc11Nm3a5Dz77LNOXFycs3LlSr+2GxbBZdCgQc748ePrxqurq51OnTo506dPb3D+66+/3rnqqqvqTUtPT3duv/32oPYZDfzdFz9UVVXltG7d2pk/f36wWowaTdkXVVVVzuDBg50XXnjBGTVqFMElQPzdF3PmzHG6du3qVFZWhqrFqOHvvhg/frzzk5/8pN603NxcZ8iQIUHtM9o0Jrg88MADTs+ePetNy8nJcbKysvzalvVTRZWVlSosLFRmZmbdtNjYWGVmZmrNmjUNLrNmzZp680tSVlbWcedH4zRlX/zQoUOHdOTIkYC+CTQaNXVfTJkyRe3bt9ctt9wSijajQlP2xeuvv66MjAyNHz9eKSkp6tWrl6ZNm6bq6upQtR2RmrIvBg8erMLCwrrTSdu2bdOKFSt4CKoFgfrdbf3t0CUlJaqurlZKSkq96SkpKdqyZUuDyxQVFTU4f1FRUdD6jAZN2Rc/NGHCBHXq1OmYv5zwT1P2xfvvv68XX3xRGzduDEGH0aMp+2Lbtm165513dOONN2rFihX64osvNG7cOB05ckT5+fmhaDsiNWVf3HDDDSopKdHFF18sx3FUVVWlO+64Qw8++GAoWsZRjve7u6ysTN9//71atGjRqPVYP+KCyDFjxgwtWrRIy5YtU0JCgu12osqBAwc0YsQIzZs3T8nJybbbiXo1NTVq3769nn/+efXv3185OTmaNGmS5s6da7u1qLN69WpNmzZNv/vd77R+/Xq99tprWr58uaZOnWq7NTSR9SMuycnJiouLU3Fxcb3pxcXF6tChQ4PLdOjQwa/50ThN2Re1nnrqKc2YMUNvv/22+vTpE8w2o4K/++LLL7/Ujh07lJ2dXTetpqZGktSsWTNt3bpV3bp1C27TEaop/y46duyo5s2bKy4urm5ajx49VFRUpMrKSsXHxwe150jVlH0xefJkjRgxQrfeeqskqXfv3iovL9dtt92mSZMmKTaW/38PleP97k5MTGz00RYpDI64xMfHq3///iooKKibVlNTo4KCAmVkZDS4TEZGRr35Jemtt9467vxonKbsC0maOXOmpk6dqpUrV2rAgAGhaDXi+bsvunfvro8//lgbN26s+1x99dUaOnSoNm7cqNTU1FC2H1Ga8u9iyJAh+uKLL+rCoyR99tln6tixI6HlFDRlXxw6dOiYcFIbKB1e1RdSAfvd7d91w8GxaNEix+fzOS+//LKzadMm57bbbnPatGnjFBUVOY7jOCNGjHAmTpxYN/8HH3zgNGvWzHnqqaeczZs3O/n5+dwOHSD+7osZM2Y48fHxztKlS51vvvmm7nPgwAFbP0LE8Hdf/BB3FQWOv/ti586dTuvWrZ1f/vKXztatW5033njDad++vfPYY4/Z+hEihr/7Ij8/32ndurXz5z//2dm2bZvz5ptvOt26dXOuv/56Wz9CxDhw4ICzYcMGZ8OGDY4kZ9asWc6GDRucr776ynEcx5k4caIzYsSIuvlrb4e+//77nc2bNzuzZ8/27u3QjuM4zz77rNOlSxcnPj7eGTRokPPRRx/VfXfppZc6o0aNqjf/q6++6px77rlOfHy807NnT2f58uUh7jhy+bMvzjrrLEfSMZ/8/PzQNx6B/P13cTSCS2D5uy8+/PBDJz093fH5fE7Xrl2dxx9/3Kmqqgpx15HJn31x5MgR55FHHnG6devmJCQkOKmpqc64ceOc7777LvSNR5h33323wf/+1/75jxo1yrn00kuPWaZfv35OfHy807VrV+cPf/iD39uNcRyOlQEAAG+wfo0LAABAYxFcAACAZxBcAACAZxBcAACAZxBcAACAZxBcAACAZxBcAACAZxBcAACAZxBcAACAZxBcAACAZxBcAACAZ/x/p3jy752OQtwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_recall(precisions, recalls):\n",
    "    plt.plot(recalls[:-1], precisions[:-1], \"b-\")\n",
    "    plt.ylim([0,1])\n",
    "    plt.xlim([0,1])\n",
    "    plt.show()\n",
    "    \n",
    "plot_precision_recall(precisions, recalls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2bb8e064",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "fpr, tpr, tresholds = roc_curve(y_train_3, y_scores)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7d289a47",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAG2CAYAAACTTOmSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABDg0lEQVR4nO3deXxTVf7/8XeSNkn3FkoXoFh2RRAEBMGFUau4DMI4o6iMIOM2CujAuIACFRdwZXQUdcQFdXRAHRe+gvATFBXEQVncgCKbrC2UQvc2bXJ+fxQilQJNaRrofT0fjzwmObn35pNembwf5557js0YYwQAAGBB9lAXAAAAECoEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkEIQAAYFkhDUJffPGFBgwYoObNm8tms+mDDz446j6LFi1S9+7d5XK51K5dO82YMSPodQIAgMYppEGouLhYXbt21bRp02q1/aZNm3TZZZfpvPPO06pVq/S3v/1NN954o+bPnx/kSgEAQGNkO14WXbXZbHr//fc1aNCgw25zzz33aM6cOfrxxx/9bVdffbX27dunefPmNUCVAACgMQkLdQGBWLp0qTIyMqq19e/fX3/7298Ou095ebnKy8v9r30+n/Ly8tS0aVPZbLZglQoAAOqRMUaFhYVq3ry57Pb6u6B1QgWh7OxsJScnV2tLTk5WQUGBSktLFRERccg+U6ZM0aRJkxqqRAAAEERbt25Vy5Yt6+14J1QQqotx48ZpzJgx/tf5+flq1aqVtm7dqtjY2BBWBgDHL5/PqKzSq4pKo7yScu3KL9e6XYUq8Xhls0k/7chXUoxblT6jbXtLVVxeqaycQp3UJFI+Y+QzVcfwGqOteaWKdjvk81W1e41kfEaVvuNiZMYxi3Y7ZJNkk002m6qe22z7/7f6c8mm/LIKGSO1T4qWw27T1rwStUuKVrQ7THZb1XHs+/ez22zyeL0qLPWqfXK0ZJPstqr37TabbDabdhWUqXVilFxhdv/n/fp+1et9xRVKaxLhbzugqjL9pu23T359UdN2B19d+bXtoO1s1T/Lv6Gpei/WHeav+cDfUDq4/qr24qJC9et+imJiYmpzWmrthApCKSkpysnJqdaWk5Oj2NjYGnuDJMnlcsnlch3SHhsbSxACcEIzxmh3Ubl2FZTL4/Vpx75ShdntqvRVPbfv/0Xx+oxKK7z6OadIv+QVa8ueEqUnRslnjLy+quP4jNG6nCK5w+1y2Gwq9nhrUUFx9Zd2t9bv8/1mG5vsrkiVGO1PCPubHYfereOw2+T1GTnsNqXGuWWMtH1fqXqelCC73aYwu00bdhfpzDZNVVbhVaQzTC3iI+Sw27SrsEwnp8TKbrfJ8ZsgUOLxqmVCRLVgcPD7Xp9RYrRLDnv1AHJgm9iIcDn2H9e2v+3gbdEwCgoKJNX/3/yECkJ9+vTR3Llzq7V98skn6tOnT4gqAoDqDoSTzbkl2r6vRJJU4TVau7NQW/eWKDHaqeW/7FVqXIS8PqPF63PVITlaxkhm//5Gkozk2/+86j0jY6RSj1d7ij1yh9tVVvHb0FF732/Lr7H9aMdsnRilts2ilRLn0o59ZeqWFi+H3abySp8So52KCHeoRXyEbAeFBbvdJqfDrhh3WFXA2N/usNkU6QpTRLiDUIGQCWkQKioq0vr16/2vN23apFWrVqlJkyZq1aqVxo0bp+3bt+v111+XJP31r3/Vs88+q7vvvlt/+ctf9Omnn+rtt9/WnDlzQvUVADRCPl9VmCksq9S+Eo++2bxXOQVl2rC7SEkx7v09KVWPpRv3KK/Yo5YJEcor9qikVj0p0rqcohqf19ZvA0uMK0xxkeHamV+mniclKNxh1+Y9xTq1eayiXGFy2GwykjyVPrVIiFCrJpFKjXP7ezcO9HQYIyXFuhRutyshKlzucIfC7DZCChqtkAahb7/9Vuedd57/9YGxPMOGDdOMGTO0c+dObdmyxf9+69atNWfOHI0ePVpPP/20WrZsqZdeekn9+/dv8NoBBJ/XZ1RUXqkKr0/7Sjwqq/Dplz1VPS1Oh13e/eNQ8ksrlFNQpoQop7y+qss8Pp/R9n1lKqvwKtoVpqUb9yg9MUol5ZX6eVeRkmJc8plfLwv59vfAFJZV1qnWbXtLa2xPbxqp1olRctjtyi0qV4uECHVpEad9JRU6OSVGYY6qy0HNYlzVxpjY7dXHlfjb9weauIhwRTkdio90yhnGIgFAXR038wg1lIKCAsXFxSk/P58xQkADM8ZoZ36Zftier615Jfp+W77iI8P1y54SFZdXKtIVpi/W7VaMK0xFnkodD//vlBjtUm5RuWLcYerduqkSo51q2yx6/1iUqnEtlT6j9KZRahbjUrjDrrQmEYp0nlAjD4DjXrB+v/mXCiAge4rKtTO/TCUer3YXlquwrEL5pRX+ga4/bM/XrsJyNY1yqtJnlJVdqDC7TRtzi49+8P0Kyw/fK5PWJEJb80rVPilaJ6fG7g8jdpVWVO5/P1IO/+Uem0rKKxUfGa5mMS6VVfh0UtNIOew2RTodinJVjVk5cJeP/aCBsMmxbnpaAAsgCAGoptLr08K1u/TFut2y22xal1Oo/23KU7jDpgpv/XbRdE2Ll90mnd8xSeWVPiVEOf0Dbk9JjVWYw6amUS6FOxijAiA4CEKABRhjlFNQrt2F5fIao0qvTzvzy7S7sFz5pRX636Y92ppXqvJKr3KLPDUe47chqGVChPJLKuRwVN1W3LlFnBKjXQqz27S7qFypcW6d2jxOYY6qO4paJkSoWbRLbZtFK8LpaIivDQBHRRACTkDGGJVV+PTjjnyt31WkH7bna8UvexXmsGnDrmKlxrlVXunT9n2lcobZ5akM/DZrm03qld5E3VrFy26zqV2zaJ3Ztqli3GGKdYcH4VsBQMMjCAHHAWOq7nwqq/DJU+nTjvxSfbI6R64wu1Zs2at9JRXKyilUSqxbO/PLjnq8g8fj/DYExbjD1DTKKYfdpl/2lKhneoJOTomVx+tTh6Rond0+Ua0Tq2a8BYDGjiAENKCVW/ZqZ36Z3l+5XZtyi7V9b6lKK2o374ykw4ag7q3i1eOkBLnCHGqfHK1Yd7hiI8IVZrcpxh2maHeYEiKdCncw+BcADkYQAurIGKN9JRVak12g8kqfKr1G2/aWKMxu0/fb8vXLnhJFuRz64udctUmM0s+7ajdp3oEZg2NcYWrVNFJnt0vU3hKP2jaLVlqTSKU3jVKk06Em0U5FO8Nkp+cGAOqMIAQcpKzCq817ilXq8Wrb3lJt3Vs1cd+6nELlFXu0ZP0exUWEK7vg6JenDvbbEHRuh2YqKK3Q709LVYv4CLVPjlHLhAi5wxlEDAANiSAESzkwU/HanQVam12oCq9Pn6zO0f825dX6GIe7lNUtLV5hdps27ynWmW2aqqCsUm0So9T9pAQZY9QyIUKpcRFqHl/zAsEAgIZHEEKjUFhWoTU7C1Va4VV5hVeL1+fqm817tTWvREXllYp1h6kgwKUT0ptGKrugTE2jXOrbtqn2FHvUMSVGTSKd6twiTumJkYp0hinWHcYcNwBwgiII4YRS4qnU5twS7S3x6PN1u/XiFxtrtd9vQ5DdJvlMVS9O+6RoucMdOqtdU53WMp4eGwCwEIIQjmvfbs7T7O926PN1u/XLnpJa7dO1ZZzs9qpVtNslReusdk3VrlmM4iPDmakYAFANQQjHjdyicm3JK9HkOWsU4XToy59zD7ttQmS4ij1edWkRp/NPTtKfe5+kuEgm+QMABIYghAbnqfTp9aWblZVdqB+252ttduFR9xnSu5XaNItWz5MSdFrLOHpzAAD1giCEeufzGZVX+pRX4tG3m/NUXuHT2uxCLcraVasVyJNjXcopKNcDA09VnzZN1T45pgGqBgBYEUEIdWKMUVZOoX7OKdKCNTlavaNAybFurcsp1K7C8lof54azW6tLizg1i3GpU2qsEqKcQawaAIDqCEKolfLKqgkGH/14rRat213jIp5Hmjk545Rk7Svx6NTmserTtqn6dUhiBXIAQMgRhFCjCq9PX6zbrXe+3aYFa3JU6TOH3bZFfITSEyPVvVXV4p1xEeHq1ipe7jC7wljbCgBwHCMIQZK0p6hcW/eW6sUvNmj5L3uVU1Dz5a0mUU6VV3g1+sIOuqJ7SzXhUhYA4ARGELKwdTmFmvjhj/p64+GXl0iIDNepzeM0pHcrXdw5hbu1AACNCkHIQlbvKNDXG/dow+4ivfm/LYfdrmNyjG47r60u6ZwqZxiXtgAAjRdBqBErKq/Uq4s3ada3W7Vtb+lhtzurXVNN/P2p6pjCbeoAAGshCDUyG3YX6ZY3lmv9Ee7gah7n1qkt4tS2WbRGX9herjDu3gIAWBNB6AS3Na9Eb3+7Vc98ul5JMa4a5/BpHufW6a0SdEu/NjqtZXzDFwkAwHGKIHSC8fmM3l2xTc8v2qAteSXyHnRb+4EQ5Ayz69LOKbrxnDY6tXksA5wBADgMgtAJYldBmYa+suyw63J1aRGn7q3ideM5bdQyIYLwAwBALRCEjnOfr9utYa8sq/G9Pm2a6m8Z7dWrdROCDwAAdUAQOg59tT5XN73+rYo93kPe69W6iZ6+uptS4yJCUBkAAI0LQeg44fUZLduUp2umf13j+4N7punhP3RmyQoAAOoRQeg4MOubLbrv/R8PWc/r1t+11S3ntlF8JMtYAAAQDAShEPH5jKZ/uVFTPl57yHtTruiia3q1CkFVAABYC0GogVV6fbpm+tf6ZvPeau0Ou00rxl+ouMjwEFUGAID1EIQa0HsrtmnM299VazupaaQmXX6qftcxKURVAQBgXQShBrApt1jnPbGoWpvDbtPy8RmM/wEAIIQIQkFUVuHV3e9+r9nf7fC3NYtxac6os5UU6w5hZQAAQCIIBU2px6su98+vdifYDWe31vjLTmHyQwAAjhMEoSD4+IedGvWflf4QdEnnFD03pDsBCACA4wxBqB5t2F2kC578vFrbv67rof6npoSoIgAAcCQEoXpgjNHzn2/QY/OyqrXPuvlM9W7TNERVAQCAoyEI1YO/v/2d3lu53f/60T920eAzmBARAIDjHUHoGO3YV+oPQe2TojV9aE+lJ0aFuCoAAFAbBKFjUOKp1DmPfeZ//fEd57AoKgAAJxCCUB3tK/Go2wOf+F+/f1tfQhAAACcYfrnrIK+4egi65dw2Or1VQggrAgAAdUGPUIByi8rV86EF/tf3XHyybv1d2xBWBAAA6oogFKCDQ9D0oT11YafkEFYDAACOBZfGAvD0gp/9z6/vm04IAgDgBEcQqqV9JR7989OqINT/1GTdf/mpIa4IAAAcK4JQLd0xc5W8PqP4yHA9P6RHqMsBAAD1gCBUC5tzi/X5ut2SpKvPaCW7ncVTAQBoDAhCtTDm7VWSJHe4XXf37xjaYgAAQL0hCB2FMUardxZIku7qfzK9QQAANCIEoaN4efEmlVX4JEl/7N4ixNUAAID6RBA6AmOMHpqzRpLUuUWs4iOdIa4IAADUJ4LQEcz8Zqv/+VODu4WuEAAAEBQEoSN4b8U2SVKY3aZ2STEhrgYAANQ3gtBhfLZ2l77ZvFeS9O8be4e4GgAAEAwEocP499e/+J+f2aZpCCsBAADBQhCqgTFGC9fukiSNPK9diKsBAADBQhCqwYerdvif39yvTQgrAQAAwUQQqsGB5TSSY12KdYeHuBoAABAsBKEafPzjTknSgwM7h7gSAAAQTASh39hTVO6fSbpnepMQVwMAAIIp5EFo2rRpSk9Pl9vtVu/evbVs2bIjbv/UU0+pY8eOioiIUFpamkaPHq2ysrJ6q2fZpjz/8yZRzCQNAEBjFtIgNGvWLI0ZM0aZmZlasWKFunbtqv79+2vXrl01bv/WW29p7NixyszM1Jo1a/Tyyy9r1qxZuvfee+utpo9+qLosdm6HZvV2TAAAcHwKaRCaOnWqbrrpJg0fPlydOnXSCy+8oMjISL3yyis1bv/VV1/prLPO0rXXXqv09HRddNFFuuaaa47aixSI/23cI0nqw9xBAAA0eiELQh6PR8uXL1dGRsavxdjtysjI0NKlS2vcp2/fvlq+fLk/+GzcuFFz587VpZdeetjPKS8vV0FBQbXH4ZR6vMot8kiSzmzD+CAAABq7sFB9cG5urrxer5KTk6u1Jycna+3atTXuc+211yo3N1dnn322jDGqrKzUX//61yNeGpsyZYomTZpUq5qWbf51fNBpLeNrtQ8AADhxhXywdCAWLVqkyZMn67nnntOKFSv03nvvac6cOXrwwQcPu8+4ceOUn5/vf2zduvWw22ZlV/UWtUuKlsNuq/f6AQDA8SVkPUKJiYlyOBzKycmp1p6Tk6OUlJQa95kwYYKuu+463XjjjZKkLl26qLi4WDfffLPuu+8+2e2H5jqXyyWXy1WrmhZlVU2k+DsGSgMAYAkh6xFyOp3q0aOHFi5c6G/z+XxauHCh+vTpU+M+JSUlh4Qdh8MhqWp9sGP17S9Vq80ncNs8AACWELIeIUkaM2aMhg0bpp49e6pXr1566qmnVFxcrOHDh0uShg4dqhYtWmjKlCmSpAEDBmjq1Kk6/fTT1bt3b61fv14TJkzQgAED/IGorgrLKuSprJpI8cJOyUfZGgAANAYhDUKDBw/W7t27NXHiRGVnZ6tbt26aN2+efwD1li1bqvUAjR8/XjabTePHj9f27dvVrFkzDRgwQA8//PAx17Jq6z7/8w7JMcd8PAAAcPyzmfq4pnQCKSgoUFxcnPLz8xUbG+tv/9vMlfpg1Q61aRalT//+u9AVCAAADnG43+9jdULdNRZMa3YWSpJ6t2YiRQAArIIgtF9ZpVeSdG77xBBXAgAAGgpBaL89+2eUbs/4IAAALIMgJKnEU6mi8kpJUrOY2s05BAAATnwEIUlb80olSdGuMMVFhIe4GgAA0FAIQpI25RZLkpJj6Q0CAMBKCEKS9pZUjQ/amV8W4koAAEBDIghJyi+tkCR1aREX4koAAEBDIghJKt4/UDqW8UEAAFgKQUjS5+uqVp0/jR4hAAAshSAkyR1etWBreBh/DgAArIRffkm/7Km6a6znSQkhrgQAADQkywchr88op6BcktSqSWSIqwEAAA3J8kHowB1jktQkyhnCSgAAQEOzfBDKK66aQ8gdbleYw/J/DgAALMXyv/w79lUtr1FW4QtxJQAAoKFZPggdmEPIyR1jAABYjuV//bMLqpbVOKddYogrAQAADc3yQejApbEkFlwFAMByLB+ENuWWSJJi3SyvAQCA1RCEcoskSSbEdQAAgIZn+SAU4axaXiMphktjAABYjeWD0IZdVctrtGkWFeJKAABAQ7N8EEqIrBobFOUMC3ElAACgoVk6CBljtLuoap2x1LiIEFcDAAAamqWDULHHqwpv1TDpxBjWGQMAwGosHYT27l9nzBVmVySXxgAAsBxLB6Fte6smU3Sy2CoAAJZk6QSwbW/VZIo+wyxCAABYkaWD0OY9VbfOJ8e5Q1wJAAAIBUsHoTB71ddvEslAaQAArMjSQSi/tEKSdEbrJiGuBAAAhIKlg9D32/ZJokcIAACrsnQQ8vqqBkk7wyz9ZwAAwLIsnQCKPV5JUqsmkSGuBAAAhIKlg9D6XUWSpKRYVp4HAMCKLBuEzEFzB8W4wkNYCQAACBXLBqGCskr/82Yx9AgBAGBFlg1ChftvnZekCKcjhJUAAIBQsWwQKq+sGigdH8llMQAArMqyQWhvcVWPEAuuAgBgXZZNARX75xDaVVge4koAAECoWDYIbcmrWnD19FbxoS0EAACEzDEFobKysvqqo8GF2W2SpB37SkNcCQAACJWAg5DP59ODDz6oFi1aKDo6Whs3bpQkTZgwQS+//HK9FxgsG3OreoTOSGfBVQAArCrgIPTQQw9pxowZeuyxx+R0/rpYaefOnfXSSy/Va3HBdGASxZ35J26vFgAAODYBB6HXX39dL774ooYMGSKH49f5d7p27aq1a9fWa3HBlJVTIEk6PS0+tIUAAICQCTgIbd++Xe3atTuk3efzqaKiooY9jk8HeoTyS0+cmgEAQP0KOAh16tRJX3755SHt7777rk4//fR6KaoheLw+SVKbZtEhrgQAAIRKWKA7TJw4UcOGDdP27dvl8/n03nvvKSsrS6+//ro++uijYNQYFKUVVWuNRbsD/hMAAIBGIuAeoYEDB+r//u//tGDBAkVFRWnixIlas2aN/u///k8XXnhhMGoMipLyqh6haBfrjAEAYFV16g4555xz9Mknn9R3LQ3KU1kVhFxhBCEAAKwq4B6hNm3aaM+ePYe079u3T23atKmXohpCaUXVoqusNQYAgHUFnAI2b94sr9d7SHt5ebm2b99eL0U1hNU7q26fDw8jCAEAYFW1vjQ2e/Zs//P58+crLi7O/9rr9WrhwoVKT0+v1+KCqWVChHaUSPtX2gAAABZU6yA0aNAgSZLNZtOwYcOqvRceHq709HQ9+eST9VpcMFV4fZIcio9wHnVbAADQONU6CPl8VYOLW7durW+++UaJiYlBK6oh5BSUy+6KlJNLYwAAWFbAd41t2rQpGHWETJiDa2MAAFhVnW6fLy4u1ueff64tW7bI4/FUe+/222+vl8IaSqw7PNQlAACAEAk4CK1cuVKXXnqpSkpKVFxcrCZNmig3N1eRkZFKSko64YJQpJN5hAAAsKqAB8iMHj1aAwYM0N69exUREaGvv/5av/zyi3r06KEnnngiGDUGVUQ4QQgAAKsKOAitWrVKf//732W32+VwOFReXq60tDQ99thjuvfee4NRY9CE2W2yc/88AACWFXAQCg8Pl91etVtSUpK2bNkiSYqLi9PWrVvrt7ogc9MbBACApQUchE4//XR98803kqR+/fpp4sSJevPNN/W3v/1NnTt3DriAadOmKT09XW63W71799ayZcuOuP2+ffs0YsQIpaamyuVyqUOHDpo7d27AnytJ4dwxBgCApQUchCZPnqzU1FRJ0sMPP6yEhATdeuut2r17t/71r38FdKxZs2ZpzJgxyszM1IoVK9S1a1f1799fu3btqnF7j8ejCy+8UJs3b9a7776rrKwsTZ8+XS1atAj0a0hifBAAAFZnM8aYUH147969dcYZZ+jZZ5+VVDVpY1pamkaNGqWxY8cesv0LL7ygxx9/XGvXrlV4eN1uey8oKFBcXJzS/va22rVspk///rtj+QoAAKABHPj9zs/PV2xsbL0dt96mVV6xYoV+//vf13p7j8ej5cuXKyMj49di7HZlZGRo6dKlNe4ze/Zs9enTRyNGjFBycrI6d+6syZMn17gI7AHl5eUqKCio9jgg3M6s0gAAWFlASWD+/Pm68847de+992rjxo2SpLVr12rQoEE644wz/Mtw1EZubq68Xq+Sk5OrtScnJys7O7vGfTZu3Kh3331XXq9Xc+fO1YQJE/Tkk0/qoYceOuznTJkyRXFxcf5HWlqa/z0Hd4wBAGBptQ5CL7/8si655BLNmDFDjz76qM4880z9+9//Vp8+fZSSkqIff/yxzoOWa8vn8ykpKUkvvviievToocGDB+u+++7TCy+8cNh9xo0bp/z8fP/j4DvbGCwNAIC11Xpm6aefflqPPvqo7rrrLv33v//VlVdeqeeee04//PCDWrZsGfAHJyYmyuFwKCcnp1p7Tk6OUlJSatwnNTVV4eHhcjh+HeR8yimnKDs7Wx6PR07noSvJu1wuuVyuGo+3q7A84LoBAEDjUeseoQ0bNujKK6+UJF1xxRUKCwvT448/XqcQJElOp1M9evTQwoUL/W0+n08LFy5Unz59atznrLPO0vr166tdglu3bp1SU1NrDEFHExfBOmMAAFhZrYNQaWmpIiMjJUk2m00ul8t/G31djRkzRtOnT9drr72mNWvW6NZbb1VxcbGGDx8uSRo6dKjGjRvn3/7WW29VXl6e7rjjDq1bt05z5szR5MmTNWLEiDp9fkqc+5jqBwAAJ7aAFl196aWXFB0dLUmqrKzUjBkzlJiYWG2bQBZdHTx4sHbv3q2JEycqOztb3bp107x58/wDqLds2eKfxVqS0tLSNH/+fI0ePVqnnXaaWrRooTvuuEP33HNPIF/Dz+ngrjEAAKys1vMIpaeny2Y78uBim83mv5vseHXwPEK/79lGzw3pEeqSAADAUQRrHqFa9wht3ry53j70eLFtb2moSwAAACFk6WtDpzavv0QJAABOPJYOQuGMEQIAwNIsnQQIQgAAWJulk0BesSfUJQAAgBCydBBqGhX4JIwAAKDxqFMQ2rBhg8aPH69rrrlGu3btkiR9/PHH+umnn+q1uGCLZWZpAAAsLeAg9Pnnn6tLly763//+p/fee09FRUWSpO+++06ZmZn1XmAwhbHoKgAAlhZwEBo7dqweeughffLJJ9XW9zr//PP19ddf12txwRZut/SVQQAALC/gJPDDDz/oD3/4wyHtSUlJys3NrZeiGgo9QgAAWFvAQSg+Pl47d+48pH3lypVq0aJFvRTVULhrDAAAaws4CF199dW65557lJ2dLZvNJp/PpyVLlujOO+/U0KFDg1Fj0KTGRYS6BAAAEEIBB6HJkyfr5JNPVlpamoqKitSpUyede+656tu3r8aPHx+MGoOGS2MAAFhbrRddPcDpdGr69OmaMGGCfvzxRxUVFen0009X+/btg1FfUIXZCUIAAFhZwEFo8eLFOvvss9WqVSu1atUqGDU1GAdBCAAASwv40tj555+v1q1b695779Xq1auDUVODsdsIQgAAWFnAQWjHjh36+9//rs8//1ydO3dWt27d9Pjjj2vbtm3BqC+ouDQGAIC1BRyEEhMTNXLkSC1ZskQbNmzQlVdeqddee03p6ek6//zzg1Fj0NgJQgAAWNoxTa3cunVrjR07Vo888oi6dOmizz//vL7qahAOLo0BAGBpdQ5CS5Ys0W233abU1FRde+216ty5s+bMmVOftQWdK5wlNgAAsLKA7xobN26cZs6cqR07dujCCy/U008/rYEDByoyMjIY9QVVGGuNAQBgaQEHoS+++EJ33XWXrrrqKiUmJgajpgbD7fMAAFhbwEFoyZIlwagjJAhCAABYW62C0OzZs3XJJZcoPDxcs2fPPuK2l19+eb0U1hC4fR4AAGurVRAaNGiQsrOzlZSUpEGDBh12O5vNJq/XW1+1BR09QgAAWFutgpDP56vx+YmOIAQAgLUFfNvU66+/rvLy8kPaPR6PXn/99XopCgAAoCEEHISGDx+u/Pz8Q9oLCws1fPjweimqoUQ6HaEuAQAAhFDAQcgYI1sNMzJv27ZNcXFx9VJUQ2HRVQAArK3Wt8+ffvrpstlsstlsuuCCCxQW9uuuXq9XmzZt0sUXXxyUIoOFtcYAALC2WgehA3eLrVq1Sv3791d0dLT/PafTqfT0dP3xj3+s9wKDiRwEAIC11ToIZWZmSpLS09M1ePBgud3uoBXVULg0BgCAtQU8s/SwYcOCUUdIEIQAALC2WgWhJk2aaN26dUpMTFRCQkKNg6UPyMvLq7figo1LYwAAWFutgtA//vEPxcTE+J8fKQidSJhQEQAAa6tVEDr4ctj1118frFoaXGMJdAAAoG4CnkdoxYoV+uGHH/yvP/zwQw0aNEj33nuvPB5PvRYXbHQIAQBgbQEHoVtuuUXr1q2TJG3cuFGDBw9WZGSk3nnnHd199931XmAwcWkMAABrCzgIrVu3Tt26dZMkvfPOO+rXr5/eeustzZgxQ//973/ru76gIggBAGBtdVpi48AK9AsWLNCll14qSUpLS1Nubm79VhdkYfaAvz4AAGhEAk4CPXv21EMPPaQ33nhDn3/+uS677DJJ0qZNm5ScnFzvBQYTHUIAAFhbwEHoqaee0ooVKzRy5Ejdd999ateunSTp3XffVd++feu9wGCx2bhrDAAAqwt4ZunTTjut2l1jBzz++ONyOBz1UlRDYFZpAAAQcBA6YPny5VqzZo0kqVOnTurevXu9FdUQCEIAACDgILRr1y4NHjxYn3/+ueLj4yVJ+/bt03nnnaeZM2eqWbNm9V1jUJCDAABAwGOERo0apaKiIv3000/Ky8tTXl6efvzxRxUUFOj2228PRo1B4eCGMQAALC/gHqF58+ZpwYIFOuWUU/xtnTp10rRp03TRRRfVa3HBxKUxAAAQcL+Iz+dTeHj4Ie3h4eH++YVOBNw6DwAAAg5C559/vu644w7t2LHD37Z9+3aNHj1aF1xwQb0WF0z0CAEAgICD0LPPPquCggKlp6erbdu2atu2rVq3bq2CggI988wzwagxKPJLK0NdAgAACLGAxwilpaVpxYoVWrhwof/2+VNOOUUZGRn1XhwAAEAwBRSEZs2apdmzZ8vj8eiCCy7QqFGjglVX0CXHukJdAgAACLFaB6Hnn39eI0aMUPv27RUREaH33ntPGzZs0OOPPx7M+oKGMUIAAKDWY4SeffZZZWZmKisrS6tWrdJrr72m5557Lpi1BRULzwMAgFrHgY0bN2rYsGH+19dee60qKyu1c+fOoBQWbA56hAAAsLxaB6Hy8nJFRUX9uqPdLqfTqdLS0qAUFmxcGgMAAAENlp4wYYIiIyP9rz0ejx5++GHFxcX526ZOnVp/1QWRnRkVAQCwvFoHoXPPPVdZWVnV2vr27auNGzf6X9tOoF4WLo0BAIBaB6FFixYFsYyGRw4CAACWvXeKS2MAAMCyQSi3sDzUJQAAgBCzbBBy0CMEAIDlWTYIJce6Q10CAAAIseMiCE2bNk3p6elyu93q3bu3li1bVqv9Zs6cKZvNpkGDBgX8mcwjBAAA6hSEvvzyS/35z39Wnz59tH37dknSG2+8ocWLFwd8rFmzZmnMmDHKzMzUihUr1LVrV/Xv31+7du064n6bN2/WnXfeqXPOOacuX0HEIAAAEHAQ+u9//6v+/fsrIiJCK1euVHl51aDj/Px8TZ48OeACpk6dqptuuknDhw9Xp06d9MILLygyMlKvvPLKYffxer0aMmSIJk2apDZt2gT8mRI9QgAAoA5B6KGHHtILL7yg6dOnKzw83N9+1llnacWKFQEdy+PxaPny5crIyPi1ILtdGRkZWrp06WH3e+CBB5SUlKQbbrjhqJ9RXl6ugoKCag9JdAkBAIDAg1BWVpbOPffcQ9rj4uK0b9++gI6Vm5srr9er5OTkau3JycnKzs6ucZ/Fixfr5Zdf1vTp02v1GVOmTFFcXJz/kZaWJknipjEAABBwEEpJSdH69esPaV+8eHGdL1PVVmFhoa677jpNnz5diYmJtdpn3Lhxys/P9z+2bt0qSbLRJQQAgOUFtOiqJN10002644479Morr8hms2nHjh1aunSp7rzzTk2YMCGgYyUmJsrhcCgnJ6dae05OjlJSUg7ZfsOGDdq8ebMGDBjgb/P5fFVfJCxMWVlZatu2bbV9XC6XXC7XIceyHxf3ywEAgFAKOAiNHTtWPp9PF1xwgUpKSnTuuefK5XLpzjvv1KhRowI6ltPpVI8ePbRw4UL/LfA+n08LFy7UyJEjD9n+5JNP1g8//FCtbfz48SosLNTTTz/tv+xVG/QIAQCAgIOQzWbTfffdp7vuukvr169XUVGROnXqpOjo6DoVMGbMGA0bNkw9e/ZUr1699NRTT6m4uFjDhw+XJA0dOlQtWrTQlClT5Ha71blz52r7x8fHS9Ih7Uf/InUqFwAANCIBB6EDnE6nOnXqdMwFDB48WLt379bEiROVnZ2tbt26ad68ef4B1Fu2bJE9CNexuHseAADYjDEmkB3OO+882Y6QIj799NNjLiqYCgoKFBcXp2umLdRbt50f6nIAAEAtHPj9zs/PV2xsbL0dN+AeoW7dulV7XVFRoVWrVunHH3/UsGHD6quuoGNCRQAAEHAQ+sc//lFj+/3336+ioqJjLqihEIMAAEC9Db7585//fMRlMY43R7q8BwAArKHegtDSpUvldrvr63BBRw4CAAABXxq74oorqr02xmjnzp369ttvA55QMZTIQQAAIOAgFBcXV+213W5Xx44d9cADD+iiiy6qt8KCbf2uE2c8EwAACI6AgpDX69Xw4cPVpUsXJSQkBKumBnFKSv3degcAAE5MAY0RcjgcuuiiiwJeZf64xLUxAAAsL+DB0p07d9bGjRuDUUuDIgcBAICAg9BDDz2kO++8Ux999JF27typgoKCao8TBbfPAwCAWo8ReuCBB/T3v/9dl156qSTp8ssvrxYmjDGy2Wzyer31X2UQEIMAAECtg9CkSZP017/+VZ999lkw62k4JCEAACyv1kHowNqs/fr1C1oxDYkcBAAAAhoj1JjG1TSm7wIAAOomoHmEOnTocNQAkZeXd0wFAQAANJSAgtCkSZMOmVn6REV/EAAACCgIXX311UpKSgpWLQ2LJAQAgOXVeoxQYxtTYyMJAQBgebUOQgfuGmssGlmuAwAAdVDrS2M+ny+YdTQ4chAAAAh4iY3Ggh4hAABg3SBEnxAAAJZn2SAEAABg2SDEpTEAAEAQAgAAlmXZIMR9YwAAwLJBiB4hAABg3SAU6gIAAEDIWTcIkYQAALA86wYh+oQAALA8ywYhAAAAywYhLo0BAADrBqFQFwAAAELOukGILiEAACzPskEIAADAukGIDiEAACzPukEIAABYnmWD0C97ikNdAgAACDHLBqG2zaJDXQIAAAgxywYh7hoDAACWDUIAAACWDUL0BwEAAMsGIZIQAACwbhACAACWZ9kgZKNLCAAAy7NsEAIAALBsEKI/CAAAEIQAAIBlWTYIAQAAWDYIMbE0AACwbhDi4hgAAJZn2SAEAABg2SDEpTEAAGDZIAQAAGDZIESHEAAAsGwQ4toYAACwbhACAACWZ9kgRH8QAACwbhAiCQEAYHmWDUIAAACWDULMLA0AACwbhAAAACwbhBgjBAAAjosgNG3aNKWnp8vtdqt3795atmzZYbedPn26zjnnHCUkJCghIUEZGRlH3B4AAOBwQh6EZs2apTFjxigzM1MrVqxQ165d1b9/f+3atavG7RctWqRrrrlGn332mZYuXaq0tDRddNFF2r59ewNXDgAATnQ2Y4wJZQG9e/fWGWecoWeffVaS5PP5lJaWplGjRmns2LFH3d/r9SohIUHPPvushg4detTtCwoKFBcXp8dmr9BdA04/5voBAEDwHfj9zs/PV2xsbL0dN6Q9Qh6PR8uXL1dGRoa/zW63KyMjQ0uXLq3VMUpKSlRRUaEmTZrU+H55ebkKCgqqPSTGCAEAgBAHodzcXHm9XiUnJ1drT05OVnZ2dq2Occ8996h58+bVwtTBpkyZori4OP8jLS3tmOsGAACNQ8jHCB2LRx55RDNnztT7778vt9td4zbjxo1Tfn6+/7F169YGrhIAAByvwkL54YmJiXI4HMrJyanWnpOTo5SUlCPu+8QTT+iRRx7RggULdNpppx12O5fLJZfLVS/1AgCAxiWkPUJOp1M9evTQwoUL/W0+n08LFy5Unz59DrvfY489pgcffFDz5s1Tz5496/TZjBECAAAh7RGSpDFjxmjYsGHq2bOnevXqpaeeekrFxcUaPny4JGno0KFq0aKFpkyZIkl69NFHNXHiRL311ltKT0/3jyWKjo5WdHR0rT+XJTYAAEDIg9DgwYO1e/duTZw4UdnZ2erWrZvmzZvnH0C9ZcsW2e2/dlw9//zz8ng8+tOf/lTtOJmZmbr//vsbsnQAAHCCC/k8Qg3twDwEU+es1OhLu4W6HAAAUAuNch6hUOLCGAAAsGwQAgAAsGwQ4q4xAABg2SAEAABg2SDE7fMAAMC6QYgcBACA5Vk2CAEAABCEAACAZVk2CNm4NgYAgOVZNggVlFSEugQAABBilg1CznDLfnUAALCfZdNAlNMR6hIAAECIWTYIMUYIAABYNggBAABYNgjRIQQAACwbhAAAACwbhOgQAgAAlg1CAAAA1g1CDBICAMDyrBuEAACA5Vk2CNEfBAAALBuEAAAArBuE6BICAMDyrBuEAACA5Vk2CNnoEgIAwPIsG4QAAAAsG4ToDwIAAJYNQgAAAJYNQvQIAQAAywYhAAAAghAAALAsywYh1lwFAACWDUIAAACWDUJMqAgAACwbhAAAAKwbhOgQAgDA8qwbhAAAgOVZNgjRIQQAACwbhAAAACwbhGxMJAQAgOVZNggBAABYNgjRHwQAACwbhAAAACwbhBgiBAAALBuEAAAALBuE6BECAACWDUIAAACWDUKsPg8AACwbhAAAACwbhBgjBAAALBuEAAAACEIAAMCyCEIAAMCyLBuEWH0eAABYNggBAABYNgjRHwQAACwbhAAAACwbhBgiBAAALBuEAAAALBuEWGsMAABYNggBAABYNggxRggAAFg2CAEAABwXQWjatGlKT0+X2+1W7969tWzZsiNu/8477+jkk0+W2+1Wly5dNHfu3IA/kw4hAAAQ8iA0a9YsjRkzRpmZmVqxYoW6du2q/v37a9euXTVu/9VXX+maa67RDTfcoJUrV2rQoEEaNGiQfvzxxwauHAAAnOhsxhgTygJ69+6tM844Q88++6wkyefzKS0tTaNGjdLYsWMP2X7w4MEqLi7WRx995G8788wz1a1bN73wwgtH/byCggLFxcXp3aVZ+uOZHerviwAAgKA58Pudn5+v2NjYejtuWL0dqQ48Ho+WL1+ucePG+dvsdrsyMjK0dOnSGvdZunSpxowZU62tf//++uCDD2rcvry8XOXl5f7X+fn5kqSS4iIVFBQc4zcAAAAN4cBvdn3334Q0COXm5srr9So5Oblae3JystauXVvjPtnZ2TVun52dXeP2U6ZM0aRJkw5pH5rRo45VAwCAUNmzZ4/i4uLq7XghDUINYdy4cdV6kPbt26eTTjpJW7Zsqdc/JAJXUFCgtLQ0bd26tV67OVE3nI/jB+fi+MG5OH7k5+erVatWatKkSb0eN6RBKDExUQ6HQzk5OdXac3JylJKSUuM+KSkpAW3vcrnkcrkOaY+Li+M/6uNEbGws5+I4wvk4fnAujh+ci+OH3V6/93mF9K4xp9OpHj16aOHChf42n8+nhQsXqk+fPjXu06dPn2rbS9Inn3xy2O0BAAAOJ+SXxsaMGaNhw4apZ8+e6tWrl5566ikVFxdr+PDhkqShQ4eqRYsWmjJliiTpjjvuUL9+/fTkk0/qsssu08yZM/Xtt9/qxRdfDOXXAAAAJ6CQB6HBgwdr9+7dmjhxorKzs9WtWzfNmzfPPyB6y5Yt1brB+vbtq7feekvjx4/Xvffeq/bt2+uDDz5Q586da/V5LpdLmZmZNV4uQ8PiXBxfOB/HD87F8YNzcfwI1rkI+TxCAAAAoRLymaUBAABChSAEAAAsiyAEAAAsiyAEAAAsq1EGoWnTpik9PV1ut1u9e/fWsmXLjrj9O++8o5NPPllut1tdunTR3LlzG6jSxi+QczF9+nSdc845SkhIUEJCgjIyMo567hCYQP9tHDBz5kzZbDYNGjQouAVaSKDnYt++fRoxYoRSU1PlcrnUoUMH/r+qngR6Lp566il17NhRERERSktL0+jRo1VWVtZA1TZeX3zxhQYMGKDmzZvLZrMddg3Rgy1atEjdu3eXy+VSu3btNGPGjMA/2DQyM2fONE6n07zyyivmp59+MjfddJOJj483OTk5NW6/ZMkS43A4zGOPPWZWr15txo8fb8LDw80PP/zQwJU3PoGei2uvvdZMmzbNrFy50qxZs8Zcf/31Ji4uzmzbtq2BK2+cAj0fB2zatMm0aNHCnHPOOWbgwIENU2wjF+i5KC8vNz179jSXXnqpWbx4sdm0aZNZtGiRWbVqVQNX3vgEei7efPNN43K5zJtvvmk2bdpk5s+fb1JTU83o0aMbuPLGZ+7cuea+++4z7733npFk3n///SNuv3HjRhMZGWnGjBljVq9ebZ555hnjcDjMvHnzAvrcRheEevXqZUaMGOF/7fV6TfPmzc2UKVNq3P6qq64yl112WbW23r17m1tuuSWodVpBoOfityorK01MTIx57bXXglWipdTlfFRWVpq+ffual156yQwbNowgVE8CPRfPP/+8adOmjfF4PA1VomUEei5GjBhhzj///GptY8aMMWeddVZQ67Sa2gShu+++25x66qnV2gYPHmz69+8f0Gc1qktjHo9Hy5cvV0ZGhr/NbrcrIyNDS5curXGfpUuXVttekvr373/Y7VE7dTkXv1VSUqKKiop6X2DPiup6Ph544AElJSXphhtuaIgyLaEu52L27Nnq06ePRowYoeTkZHXu3FmTJ0+W1+ttqLIbpbqci759+2r58uX+y2cbN27U3LlzdemllzZIzfhVff1+h3xm6fqUm5srr9frn5X6gOTkZK1du7bGfbKzs2vcPjs7O2h1WkFdzsVv3XPPPWrevPkh/6EjcHU5H4sXL9bLL7+sVatWNUCF1lGXc7Fx40Z9+umnGjJkiObOnav169frtttuU0VFhTIzMxui7EapLufi2muvVW5urs4++2wZY1RZWam//vWvuvfeexuiZBzkcL/fBQUFKi0tVURERK2O06h6hNB4PPLII5o5c6bef/99ud3uUJdjOYWFhbruuus0ffp0JSYmhrocy/P5fEpKStKLL76oHj16aPDgwbrvvvv0wgsvhLo0y1m0aJEmT56s5557TitWrNB7772nOXPm6MEHHwx1aaijRtUjlJiYKIfDoZycnGrtOTk5SklJqXGflJSUgLZH7dTlXBzwxBNP6JFHHtGCBQt02mmnBbNMywj0fGzYsEGbN2/WgAED/G0+n0+SFBYWpqysLLVt2za4RTdSdfm3kZqaqvDwcDkcDn/bKaecouzsbHk8HjmdzqDW3FjV5VxMmDBB1113nW688UZJUpcuXVRcXKybb75Z9913X7W1MRFch/v9jo2NrXVvkNTIeoScTqd69OihhQsX+tt8Pp8WLlyoPn361LhPnz59qm0vSZ988slht0ft1OVcSNJjjz2mBx98UPPmzVPPnj0bolRLCPR8nHzyyfrhhx+0atUq/+Pyyy/Xeeedp1WrViktLa0hy29U6vJv46yzztL69ev9YVSS1q1bp9TUVELQMajLuSgpKTkk7BwIqIalOxtUvf1+BzaO+/g3c+ZM43K5zIwZM8zq1avNzTffbOLj4012drYxxpjrrrvOjB071r/9kiVLTFhYmHniiSfMmjVrTGZmJrfP15NAz8UjjzxinE6neffdd83OnTv9j8LCwlB9hUYl0PPxW9w1Vn8CPRdbtmwxMTExZuTIkSYrK8t89NFHJikpyTz00EOh+gqNRqDnIjMz08TExJj//Oc/ZuPGjeb//b//Z9q2bWuuuuqqUH2FRqOwsNCsXLnSrFy50kgyU6dONStXrjS//PKLMcaYsWPHmuuuu86//YHb5++66y6zZs0aM23aNG6fP+CZZ54xrVq1Mk6n0/Tq1ct8/fXX/vf69etnhg0bVm37t99+23To0ME4nU5z6qmnmjlz5jRwxY1XIOfipJNOMpIOeWRmZjZ84Y1UoP82DkYQql+BnouvvvrK9O7d27hcLtOmTRvz8MMPm8rKygauunEK5FxUVFSY+++/37Rt29a43W6TlpZmbrvtNrN3796GL7yR+eyzz2r8DTjw9x82bJjp16/fIft069bNOJ1O06ZNG/Pqq68G/Lk2Y+jLAwAA1tSoxggBAAAEgiAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEAAAsiyAEoJoZM2YoPj4+1GXUmc1m0wcffHDEba6//noNGjSoQeoBcHwjCAGN0PXXXy+bzXbIY/369aEuTTNmzPDXY7fb1bJlSw0fPly7du2ql+Pv3LlTl1xyiSRp8+bNstlsWrVqVbVtnn76ac2YMaNePu9w7r//fv/3dDgcSktL080336y8vLyAjkNoA4KrUa0+D+BXF198sV599dVqbc2aNQtRNdXFxsYqKytLPp9P3333nYYPH64dO3Zo/vz5x3zsw60afrC4uLhj/pzaOPXUU7VgwQJ5vV6tWbNGf/nLX5Sfn69Zs2Y1yOcDODp6hIBGyuVyKSUlpdrD4XBo6tSp6tKli6KiopSWlqbbbrtNRUVFhz3Od999p/POO08xMTGKjY1Vjx499O233/rfX7x4sc455xxFREQoLS1Nt99+u4qLi49Ym81mU0pKipo3b65LLrlEt99+uxYsWKDS0lL5fD498MADatmypVwul7p166Z58+b59/V4PBo5cqRSU1Pldrt10kknacqUKdWOfeDSWOvWrSVJp59+umw2m373u99Jqt7L8uKLL6p58+bVVnaXpIEDB+ovf/mL//WHH36o7t27y+12q02bNpo0aZIqKyuP+D3DwsKUkpKiFi1aKCMjQ1deeaU++eQT//ter1c33HCDWrdurYiICHXs2FFPP/20//37779fr732mj788EN/79KiRYskSVu3btVVV12l+Ph4NWnSRAMHDtTmzZuPWA+AQxGEAIux2+365z//qZ9++kmvvfaaPv30U919992H3X7IkCFq2bKlvvnmGy1fvlxjx45VeHi4JGnDhg26+OKL9cc//lHff/+9Zs2apcWLF2vkyJEB1RQRESGfz6fKyko9/fTTevLJJ/XEE0/o+++/V//+/XX55Zfr559/liT985//1OzZs/X2228rKytLb775ptLT02s87rJlyyRJCxYs0M6dO/Xee+8dss2VV16pPXv26LPPPvO35eXlad68eRoyZIgk6csvv9TQoUN1xx13aPXq1frXv/6lGTNm6OGHH671d9y8ebPmz58vp9Ppb/P5fGrZsqXeeecdrV69WhMnTtS9996rt99+W5J055136qqrrtLFF1+snTt3aufOnerbt68qKirUv39/xcTE6Msvv9SSJUsUHR2tiy++WB6Pp9Y1AZAa5erzgNUNGzbMOBwOExUV5X/86U9/qnHbd955xzRt2tT/+tVXXzVxcXH+1zExMWbGjBk17nvDDTeYm2++uVrbl19+aex2uyktLa1xn98ef926daZDhw6mZ8+exhhjmjdvbh5++OFq+5xxxhnmtttuM8YYM2rUKHP++ecbn89X4/Elmffff98YY8ymTZuMJLNy5cpq2wwbNswMHDjQ/3rgwIHmL3/5i//1v/71L9O8eXPj9XqNMcZccMEFZvLkydWO8cYbb5jU1NQaazDGmMzMTGO3201UVJRxu93+lbSnTp162H2MMWbEiBHmj3/842FrPfDZHTt2rPY3KC8vNxEREWb+/PlHPD6A6hgjBDRS5513np5//nn/66ioKElVvSNTpkzR2rVrVVBQoMrKSpWVlamkpESRkZGHHGfMmDG68cYb9cYbb/gv77Rt21ZS1WWz77//Xm+++aZ/e2OMfD6fNm3apFNOOaXG2vLz8xUdHS2fz6eysjKdffbZeumll1RQUKAdO3borLPOqrb9WWedpe+++05S1WWtCy+8UB07dtTFF1+s3//+97rooouO6W81ZMgQ3XTTTXruuefkcrn05ptv6uqrr5bdbvd/zyVLllTrAfJ6vUf8u0lSx44dNXv2bJWVlenf//63Vq1apVGjRlXbZtq0aXrllVe0ZcsWlZaWyuPxqFu3bkes97vvvtP69esVExNTrb2srEwbNmyow18AsC6CENBIRUVFqV27dtXaNm/erN///ve69dZb9fDDD6tJkyZavHixbrjhBnk8nhp/0O+//35de+21mjNnjj7++GNlZmZq5syZ+sMf/qCioiLdcsstuv322w/Zr1WrVoetLSYmRitWrJDdbldqaqoiIiIkSQUFBUf9Xt27d9emTZv08ccfa8GCBbrqqquUkZGhd99996j7Hs6AAQNkjNGcOXN0xhln6Msvv9Q//vEP//tFRUWaNGmSrrjiikP2dbvdhz2u0+n0n4NHHnlEl112mSZNmqQHH3xQkjRz5kzdeeedevLJJ9WnTx/FxMTo8ccf1//+978j1ltUVKQePXpUC6AHHC8D4oETBUEIsJDly5fL5/PpySef9Pd2HBiPciQdOnRQhw4dNHr0aF1zzTV69dVX9Yc//EHdu3fX6tWrDwlcR2O322vcJzY2Vs2bN9eSJUvUr18/f/uSJUvUq1evatsNHjxYgwcP1p/+9CddfPHFysvLU5MmTaod78B4HK/Xe8R63G63rrjiCr355ptav369OnbsqO7du/vf7969u7KysgL+nr81fvx4nX/++br11lv937Nv37667bbb/Nv8tkfH6XQeUn/37t01a9YsJSUlKTY29phqAqyOwdKAhbRr104VFRV65plntHHjRr3xxht64YUXDrt9aWmpRo4cqUWLFumXX37RkiVL9M033/gved1zzz366quvNHLkSK1atUo///yzPvzww4AHSx/srrvu0qOPPqpZs2YpKytLY8eO1apVq3THHXdIkqZOnar//Oc/Wrt2rdatW6d33nlHKSkpNU4CmZSUpIiICM2bN085OTnKz88/7OcOGTJEc+bM0SuvvOIfJH3AxIkT9frrr2vSpEn66aeftGbNGs2cOVPjx48P6Lv16dNHp512miZPnixJat++vb799lvNnz9f69at04QJE/TNN99U2yc9PV3ff/+9srKylJubq4qKCg0ZMkSJiYkaOHCgvvzyS23atEmLFi3S7bffrm3btgVUE2B5oR6kBKD+1TTA9oCpU6ea1NRUExERYfr3729ef/11I8ns3bvXGFN9MHN5ebm5+uqrTVpamnE6naZ58+Zm5MiR1QZCL1u2zFx44YUmOjraREVFmdNOO+2Qwc4H++1g6d/yer3m/vvvNy1atDDh4eGma9eu5uOPP/a//+KLL5pu3bqZqKgoExsbay644AKzYsUK//s6aLC0McZMnz7dpKWlGbvdbvr163fYv4/X6zWpqalGktmwYcMhdc2bN8/07dvXREREmNjYWNOrVy/z4osvHvZ7ZGZmmq5dux7S/p///Me4XC6zZcsWU1ZWZq6//noTFxdn4uPjza233mrGjh1bbb9du3b5/76SzGeffWaMMWbnzp1m6NChJjEx0bhcLtOmTRtz0003mfz8/MPWBOBQNmOMCW0UAwAACA0ujQEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMsiCAEAAMv6/5+M1TpFbs0/AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_roc_curve(fpr, tpr):\n",
    "    plt.plot(fpr, tpr)\n",
    "    plt.axis([0, 1, 0, 1])\n",
    "    plt.xlabel(\"False Positive Rate\")\n",
    "    plt.ylabel(\"True Positive Rate\")\n",
    "    \n",
    "    \n",
    "    \n",
    "plot_roc_curve(fpr, tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ffd4bcd6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9645720735278116"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "roc_auc_score(y_train_3, y_scores)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "770ab36e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAG2CAYAAACTTOmSAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAABHtUlEQVR4nO3deXxTVf7/8XeSNknXtFC6UIplERRBEBAGUBm1issgjDOKyleQcRdQYVxAgYoLuDI6ijrigjrOgDoujCD8BEUFcVAWNzbZZG2hFLq3aZP7+yMQKW2hqU0Cva/n45FHk5N7bz63F8ibc8+9x2IYhiEAAAATsoa7AAAAgHAhCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMKaxD64osvNHDgQLVs2VIWi0UffPDBMddZvHixunfvLofDofbt22vmzJlBrxMAADRNYQ1CJSUl6tq1q6ZPn16v5bds2aJLL71U5557rlavXq0777xTN9xwgxYsWBDkSgEAQFNkOV4mXbVYLHr//fc1ePDgOpe59957NXfuXP3444/+tquuukoHDhzQ/PnzQ1AlAABoSiLCXUAgli1bpqysrGptAwYM0J133lnnOhUVFaqoqPC/9nq9ys/PV/PmzWWxWIJVKgAAaESGYaioqEgtW7aU1dp4J7ROqCCUk5OjlJSUam0pKSkqLCxUWVmZoqKiaqwzdepUTZ48OVQlAgCAINq+fbtatWrVaNs7oYJQQ4wfP15jx471vy4oKFDr1q21fft2xcfHh7GyIDMMyV0sle6XyvKlsv1Sab7vebWf+6WyA1JVmVRZJlWV+356K8O9ByFikWSVLBbfQxbJcui19YjXR7bpsNeWI36q9nYdudzBXsljrm+tZZmjrVPbNuuopcayR2zH/77qsezh26/PsqplWdWx7DG2q6PVfqx9q2+9hy9b1/tH7pOqtx3xtHq75Sht9Wmvo5f7mNuo2eY1pEqvIY/XUKm7SgVlVcotrJC7yiOLxaqdB0oVF2WXxzC0v8StiipDOYVlah7tkPfg+obh+7mvpEJOu01er2RI8hi+/917Dr6u/Rfz63PD//Pwtprv17VMXZ9Rfb26tnHsz4622w6+sv76J/Gw363FcnibRSXuSnllUauEKFmtUm5hpVomOBQdGXHwTIXl4F95iyyyqNLrVWmFR60SoyWLRdZD27T4tnmgtFJpLqcibb5/JywWi6wW3/o6uExJWZWS452+bdby56FaW7U/4r/uh3H4S1lkkWQcrPHgRg7761vb9iy//u4Ovm+RFOuI9Nd86Pflr+ngflpkVUlJifqed6ni4uLUmE6oIJSamqrc3Nxqbbm5uYqPj6+1N0iSHA6HHA5Hjfb4+PgTKwhVuaXyAqn8gO9n2QGpdJ9UsvfgI08q2XPY872+UNMQkVK1f5AioqTIKCky+uBPp2RzSLZIyRpx8GfkwZ+2w54f+V7EYW0BvudfJlKyRdT+GdYIyWqV71+Qg49jBRvgBGYYhvYWV2hPYYXcHq92HShThNWqKq/vufXgn3GP11BZpUc/5xbrl/wSbdtXqsykGHkPhRHDkNcwtCG3WM5Iq2wWi0rcnsALskgqq6U9Ur7EcGTuslVfzGa1yOM1ZLNalOZyyjCknQfK1POkRFmtFkVYLdq0t1i/a9tc5ZUeRdsjlJ4QJZvVoj1F5TolNV5Wq0W2g1+qVovvS7XU7VGrxCj/a8sR73u8hpJiHbJZfw0Q1sOWiY+KlO3gdi0H2w5fFqFRWFgoqfF/5ydUEOrTp4/mzZtXre2TTz5Rnz59wlRRI/F6pL3rpfzNUsEOqWC773Fgu1SU4ws/laUN27bNIcUkSdHNpOjmNR9Rib6HPeaIsHPwZ4STwAAE4FA42ZpXqp0HfH9vKz2G1u0u0vb9pUqKtWvFL/uV5oqSx2toycY8dUiJlWH4soJhGL5eCEPyHnzue8+QYUhlbo/2lbjljLSqvNLb4Dq/31FQa/uxttkmKUbtWsQq1eXQrgPl6paRIJvVoooqr5Ji7YqKtCk9IUqWw8KC1WqR3WZVnDPCFzAOttssFkU7IhQVaSNUIGzCGoSKi4u1ceNG/+stW7Zo9erVatasmVq3bq3x48dr586deuONNyRJt9xyi5577jndc889+stf/qJPP/1Ub7/9tubOnRuuXQicYUhFu6Vdq6Ud3/geu1b5TmPVh8MlOV1SlMsXYGKSpdhkX9iJaXHYI0mKTvIFHP5xAQLi9frCTFF5lQ6UuvXN1v3KLSzXpr3FSo5zHuxJ8T2Wbd6n/BK3WiVGKb/ErdJ69qRsyC2u9Xl9HRlY4hwRckVHandBuXqelKhIm1Vb95XotJbxinFEyGbxndpwV3mVnhil1s2ileZy+ns3DvV0GIaUHO9QpNWqxJhIOSNtirAeeToFaDrCGoS+/fZbnXvuuf7Xh8byDB8+XDNnztTu3bu1bds2//tt2rTR3LlzNWbMGD3zzDNq1aqVXn75ZQ0YMCDktR/V3g3SnjVSaZ5Uss/388B2af9W36Oqlr5je6yU1EFKyJBchx6tpPiWvt4cp0tyxPtOPQEm4fEaKq6oUqXHqwOlbpVXevXLPl9Pi91mlcfwhZaCskrlFpYrMcYuj9d3msfrNbTzQLnKKz2KdURo2eZ9ykyKUWlFlX7eU6zkOMfBcSwHlz/YA1NUXtWgWnfsr+2ckJTZPFptkmJks1qVV1yh9MQodUl36UBppU5JjVOEzXc6qEWcQ5bDxpNYrZZDwzt0+DgT68FA44qKVIzdpoRou+wRTBIANNRxcx+hUCksLJTL5VJBQUHjjRHyeqW89dLaj6Sf3vOFoKOxWH2hp9WZvz5adCTkoMkzDEO7C8r1w84Cbc8v1fc7CpQQHalf9pWqpKJK0Y4IfbFhr+IcESp2V+l4+NcpKdahvOIKxTkj1LtNcyXF2tWuRezBsSi+cS1VXkOZzWPUIs6hSJtVGc2iFG0/oUYeAMe9oHx/6wQbI3Rc8XqlH96Wvvu3tHOVVHHY+XZrpNTyDN8pq+jmvh6d+HSpWRspsY2vtyfCHr7agd9gX3GFdheUq9Tt0d6iChWVV6qgrNI/0PWHnQXaU1Sh5jF2VXkNrc8pUoTVos15JfX+jKKKuntlMppFaXt+mU5OjtUpafEHw4hVZZVVB9+Pls1/usei0ooqJURHqkWcQ+WVXp3UPFo2q0XRdptiHL4xK76el18Hx9qsFqXEO+lpAUyAIBQIr0fat0nK+V76+nlp54pf34uIkjL7SaddLp1yiW/8DnACqvJ4tWjdHn2xYa+sFos25Bbpf1vyFWmzqNLTuF00XTMSZLVI53VMVkWVV4kxdv+A21PT4hVhs6h5jEORNsaoAAgOglB9lORJ37wiffuKVHzY5fv2WKnfnVKHAVJyJ99l3cBxyDAM5RZWaG9RhTyGoSqPV7sLyrW3qEIFZZX635Z92p5fpooqj/KK3bVu48gQ1CoxSgWllbLZfJcVd053KSnWoQirRXuLK5Tmcuq0li5F2HxXFLVKjFKLWIfatYhVlJ3TwACOD3xzH427RFo8VVr+8q8DnCOjpZTTpIzeUt/bpbiUo28DCALDMFRe6dWPuwq0cU+xfthZoJW/7FeEzaJNe0qU5nKqosqrnQfKZI+wyl0V+GXWFovUK7OZurVOkNViUfsWsfpdu+aKc0Yo3hkZhL0CgNAjCNVl3yZp9rXSnp98r9O6SX1HS6dexvgeNDrD8F35VF7plbvKq10FZfpkTa4cEVat3LZfB0ortT63SKnxTu0uOPaNMg8fj3NkCIpzRqh5jF02q0W/7CtVz8xEnZIaL7fHqw7JsTrr5CS1SYqVzcqpKABNH0GoNmv/K71/i+/ePjHJ0mXP+k5/MUYBv9Gqbfu1u6Bc76/aqS15Jdq5v0xllfW/g29dIah76wT1OClRjgibTk6JVbwzUvFRkYqwWhTnjFCsM0KJ0XbfLfgBAH4EoSOt/a/09nDJ8EgnnSX9aYbvXj7AEQzD0IHSSq3NKVRFlVdVHkM79pcqwmrR9zsK9Mu+UsU4bPri5zy1TYrRz3vqd9O8Q3cMjnNEqHXzaJ3VPkn7S91q1yJWGc2ildk8RtF2m5rF2hVrj5CVnhsAaDCC0OG2fyO9e70vBHW9Who0nXv7mEx5pUdb95WozO3Rjv1l2r7fd+O+DblFyi9xa+nGfXJFRSqnMLB53I4MQed0aKHCskr94fQ0pSdE6eSUOLVKjJIzkj9vABBKBKFDygukd0dIngqpw8WEoCbq0J2K1+0u1LqcIlV6vPpkTa7+tyW/3tuo61RWt4wERVgt2rqvRL9r21yF5VVqmxSj7iclyjAMtUqMUporSi0Tap8gGAAQegShQ756zjfRaWKmdPlLhKATTFF5pdbuLlJZpUcVlR4t2Zinb7bu1/b8UhVXVCneGaHCAKdOyGwerZzCcjWPcahvu+baV+JWx9Q4NYu2q3O6S5lJ0Yq2RyjeGcE9bgDgBEUQknyXyf/vRd/zCx6SnI136240rlJ3lbbmlWp/qVufb9irl77YXK/1jgxBVovkNXy9OCcnx8oZaVO/9s11eqsEemwAwEQIQpK0/mOpotDXG3TKH8JdDQ7z7dZ8zflulz7fsFe/7Cut1zpdW7lktfpm0W6fHKt+7ZurfYs4JURHcqdiAEA1BCFJWjHT97PLlZKVy4vDJa+4QtvySzVl7lpF2W368ue8OpdNjI5UidujLukunXdKsv6v90lyRXOTPwBAYAhCezdIW7+ULDapx/BwV2MK7iqv3li2VetzivTDzgKtyyk65jpDe7dW2xax6nlSok5v5aI3BwDQKAhCq9/y/Wx/vuRqFd5amgiv11BFlVf5pW59uzVfFZVercsp0uL1e+o1A3lKvEO5hRV6cNBp6tO2uU5OiQtB1QAAMzJ3ECrNl75+wfe8+7Dw1nKCMQxD63OL9HNusRauzdWaXYVKiXdqQ26R9hRV1Hs715/VRl3SXWoR51CntHglxjB9CQAgdMwdhDYv9t03KKkjg6SPoaLKd4PBxz5ep8Ub9tY6iefR7pycdWqKDpS6dVrLePVp11z9OyQzAzkAIOzMHYQ2LvT9bH8+84gdodLj1Rcb9uqdb3do4dpcVXmNOpdNT4hSZlK0urf2Td7piopUt9YJckZYFcHcVgCA45h5g5DXI22Y73ve8eLw1nIc2Fdcoe37y/TSF5u04pf9yi2s/fRWsxi7Kio9GnNBB13evZWacSoLAHACM28Q2v2dVLpPcrik1n3CXU1YbMgt0qQPf9TXm+ueXiIxOlKntXRpaO/WuqhzKldrAQCaFPMGoV++8v1sc7ZkM8f9Z9bsKtTXm/dp095ivfW/bXUu1zElTred204Xd06TPYJTWwCApsu8QWjHCt/Pk/qGt44gKq6o0mtLtmj2t9u1Y39Zncv1a99ck/5wmjqmcpk6AMBczBuECnf4frboGN46GtmmvcW6+c0V2niUK7haupw6Ld2ldi1iNeaCk+WI4OotAIA5mTcIlR6cviE6Kbx1/Ebb80v19rfb9eynG5Uc56j1Hj4tXU6d0TpRN/dvq9NbJYS+SAAAjlPmDUIleZJDUnx6uCsJiNdr6N2VO/TC4k3all8qz2GXtR8KQfYIqy7pnKobzm6r01rGM8AZAIA6mDcIyZBsTinmxOgR2lNYrmGvLq9zXq4u6S51b52gG85uq1aJUYQfAADqwcRBSFLzdsf9jRQ/37BXw19dXut7fdo2151ZJ6tXm2YEHwAAGsDcQahZ23BXUKuvNubpxje+VYnbU+O9Xm2a6ZmruinNFRWGygAAaFrMHYSOo9NiHq+h5VvydfWMr2t9f0jPDD3yx85MWQEAQCMydxByxIe7AknS7G+26f73f6wxn9etv2+nm89pq4RoprEAACAYTB6EwncDQa/X0IwvN2vqx+tqvDf18i66ulfrMFQFAIC5mDsI2WND/pFVHq+unvG1vtm6v1q7zWrRygkXyBVtjuk+AAA4Hpg7CIW4R+i9lTs09u3vqrWd1Dxaky87Tb/vmBzSWgAAgNmDkD0mJB+zJa9E5z65uFqbzWrRiglZjP8BACCMzB2EbMENIeWVHt3z7vea890uf1uLOIfmjj5LyfHOoH42AAA4NnMHIWvwdr/M7VGXBxZUuxLs+rPaaMKlp3LzQwAAjhPmDkKRwemV+fiH3Rr971X+EHRx51Q9P7Q7AQgAgOOMuYNQROPenXnT3mKd/9Tn1dr+cW0PDTgttVE/BwAANA5zByF7dKNsxjAMvfD5Jj0+f3219tk3/U692zZvlM8AAACNz+RBqHHuI/TXt7/Te6t2+l8/9qcuGnImN0QEAOB4Z+4g1AhTbOw6UOYPQScnx2rGsJ7KTArNZfkAAOC3MXcQcrp+0+ql7iqd/fhn/tcf33E2k6ICAHACMW8QskRItobv/oFSt7o9+In/9fu39SUEAQBwgjHvN3dEwy+dzy+pHoJuPqetzmid2BhVAQCAEDJvj1AD7yGUV1yhng8v9L++96JTdOvv2zVWVQAAIITMG4Qa2CN0eAiaMaynLuiU0lgVAQCAEOPUWACeWfiz//l1fTMJQQAAnOBMHIQcAS1+oNStv3/qC0IDTkvRA5edFoyqAABACJk3CEUlBLT4HbNWy+M1lBAdqReG9ghOTQAAIKRMHISa1XvRrXkl+nzDXknSVWe2ltXK5KkAADQF5g1CkfWfcHXs26slSc5Iq+4Z0DFIBQEAgFAzbxCy1G/XDcPQmt2FkqS7B5xCbxAAAE2IeYOQtX67/sqSLSqv9EqS/tQ9PZgVAQCAEDNvELLYjrmIYRh6eO5aSVLn9HglRNuDXRUAAAgh8wYh67GD0KxvtvufPz2kWxCLAQAA4WDeIGSJPOYi763cIUmKsFrUPjku2BUBAIAQM28QOsYYoc/W7dE3W/dLkv55Q+9QVAQAAELMvEHoGGOE/vn1L/7nv2vbPNjVAACAMDBvEDpKj5BhGFq0bo8kadS57UNVEQAACDHzBqGj3Efow9W7/M9v6t82FNUAAIAwMG8QUt03Rjw0nUZKvEPxzmMPqgYAACcm8wYhS91B6OMfd0uSHhrUOVTVAACAMDBxEKp91/cVV/jvJN0zs/4TswIAgBNP2IPQ9OnTlZmZKafTqd69e2v58uVHXf7pp59Wx44dFRUVpYyMDI0ZM0bl5eUN+OTae4SWb8n3P28Ww52kAQBoysIahGbPnq2xY8cqOztbK1euVNeuXTVgwADt2bOn1uX/9a9/ady4ccrOztbatWv1yiuvaPbs2brvvvsC//A6To199IPvtNg5HVoEvk0AAHBCCWsQmjZtmm688UaNGDFCnTp10osvvqjo6Gi9+uqrtS7/1VdfqV+/frrmmmuUmZmpCy+8UFdfffUxe5FqVcepsf9t3idJ6sO9gwAAaPLCFoTcbrdWrFihrKysX4uxWpWVlaVly5bVuk7fvn21YsUKf/DZvHmz5s2bp0suuaTOz6moqFBhYWG1h6Rag1CZ26O8Yrck6XdtGR8EAEBTFxGuD87Ly5PH41FKSkq19pSUFK1bt67Wda655hrl5eXprLPOkmEYqqqq0i233HLUU2NTp07V5MmTa75Ry6mx5Vt/HR90equE+u0IAAA4YYV9sHQgFi9erClTpuj555/XypUr9d5772nu3Ll66KGH6lxn/PjxKigo8D+2bz84o7y1ZgZcn+PrLWqfHCubte7L6wEAQNMQth6hpKQk2Ww25ebmVmvPzc1VampqretMnDhR1157rW644QZJUpcuXVRSUqKbbrpJ999/v6y1TJvhcDjkcDhqbqyWILR4ve9Gir9noDQAAKYQth4hu92uHj16aNGiRf42r9erRYsWqU+fPrWuU1paWiPs2Gy+yVMNwwisgFrGCH37i2+2+UQumwcAwBTC1iMkSWPHjtXw4cPVs2dP9erVS08//bRKSko0YsQISdKwYcOUnp6uqVOnSpIGDhyoadOm6YwzzlDv3r21ceNGTZw4UQMHDvQHovqrfuqrqLxS7irfjRQv6JRS2woAAKCJCWsQGjJkiPbu3atJkyYpJydH3bp10/z58/0DqLdt21atB2jChAmyWCyaMGGCdu7cqRYtWmjgwIF65JFHAv/wI3qEVm8/4H/eISWuQfsDAABOLBYj4HNKJ7bCwkK5XC4VfDZd8b+/zd9+56xV+mD1LrVtEaNP//r78BUIAABq8H9/FxQoPj6+0bZ7Ql011qiOuHx+7e4iSVLvNtxIEQAAszBvEDpijFB5lUeSdM7JSeEoBgAAhIF5g9ARPUL7Dt5R+mTGBwEAYBrmDUKH9QiVuqtUXFElSWoRV8s9hwAAQJNk3iB0WI/Q9vwySVKsI0KuqMhwVQQAAELMvEHoMFvySiRJKfH0BgEAYCbmDUKH9QjtL/WND9pdUB6uagAAQBiYOAj9uusFZZWSpC7prnBVAwAAwoAgJKnk4EDpeMYHAQBgKuYNQoddNfb5Bt+s86fTIwQAgKmYNwgdNkbIGembsDUywry/DgAAzMi83/yHBaFf9vmuGut5UmK4qgEAAGFg3iB08NSYx2sot7BCktS6WXQ4CwIAACFm4iDkc+iKMUlqFmMPYyUAACDUTB+E8kt89xByRloVYTP9rwMAAFMx/Tf/rgO+6TXKK71hrgQAAISa6YPQoXsI2bliDAAA0zH9t39OoW9ajbPbJ4W5EgAAEGqmD0KHTo0lM+EqAACmY/ogtCWvVJIU72R6DQAAzIYglFcsSTLCXAcAAAg90wehKLtveo3kOE6NAQBgNqYPQpv2+KbXaNsiJsyVAACAUDN9EEqM9o0NirFHhLkSAAAQaqYOQoZhaG+xb56xNFdUmKsBAAChZuogVOL2qNLjGyadFMc8YwAAmI2pg9D+g/OMOSKsiubUGAAApmPeIGSxaMd+380U7Uy2CgCAKZk6AezY77uZotfgLkIAAJiRiYOQRVv3+S6dT3E5w1wLAAAIBxMHISnC6tv9ZtEMlAYAwIxMHYQKyiolSWe2aRbmSgAAQDiYOgh9v+OAJHqEAAAwK1MHIY/XN0jaHmHqXwMAAKZl6gRQ4vZIklo3iw5zJQAAIBxMHYQ27imWJCXHM/M8AABmZNogdPidg+IckWGrAwAAhI9pg1DpwdNiktQijh4hAADMyLRBqLzy1yAUZbeFsRIAABAupg1ClR5fEEqI5rQYAABmZdogVFLhC0JMuAoAgHmZNgVUeX0/9xRVhLcQAAAQNqYNQnnF5ZKkM1onhLcQAAAQNr8pCJWXlzdWHSFnOzjh6q4DZWGuBAAAhEvAQcjr9eqhhx5Senq6YmNjtXnzZknSxIkT9corrzR6gcGSU+gLcWdmMuEqAABmFXAQevjhhzVz5kw9/vjjstt/nay0c+fOevnllxu1uGCKioyQJO0uOHF7tQAAwG8TcBB644039NJLL2no0KGy2X69/07Xrl21bt26Ri0umHYePCV2RkZCeAsBAABhE3AQ2rlzp9q3b1+j3ev1qrKyslGKCoWoSF+IKyg7cWoGAACNK+Ag1KlTJ3355Zc12t99912dccYZjVJUKBy6fL5ti9jwFgIAAMImItAVJk2apOHDh2vnzp3yer167733tH79er3xxhv66KOPglFjULirfDdUjHUG/CsAAABNRMA9QoMGDdJ///tfLVy4UDExMZo0aZLWrl2r//73v7rggguCUWNQVBzsEop1MM8YAABm1aDukLPPPluffPJJY9cSUlVeXxByRBCEAAAwq4B7hNq2bat9+/bVaD9w4IDatm3bKEWFgvtgjxBzjQEAYF4Bp4CtW7fKc3Dm9sNVVFRo586djVJUKGzf77t8PjKCIAQAgFnV+9TYnDlz/M8XLFggl8vlf+3xeLRo0SJlZmY2anHBlBRjl9yS1RLuSgAAQLjUOwgNHjxYkmSxWDR8+PBq70VGRiozM1NPPfVUoxYXTFVeQ5KUEGU/xpIAAKCpqncQ8h4cXNymTRt98803SkpKClpRoXCgrFKKlOycGgMAwLQCvmpsy5YtwagjbCJsnBsDAMCsGnT5fElJiT7//HNt27ZNbre72nu33357oxQWKvHOyHCXAAAAwiTgILRq1SpdcsklKi0tVUlJiZo1a6a8vDxFR0crOTn5hAtC0XbuIwQAgFkFPEBmzJgxGjhwoPbv36+oqCh9/fXX+uWXX9SjRw89+eSTwagxqA5NvgoAAMwn4CC0evVq/fWvf5XVapXNZlNFRYUyMjL0+OOP67777gtGjUETYbXIyvXzAACYVsBBKDIyUlarb7Xk5GRt27ZNkuRyubR9+/bGrS7InPQGAQBgagEHoTPOOEPffPONJKl///6aNGmS3nrrLd15553q3LlzwAVMnz5dmZmZcjqd6t27t5YvX37U5Q8cOKCRI0cqLS1NDodDHTp00Lx58wL+XEmK5IoxAABMLeAgNGXKFKWlpUmSHnnkESUmJurWW2/V3r179Y9//COgbc2ePVtjx45Vdna2Vq5cqa5du2rAgAHas2dPrcu73W5dcMEF2rp1q959912tX79eM2bMUHp6eqC7IYnxQQAAmJ3FMAwjXB/eu3dvnXnmmXruueck+W7amJGRodGjR2vcuHE1ln/xxRf1xBNPaN26dYqMbNhl74WFhXK5XLpu7L36peVF+vSvv/8tuwAAAELg0Pd3QUGB4uPjG227jXZb5ZUrV+oPf/hDvZd3u91asWKFsrKyfi3GalVWVpaWLVtW6zpz5sxRnz59NHLkSKWkpKhz586aMmVKrZPAHlJRUaHCwsJqj0MirdxVGgAAMwsoCSxYsEB33XWX7rvvPm3evFmStG7dOg0ePFhnnnmmfxqO+sjLy5PH41FKSkq19pSUFOXk5NS6zubNm/Xuu+/K4/Fo3rx5mjhxop566ik9/PDDdX7O1KlT5XK5/I+MjAz/ezauGAMAwNTqHYReeeUVXXzxxZo5c6Yee+wx/e53v9M///lP9enTR6mpqfrxxx8bPGi5vrxer5KTk/XSSy+pR48eGjJkiO6//369+OKLda4zfvx4FRQU+B+HX9nGYGkAAMyt3neWfuaZZ/TYY4/p7rvv1n/+8x9dccUVev755/XDDz+oVatWAX9wUlKSbDabcnNzq7Xn5uYqNTW11nXS0tIUGRkpm+3XQc6nnnqqcnJy5Ha7ZbfXnEne4XDI4XDUur09RRUB1w0AAJqOevcIbdq0SVdccYUk6fLLL1dERISeeOKJBoUgSbLb7erRo4cWLVrkb/N6vVq0aJH69OlT6zr9+vXTxo0bq52C27Bhg9LS0moNQcfiimKeMQAAzKzeQaisrEzR0dGSJIvFIofD4b+MvqHGjh2rGTNm6PXXX9fatWt16623qqSkRCNGjJAkDRs2TOPHj/cvf+uttyo/P1933HGHNmzYoLlz52rKlCkaOXJkgz4/1eX8TfUDAIATW0CTrr788suKjY2VJFVVVWnmzJlKSkqqtkwgk64OGTJEe/fu1aRJk5STk6Nu3bpp/vz5/gHU27Zt89/FWpIyMjK0YMECjRkzRqeffrrS09N1xx136N577w1kN/zsNq4aAwDAzOp9H6HMzExZLEcfXGyxWPxXkx2vDr+PUFT3K/T80B7hLgkAABxDsO4jVO8eoa1btzbahx4vduwvC3cJAAAgjEx9bui0lo2XKAEAwInH1EEokjFCAACYmqmTAEEIAABzM3USyC9xh7sEAAAQRqYOQs1jAr8JIwAAaDoaFIQ2bdqkCRMm6Oqrr9aePXskSR9//LF++umnRi0u2OK5szQAAKYWcBD6/PPP1aVLF/3vf//Te++9p+LiYknSd999p+zs7EYvMJgimHQVAABTCzgIjRs3Tg8//LA++eSTavN7nXfeefr6668btbhgi7Sa+swgAACmF3AS+OGHH/THP/6xRntycrLy8vIapahQoUcIAABzCzgIJSQkaPfu3TXaV61apfT09EYpKlS4agwAAHMLOAhdddVVuvfee5WTkyOLxSKv16ulS5fqrrvu0rBhw4JRY9CkuaLCXQIAAAijgIPQlClTdMoppygjI0PFxcXq1KmTzjnnHPXt21cTJkwIRo1Bw6kxAADMrd6Trh5it9s1Y8YMTZw4UT/++KOKi4t1xhln6OSTTw5GfUEVYSUIAQBgZgEHoSVLluiss85S69at1bp162DUFDI2ghAAAKYW8Kmx8847T23atNF9992nNWvWBKOmkLFaCEIAAJhZwEFo165d+utf/6rPP/9cnTt3Vrdu3fTEE09ox44dwagvqDg1BgCAuQUchJKSkjRq1CgtXbpUmzZt0hVXXKHXX39dmZmZOu+884JRY9BYCUIAAJjab7q1cps2bTRu3Dg9+uij6tKliz7//PPGqiskbJwaAwDA1BochJYuXarbbrtNaWlpuuaaa9S5c2fNnTu3MWsLOkckU2wAAGBmAV81Nn78eM2aNUu7du3SBRdcoGeeeUaDBg1SdHR0MOoLqgjmGgMAwNQCDkJffPGF7r77bl155ZVKSkoKRk0hw+XzAACYW8BBaOnSpcGoIywIQgAAmFu9gtCcOXN08cUXKzIyUnPmzDnqspdddlmjFBYKXD4PAIC51SsIDR48WDk5OUpOTtbgwYPrXM5iscjj8TRWbUFHjxAAAOZWryDk9XprfX6iIwgBAGBuAV829cYbb6iioqJGu9vt1htvvNEoRQEAAIRCwEFoxIgRKigoqNFeVFSkESNGNEpRoRJtt4W7BAAAEEYBByHDMGSp5Y7MO3bskMvlapSiQoVJVwEAMLd6Xz5/xhlnyGKxyGKx6Pzzz1dExK+rejwebdmyRRdddFFQigwW5hoDAMDc6h2EDl0ttnr1ag0YMECxsbH+9+x2uzIzM/WnP/2p0QsMJnIQAADmVu8glJ2dLUnKzMzUkCFD5HQ6g1ZUqHBqDAAAcwv4ztLDhw8PRh1hQRACAMDc6hWEmjVrpg0bNigpKUmJiYm1DpY+JD8/v9GKCzZOjQEAYG71CkJ/+9vfFBcX539+tCB0IuGGigAAmFu9gtDhp8Ouu+66YNUSck0l0AEAgIYJ+D5CK1eu1A8//OB//eGHH2rw4MG677775Ha7G7W4YKNDCAAAcws4CN18883asGGDJGnz5s0aMmSIoqOj9c477+iee+5p9AKDiVNjAACYW8BBaMOGDerWrZsk6Z133lH//v31r3/9SzNnztR//vOfxq4vqAhCAACYW4Om2Dg0A/3ChQt1ySWXSJIyMjKUl5fXuNUFWYQ14N0HAABNSMBJoGfPnnr44Yf15ptv6vPPP9ell14qSdqyZYtSUlIavcBgokMIAABzCzgIPf3001q5cqVGjRql+++/X+3bt5ckvfvuu+rbt2+jFxgsFgtXjQEAYHYB31n69NNPr3bV2CFPPPGEbDZboxQVChYRggAAMLuAg9AhK1as0Nq1ayVJnTp1Uvfu3RutqFCwkYMAADC9gIPQnj17NGTIEH3++edKSEiQJB04cEDnnnuuZs2apRYtWjR2jcFBEAIAwPQCHiM0evRoFRcX66efflJ+fr7y8/P1448/qrCwULfffnswagwKBkoDAICAe4Tmz5+vhQsX6tRTT/W3derUSdOnT9eFF17YqMUFEwOlAQBAwD1CXq9XkZGRNdojIyP99xc6EZCDAABAwEHovPPO0x133KFdu3b523bu3KkxY8bo/PPPb9TigomrxgAAQMBB6LnnnlNhYaEyMzPVrl07tWvXTm3atFFhYaGeffbZYNQYFKVuT7hLAAAAYRbwGKGMjAytXLlSixYt8l8+f+qppyorK6vRiwMAAAimgILQ7NmzNWfOHLndbp1//vkaPXp0sOoKuoSomuOcAACAudQ7CL3wwgsaOXKkTj75ZEVFRem9997Tpk2b9MQTTwSzvqDh8nkAAFDvMULPPfecsrOztX79eq1evVqvv/66nn/++WDWFlRcNQYAAOodhDZv3qzhw4f7X19zzTWqqqrS7t27g1JYsFlJQgAAmF69g1BFRYViYmJ+XdFqld1uV1lZWVAKCzaCEAAACGiw9MSJExUdHe1/7Xa79cgjj8jlcvnbpk2b1njVBRE5CAAA1DsInXPOOVq/fn21tr59+2rz5s3+1yfStBUnUq0AACA46h2EFi9eHMQyQo8YBAAAAr6zdFNh5fp5AABMz7RBqLCsMtwlAACAMDNtEOKqMQAAYNoglBDNFBsAAJjdcRGEpk+frszMTDmdTvXu3VvLly+v13qzZs2SxWLR4MGDA/5MrhoDAAANCkJffvml/u///k99+vTRzp07JUlvvvmmlixZEvC2Zs+erbFjxyo7O1srV65U165dNWDAAO3Zs+eo623dulV33XWXzj777IbsAleNAQCAwIPQf/7zHw0YMEBRUVFatWqVKioqJEkFBQWaMmVKwAVMmzZNN954o0aMGKFOnTrpxRdfVHR0tF599dU61/F4PBo6dKgmT56stm3bBvyZEjdUBAAADQhCDz/8sF588UXNmDFDkZG/jrPp16+fVq5cGdC23G63VqxYoaysrF8LslqVlZWlZcuW1bnegw8+qOTkZF1//fXH/IyKigoVFhZWewAAAEgNCELr16/XOeecU6Pd5XLpwIEDAW0rLy9PHo9HKSkp1dpTUlKUk5NT6zpLlizRK6+8ohkzZtTrM6ZOnSqXy+V/ZGRkSKJHCAAANCAIpaamauPGjTXalyxZ0uDTVPVVVFSka6+9VjNmzFBSUlK91hk/frwKCgr8j+3bt0uSLIwSAgDA9AKadFWSbrzxRt1xxx169dVXZbFYtGvXLi1btkx33XWXJk6cGNC2kpKSZLPZlJubW609NzdXqampNZbftGmTtm7dqoEDB/rbvF6vb0ciIrR+/Xq1a9eu2joOh0MOh6PGtugRAgAAAQehcePGyev16vzzz1dpaanOOeccORwO3XXXXRo9enRA27Lb7erRo4cWLVrkvwTe6/Vq0aJFGjVqVI3lTznlFP3www/V2iZMmKCioiI988wz/tNe9UGPEAAACDgIWSwW3X///br77ru1ceNGFRcXq1OnToqNjW1QAWPHjtXw4cPVs2dP9erVS08//bRKSko0YsQISdKwYcOUnp6uqVOnyul0qnPnztXWT0hIkKQa7cfcjwZVCwAAmpKAg9AhdrtdnTp1+s0FDBkyRHv37tWkSZOUk5Ojbt26af78+f4B1Nu2bZPV2vj3feTUGAAAsBiGYQSywrnnnnvUuzJ/+umnv7moYCosLJTL5dJjTzyse+66P9zlAACAejj0/V1QUKD4+PhG227APULdunWr9rqyslKrV6/Wjz/+qOHDhzdWXUHHGCEAABBwEPrb3/5Wa/sDDzyg4uLi31xQqBCDAABAow2++b//+7+jTotxvGHSVQAA0GhBaNmyZXI6nY21ueAjBwEAYHoBnxq7/PLLq702DEO7d+/Wt99+G/ANFcOJHAQAAAIOQi6Xq9prq9Wqjh076sEHH9SFF17YaIUFW05BWbhLAAAAYRZQEPJ4PBoxYoS6dOmixMTEYNUUEumJ0eEuAQAAhFlAY4RsNpsuvPDCgGeZBwAAOB4FPFi6c+fO2rx5czBqCSnGCAEAgICD0MMPP6y77rpLH330kXbv3q3CwsJqjxMFl88DAIB6jxF68MEH9de//lWXXHKJJOmyyy6rFiYMw5DFYpHH42n8KgEAAIKg3kFo8uTJuuWWW/TZZ58Fs56QoUMIAADUOwgdmpu1f//+QSsmlMhBAAAgoDFCTWtcTVPaFwAA0BAB3UeoQ4cOxwxD+fn5v6kgAACAUAkoCE2ePLnGnaVPVPQHAQCAgILQVVddpeTk5GDVElokIQAATK/eY4Sa1vggchAAAAggCB26aqzJIAkBAGB69T415vV6g1lHyJGDAABAwFNsNBUEIQAAYNogBAAAQBACAACmZdog1MQuggMAAA1AEAIAAKZl2iDEcGkAAGDiIAQAAMyOIAQAAEzLtEGIMUIAAMC8QSjcBQAAgLAzbRACAAAwbxCiSwgAANMzbRAiBwEAABMHIaIQAABmZ9ogBAAAYN4gRIcQAACmZ94gBAAATM+0QWhvUUW4SwAAAGFm2iCUGu8MdwkAACDMTBuEmGIDAACYNggBAAAQhAAAgGkRhAAAgGkRhAAAgGmZNggxxQYAADBtEAIAADBtEOLyeQAAYNogBAAAQBACAACmZdogxJkxAABg4iBEFAIAwOxMG4QAAADMG4ToEAIAwPTMG4QAAIDpmTYI0SEEAABMG4RIQgAAwLxBCAAAmJ5pgxAdQgAAwLRBCAAAgCAEAABMy7RBiDtLAwAA0wYhAAAA0wYhCx1CAACY3nERhKZPn67MzEw5nU717t1by5cvr3PZGTNm6Oyzz1ZiYqISExOVlZV11OXrQg4CAABhD0KzZ8/W2LFjlZ2drZUrV6pr164aMGCA9uzZU+vyixcv1tVXX63PPvtMy5YtU0ZGhi688ELt3LkzxJUDAIATncUwDCOcBfTu3VtnnnmmnnvuOUmS1+tVRkaGRo8erXHjxh1zfY/Ho8TERD333HMaNmzYMZcvLCyUy+XSO6/9TX++7s7fWj4AAAiBQ9/fBQUFio+Pb7TthrVHyO12a8WKFcrKyvK3Wa1WZWVladmyZfXaRmlpqSorK9WsWbNa36+oqFBhYWG1h8RVYwAAIMxBKC8vTx6PRykpKdXaU1JSlJOTU69t3HvvvWrZsmW1MHW4qVOnyuVy+R8ZGRm/uW4AANA0hH2M0G/x6KOPatasWXr//ffldDprXWb8+PEqKCjwP7Zv3x7iKgEAwPEqIpwfnpSUJJvNptzc3Grtubm5Sk1NPeq6Tz75pB599FEtXLhQp59+ep3LORwOORyORqkXAAA0LWHtEbLb7erRo4cWLVrkb/N6vVq0aJH69OlT53qPP/64HnroIc2fP189e/Zs2IczRAgAANMLa4+QJI0dO1bDhw9Xz5491atXLz399NMqKSnRiBEjJEnDhg1Tenq6pk6dKkl67LHHNGnSJP3rX/9SZmamfyxRbGysYmNjw7YfAADgxBP2IDRkyBDt3btXkyZNUk5Ojrp166b58+f7B1Bv27ZNVuuvHVcvvPCC3G63/vznP1fbTnZ2th544IFQlg4AAE5wYb+PUKgdug/Bf15/WpcPuyPc5QAAgHpokvcRCieGCAEAANMGIQAAAIIQAAAwLYIQAAAwLdMGIeYaAwAA5g1C5CAAAEzPtEEIAACAIAQAAEzLvEGIc2MAAJieaYNQWUVVuEsAAABhZtogFGkz7a4DAICDTJsG7BGm3XUAAHCQadMAQ4QAAIBpgxAAAIBpgxA9QgAAwLRBCAAAgCAEAABMiyAEAABMy8RBiEFCAACYnYmDEAAAMDvTBiH6gwAAgGmDEAAAgGmDEPcRAgAApg1CAAAAJg5CdAkBAGB2Jg5CAADA7EwbhBgjBAAATBuEAAAATBuE6BACAACmDUIAAAAEIQAAYFoEIQAAYFoEIQAAYFqmDUIMlgYAAKYNQgAAAOYNQnQJAQBgeuYNQgAAwPRMG4ToEAIAAKYNQgAAAOYNQsy6CgCA6Zk3CAEAANMzbRCiPwgAAJg2CAEAAJg2CNEjBAAATBuEAAAATBuEuGgMAACYNggBAACYOAjRJQQAgNmZOAgBAACzM28QokMIAADTM28QAgAApkcQAgAApkUQAgAApmXaIMQQIQAAYNogBAAAQBACAACmRRACAACmZdogxFxjAADAtEEIAADAtEHIwnVjAACYnmmDEAAAAEEIAACYFkEIAACY1nERhKZPn67MzEw5nU717t1by5cvP+ry77zzjk455RQ5nU516dJF8+bNC/gzGSEEAADCHoRmz56tsWPHKjs7WytXrlTXrl01YMAA7dmzp9blv/rqK1199dW6/vrrtWrVKg0ePFiDBw/Wjz/+GOLKAQDAic5iGIYRzgJ69+6tM888U88995wkyev1KiMjQ6NHj9a4ceNqLD9kyBCVlJToo48+8rf97ne/U7du3fTiiy8e8/MKCwvlcrn0xX9e0NmX39J4OwIAAILm0Pd3QUGB4uPjG227EY22pQZwu91asWKFxo8f72+zWq3KysrSsmXLal1n2bJlGjt2bLW2AQMG6IMPPqh1+YqKClVUVPhfFxQUSJJKSstUWFj4G/cAAACEwqHv7MbuvwlrEMrLy5PH41FKSkq19pSUFK1bt67WdXJycmpdPicnp9blp06dqsmTJ9dov/jasZLG1lwBAAAct/bt2yeXy9Vo2wtrEAqF8ePHV+tBOnDggE466SRt27atUX+RCFxhYaEyMjK0ffv2Ru3mRMNwPI4fHIvjB8fi+FFQUKDWrVurWbNmjbrdsAahpKQk2Ww25ebmVmvPzc1VampqreukpqYGtLzD4ZDD4ajR7nK5+EN9nIiPj+dYHEc4HscPjsXxg2Nx/LBaG/c6r7BeNWa329WjRw8tWrTI3+b1erVo0SL16dOn1nX69OlTbXlJ+uSTT+pcHgAAoC5hPzU2duxYDR8+XD179lSvXr309NNPq6SkRCNGjJAkDRs2TOnp6Zo6daok6Y477lD//v311FNP6dJLL9WsWbP07bff6qWXXgrnbgAAgBNQ2IPQkCFDtHfvXk2aNEk5OTnq1q2b5s+f7x8QvW3btmrdYH379tW//vUvTZgwQffdd59OPvlkffDBB+rcuXO9Ps/hcCg7O7vW02UILY7F8YXjcfzgWBw/OBbHj2Adi7DfRwgAACBcwn5naQAAgHAhCAEAANMiCAEAANMiCAEAANNqkkFo+vTpyszMlNPpVO/evbV8+fKjLv/OO+/olFNOkdPpVJcuXTRv3rwQVdr0BXIsZsyYobPPPluJiYlKTExUVlbWMY8dAhPo341DZs2aJYvFosGDBwe3QBMJ9FgcOHBAI0eOVFpamhwOhzp06MC/VY0k0GPx9NNPq2PHjoqKilJGRobGjBmj8vLyEFXbdH3xxRcaOHCgWrZsKYvFUuccoodbvHixunfvLofDofbt22vmzJmBf7DRxMyaNcuw2+3Gq6++avz000/GjTfeaCQkJBi5ubm1Lr906VLDZrMZjz/+uLFmzRpjwoQJRmRkpPHDDz+EuPKmJ9Bjcc011xjTp083Vq1aZaxdu9a47rrrDJfLZezYsSPElTdNgR6PQ7Zs2WKkp6cbZ599tjFo0KDQFNvEBXosKioqjJ49exqXXHKJsWTJEmPLli3G4sWLjdWrV4e48qYn0GPx1ltvGQ6Hw3jrrbeMLVu2GAsWLDDS0tKMMWPGhLjypmfevHnG/fffb7z33nuGJOP9998/6vKbN282oqOjjbFjxxpr1qwxnn32WcNmsxnz588P6HObXBDq1auXMXLkSP9rj8djtGzZ0pg6dWqty1955ZXGpZdeWq2td+/exs033xzUOs0g0GNxpKqqKiMuLs54/fXXg1WiqTTkeFRVVRl9+/Y1Xn75ZWP48OEEoUYS6LF44YUXjLZt2xputztUJZpGoMdi5MiRxnnnnVetbezYsUa/fv2CWqfZ1CcI3XPPPcZpp51WrW3IkCHGgAEDAvqsJnVqzO12a8WKFcrKyvK3Wa1WZWVladmyZbWus2zZsmrLS9KAAQPqXB7105BjcaTS0lJVVlY2+gR7ZtTQ4/Hggw8qOTlZ119/fSjKNIWGHIs5c+aoT58+GjlypFJSUtS5c2dNmTJFHo8nVGU3SQ05Fn379tWKFSv8p882b96sefPm6ZJLLglJzfhVY31/h/3O0o0pLy9PHo/Hf1fqQ1JSUrRu3bpa18nJyal1+ZycnKDVaQYNORZHuvfee9WyZcsaf9ARuIYcjyVLluiVV17R6tWrQ1CheTTkWGzevFmffvqphg4dqnnz5mnjxo267bbbVFlZqezs7FCU3SQ15Fhcc801ysvL01lnnSXDMFRVVaVbbrlF9913XyhKxmHq+v4uLCxUWVmZoqKi6rWdJtUjhKbj0Ucf1axZs/T+++/L6XSGuxzTKSoq0rXXXqsZM2YoKSkp3OWYntfrVXJysl566SX16NFDQ4YM0f33368XX3wx3KWZzuLFizVlyhQ9//zzWrlypd577z3NnTtXDz30ULhLQwM1qR6hpKQk2Ww25ebmVmvPzc1VampqreukpqYGtDzqpyHH4pAnn3xSjz76qBYuXKjTTz89mGWaRqDHY9OmTdq6dasGDhzob/N6vZKkiIgIrV+/Xu3atQtu0U1UQ/5upKWlKTIyUjabzd926qmnKicnR263W3a7Pag1N1UNORYTJ07UtddeqxtuuEGS1KVLF5WUlOimm27S/fffX21uTARXXd/f8fHx9e4NkppYj5DdblePHj20aNEif5vX69WiRYvUp0+fWtfp06dPteUl6ZNPPqlzedRPQ46FJD3++ON66KGHNH/+fPXs2TMUpZpCoMfjlFNO0Q8//KDVq1f7H5dddpnOPfdcrV69WhkZGaEsv0lpyN+Nfv36aePGjf4wKkkbNmxQWloaIeg3aMixKC0trRF2DgVUg6k7Q6rRvr8DG8d9/Js1a5bhcDiMmTNnGmvWrDFuuukmIyEhwcjJyTEMwzCuvfZaY9y4cf7lly5dakRERBhPPvmksXbtWiM7O5vL5xtJoMfi0UcfNex2u/Huu+8au3fv9j+KiorCtQtNSqDH40hcNdZ4Aj0W27ZtM+Li4oxRo0YZ69evNz766CMjOTnZePjhh8O1C01GoMciOzvbiIuLM/79738bmzdvNv7f//t/Rrt27Ywrr7wyXLvQZBQVFRmrVq0yVq1aZUgypk2bZqxatcr45ZdfDMMwjHHjxhnXXnutf/lDl8/ffffdxtq1a43p06dz+fwhzz77rNG6dWvDbrcbvXr1Mr7++mv/e/379zeGDx9ebfm3337b6NChg2G3243TTjvNmDt3bogrbroCORYnnXSSIanGIzs7O/SFN1GB/t04HEGocQV6LL766iujd+/ehsPhMNq2bWs88sgjRlVVVYirbpoCORaVlZXGAw88YLRr185wOp1GRkaGcdtttxn79+8PfeFNzGeffVbrd8Ch3//w4cON/v3711inW7duht1uN9q2bWu89tprAX+uxTDoywMAAObUpMYIAQAABIIgBAAATIsgBAAATIsgBAAATIsgBAAATIsgBAAATIsgBAAATIsgBKCamTNnKiEhIdxlNJjFYtEHH3xw1GWuu+46DR48OCT1ADi+EYSAJui6666TxWKp8di4cWO4S9PMmTP99VitVrVq1UojRozQnj17GmX7u3fv1sUXXyxJ2rp1qywWi1avXl1tmWeeeUYzZ85slM+rywMPPODfT5vNpoyMDN10003Kz88PaDuENiC4mtTs8wB+ddFFF+m1116r1taiRYswVVNdfHy81q9fL6/Xq++++04jRozQrl27tGDBgt+87bpmDT+cy+X6zZ9TH6eddpoWLlwoj8ejtWvX6i9/+YsKCgo0e/bskHw+gGOjRwhoohwOh1JTU6s9bDabpk2bpi5duigmJkYZGRm67bbbVFxcXOd2vvvuO5177rmKi4tTfHy8evTooW+//db//pIlS3T22WcrKipKGRkZuv3221VSUnLU2iwWi1JTU9WyZUtdfPHFuv3227Vw4UKVlZXJ6/XqwQcfVKtWreRwONStWzfNnz/fv67b7daoUaOUlpYmp9Opk046SVOnTq227UOnxtq0aSNJOuOMM2SxWPT73/9eUvVelpdeekktW7asNrO7JA0aNEh/+ctf/K8//PBDde/eXU6nU23bttXkyZNVVVV11P2MiIhQamqq0tPTlZWVpSuuuEKffPKJ/32Px6Prr79ebdq0UVRUlDp27KhnnnnG//4DDzyg119/XR9++KG/d2nx4sWSpO3bt+vKK69UQkKCmjVrpkGDBmnr1q1HrQdATQQhwGSsVqv+/ve/66efftLrr7+uTz/9VPfcc0+dyw8dOlStWrXSN998oxUrVmjcuHGKjIyUJG3atEkXXXSR/vSnP+n777/X7NmztWTJEo0aNSqgmqKiouT1elVVVaVnnnlGTz31lJ588kl9//33GjBggC677DL9/PPPkqS///3vmjNnjt5++22tX79eb731ljIzM2vd7vLlyyVJCxcu1O7du/Xee+/VWOaKK67Qvn379Nlnn/nb8vPzNX/+fA0dOlSS9OWXX2rYsGG64447tGbNGv3jH//QzJkz9cgjj9R7H7du3aoFCxbIbrf727xer1q1aqV33nlHa9as0aRJk3Tffffp7bffliTddddduvLKK3XRRRdp9+7d2r17t/r27avKykoNGDBAcXFx+vLLL7V06VLFxsbqoosuktvtrndNAKQmOfs8YHbDhw83bDabERMT43/8+c9/rnXZd955x2jevLn/9WuvvWa4XC7/67i4OGPmzJm1rnv99dcbN910U7W2L7/80rBarUZZWVmt6xy5/Q0bNhgdOnQwevbsaRiGYbRs2dJ45JFHqq1z5plnGrfddpthGIYxevRo47zzzjO8Xm+t25dkvP/++4ZhGMaWLVsMScaqVauqLTN8+HBj0KBB/teDBg0y/vKXv/hf/+Mf/zBatmxpeDwewzAM4/zzzzemTJlSbRtvvvmmkZaWVmsNhmEY2dnZhtVqNWJiYgyn0+mfSXvatGl1rmMYhjFy5EjjT3/6U521Hvrsjh07VvsdVFRUGFFRUcaCBQuOun0A1TFGCGiizj33XL3wwgv+1zExMZJ8vSNTp07VunXrVFhYqKqqKpWXl6u0tFTR0dE1tjN27FjdcMMNevPNN/2nd9q1ayfJd9rs+++/11tvveVf3jAMeb1ebdmyRaeeemqttRUUFCg2NlZer1fl5eU666yz9PLLL6uwsFC7du1Sv379qi3fr18/fffdd5J8p7UuuOACdezYURdddJH+8Ic/6MILL/xNv6uhQ4fqxhtv1PPPPy+Hw6G33npLV111laxWq38/ly5dWq0HyOPxHPX3JkkdO3bUnDlzVF5ern/+859avXq1Ro8eXW2Z6dOn69VXX9W2bdtUVlYmt9utbt26HbXe7777Ths3blRcXFy19vLycm3atKkBvwHAvAhCQBMVExOj9u3bV2vbunWr/vCHP+jWW2/VI488ombNmmnJkiW6/vrr5Xa7a/1Cf+CBB3TNNddo7ty5+vjjj5Wdna1Zs2bpj3/8o4qLi3XzzTfr9ttvr7Fe69at66wtLi5OK1eulNVqVVpamqKioiRJhYWFx9yv7t27a8uWLfr444+1cOFCXXnllcrKytK77757zHXrMnDgQBmGoblz5+rMM8/Ul19+qb/97W/+94uLizV58mRdfvnlNdZ1Op11btdut/uPwaOPPqpLL71UkydP1kMPPSRJmjVrlu666y499dRT6tOnj+Li4vTEE0/of//731HrLS4uVo8ePaoF0EOOlwHxwImCIASYyIoVK+T1evXUU0/5ezsOjUc5mg4dOqhDhw4aM2aMrr76ar322mv64x//qO7du2vNmjU1AtexWK3WWteJj49Xy5YttXTpUvXv39/fvnTpUvXq1avackOGDNGQIUP05z//WRdddJHy8/PVrFmzats7NB7H4/EctR6n06nLL79cb731ljZu3KiOHTuqe/fu/ve7d++u9evXB7yfR5owYYLOO+883Xrrrf797Nu3r2677Tb/Mkf26Njt9hr1d+/eXbNnz1ZycrLi4+N/U02A2TFYGjCR9u3bq7KyUs8++6w2b96sN998Uy+++GKdy5eVlWnUqFFavHixfvnlFy1dulTffPON/5TXvffeq6+++kqjRo3S6tWr9fPPP+vDDz8MeLD04e6++2499thjmj17ttavX69x48Zp9erVuuOOOyRJ06ZN07///W+tW7dOGzZs0DvvvKPU1NRabwKZnJysqKgozZ8/X7m5uSooKKjzc4cOHaq5c+fq1Vdf9Q+SPmTSpEl64403NHnyZP30009au3atZs2apQkTJgS0b3369NHpp5+uKVOmSJJOPvlkffvtt1qwYIE2bNigiRMn6ptvvqm2TmZmpr7//nutX79eeXl5qqys1NChQ5WUlKRBgwbpyy+/1JYtW7R48WLdfvvt2rFjR0A1AaYX7kFKABpfbQNsD5k2bZqRlpZmREVFGQMGDDDeeOMNQ5Kxf/9+wzCqD2auqKgwrrrqKiMjI8Ow2+1Gy5YtjVGjRlUbCL18+XLjggsuMGJjY42YmBjj9NNPrzHY+XBHDpY+ksfjMR544AEjPT3diIyMNLp27Wp8/PHH/vdfeuklo1u3bkZMTIwRHx9vnH/++cbKlSv97+uwwdKGYRgzZswwMjIyDKvVavTv37/O34/H4zHS0tIMScamTZtq1DV//nyjb9++RlRUlBEfH2/06tXLeOmll+rcj+zsbKNr16412v/9738bDofD2LZtm1FeXm5cd911hsvlMhISEoxbb73VGDduXLX19uzZ4//9SjI+++wzwzAMY/fu3cawYcOMpKQkw+FwGG3btjVuvPFGo6CgoM6aANRkMQzDCG8UAwAACA9OjQEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANMiCAEAANP6/6R5yT/fCxqVAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "forest_classifier = RandomForestClassifier()\n",
    "\n",
    "y_probas_forest = cross_val_predict(forest_classifier, X_train, y_train_3, cv = 3, method = \"predict_proba\")\n",
    "y_scores_forest = y_probas_forest[ : , 1]\n",
    "\n",
    "fpr_forest, tpr_forest, thresholds_forest = roc_curve(y_train_3, y_scores_forest)\n",
    "\n",
    "plot_roc_curve(fpr, tpr)\n",
    "plot_roc_curve(fpr_forest, tpr_forest)\n",
    "plt.show()\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "00df1e0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9959419623480594"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(y_train_3, y_scores_forest)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81b27a09",
   "metadata": {},
   "source": [
    "# Multiclass Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1e20426c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-2 {color: black;background-color: white;}#sk-container-id-2 pre{padding: 0;}#sk-container-id-2 div.sk-toggleable {background-color: white;}#sk-container-id-2 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-2 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-2 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-2 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-2 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-2 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-2 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-2 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-2 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-2 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-2 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-2 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-2 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-2 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-2 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-2 div.sk-item {position: relative;z-index: 1;}#sk-container-id-2 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-2 div.sk-item::before, #sk-container-id-2 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-2 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-2 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-2 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-2 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-2 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-2 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-2 div.sk-label-container {text-align: center;}#sk-container-id-2 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-2 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-2\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>SGDClassifier(random_state=42)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-2\" type=\"checkbox\" checked><label for=\"sk-estimator-id-2\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">SGDClassifier</label><div class=\"sk-toggleable__content\"><pre>SGDClassifier(random_state=42)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "SGDClassifier(random_state=42)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_classifier.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "f88c3337",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2'], dtype='<U1')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sgd_classifier.predict([X[31313]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "1b9c28f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-38991.08943273, -11127.41230174,   2054.06961799,\n",
       "         -2605.05226486, -22561.3793939 , -13744.89246399,\n",
       "         -3646.22933529, -38999.66981916,  -3697.27675865,\n",
       "        -14285.45168457]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "some_digit_scores = sgd_classifier.decision_function([X[31313]])\n",
    "some_digit_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8dd315ad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-3 {color: black;background-color: white;}#sk-container-id-3 pre{padding: 0;}#sk-container-id-3 div.sk-toggleable {background-color: white;}#sk-container-id-3 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-3 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-3 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-3 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-3 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-3 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-3 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-3 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-3 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-3 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-3 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-3 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-3 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-3 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-3 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-3 div.sk-item {position: relative;z-index: 1;}#sk-container-id-3 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-3 div.sk-item::before, #sk-container-id-3 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-3 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-3 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-3 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-3 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-3 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-3 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-3 div.sk-label-container {text-align: center;}#sk-container-id-3 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-3 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-3\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>RandomForestClassifier()</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-3\" type=\"checkbox\" checked><label for=\"sk-estimator-id-3\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">RandomForestClassifier</label><div class=\"sk-toggleable__content\"><pre>RandomForestClassifier()</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "RandomForestClassifier()"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_classifier.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0ead1908",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['8'], dtype=object)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_classifier.predict([X[31313]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "dcfe7c1e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.  , 0.07, 0.05, 0.  , 0.  , 0.02, 0.05, 0.  , 0.81, 0.  ]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_scores = forest_classifier.predict_proba([X[31313]])\n",
    "forest_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d9213e89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.8481, 0.8687, 0.8519])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(sgd_classifier, X_train, y_train, cv = 3, scoring = \"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "b0c2c164",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.96305, 0.96365, 0.96765])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(forest_classifier, X_train, y_train, cv = 3, scoring = \"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "6073380b",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_scaled = scaler.fit_transform(X_train.astype(np.float64))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "25f9b3dd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tuncer/Library/Python/3.9/lib/python/site-packages/sklearn/linear_model/_stochastic_gradient.py:702: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  warnings.warn(\n",
      "/Users/tuncer/Library/Python/3.9/lib/python/site-packages/sklearn/linear_model/_stochastic_gradient.py:702: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  warnings.warn(\n",
      "/Users/tuncer/Library/Python/3.9/lib/python/site-packages/sklearn/linear_model/_stochastic_gradient.py:702: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([0.9055 , 0.90095, 0.9078 ])"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(sgd_classifier, X_train_scaled, y_train, cv = 3, scoring = \"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "bcee7374",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.96375, 0.9641 , 0.96725])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(forest_classifier, X_train_scaled, y_train, cv = 3, scoring = \"accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "5f9d6362",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/tuncer/Library/Python/3.9/lib/python/site-packages/sklearn/linear_model/_stochastic_gradient.py:702: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  warnings.warn(\n",
      "/Users/tuncer/Library/Python/3.9/lib/python/site-packages/sklearn/linear_model/_stochastic_gradient.py:702: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  warnings.warn(\n",
      "/Users/tuncer/Library/Python/3.9/lib/python/site-packages/sklearn/linear_model/_stochastic_gradient.py:702: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "y_train_predict = cross_val_predict(sgd_classifier, X_train_scaled, y_train, cv = 3)\n",
    "\n",
    "conf_matrix = confusion_matrix(y_train, y_train_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "68b4a18b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5607,    0,   13,    8,   11,   45,   35,    6,  197,    1],\n",
       "       [   1, 6419,   39,   25,    3,   49,    7,    9,  177,   13],\n",
       "       [  30,   26, 5276,   89,   71,   26,   67,   41,  321,   11],\n",
       "       [  30,   23,  114, 5277,    1,  219,   26,   43,  323,   75],\n",
       "       [  12,   14,   43,   10, 5263,   10,   46,   22,  254,  168],\n",
       "       [  28,   15,   29,  159,   50, 4547,   83,   21,  420,   69],\n",
       "       [  31,   18,   50,    3,   41,   91, 5562,    6,  115,    1],\n",
       "       [  18,   16,   49,   28,   48,   13,    5, 5742,  125,  221],\n",
       "       [  17,   65,   43,  101,    0,  121,   32,    9, 5412,   51],\n",
       "       [  27,   21,   29,   66,  133,   44,    1,  169,  279, 5180]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "63d6243e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x14c3f1f70>"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAGkCAYAAAAIduO+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAWPElEQVR4nO3df6iW9f348dc5x87R1TmHsmm5tJwMLH+UdixKaBtJEa0tGG2BgRiMsZ1SE2K6US2anRxbCNksY2vCsh8wotaoEY5yrcRf5Yq13AjaqVAL4txp7JTn3N8/xsdvZ+dY51Zf57ru4+MBF+V17tvrxdtz7qfXfd9ed0O1Wq0GACRpLHoAAEY3oQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIFXdhubee++Ns846K8aOHRsXXnhhbN26teiRSqWrqyvmzZsXra2tMWHChLj66qvjjTfeKHqs0rvrrruioaEhli1bVvQopfTOO+/EddddF+PHj49x48bFrFmzYvv27UWPVSp9fX1xyy23xNSpU2PcuHExbdq0uOOOO+J4vtpXXYbm0UcfjeXLl8dtt90WO3fujHPPPTcuv/zy2LdvX9Gjlcbzzz8fnZ2dsWXLlnj22Wfjk08+icsuuywOHDhQ9GiltW3btrj//vtj9uzZRY9SSh988EHMnz8/TjjhhHj66afj73//e/zyl7+Mk08+uejRSmX16tWxbt26WLt2bbz++uuxevXq+PnPfx733HNP0aMVpqEeL6p54YUXxrx582Lt2rUREdHf3x+TJ0+OG2+8MVasWFHwdOX03nvvxYQJE+L555+PSy65pOhxSmf//v0xd+7c+NWvfhU/+9nP4rzzzos1a9YUPVaprFixIv7617/GX/7yl6JHKbVvfOMbMXHixPj1r399aN+3v/3tGDduXPzud78rcLLi1N0Zzccffxw7duyIBQsWHNrX2NgYCxYsiJdeeqnAycqtp6cnIiJOOeWUgicpp87OzrjyyisHfF8x0JNPPhkdHR1xzTXXxIQJE2LOnDnxwAMPFD1W6Vx88cWxadOm2L17d0RE7Nq1K1544YW44oorCp6sOGOKHqBW77//fvT19cXEiRMH7J84cWL84x//KGiqcuvv749ly5bF/PnzY+bMmUWPUzqPPPJI7Ny5M7Zt21b0KKX25ptvxrp162L58uXx4x//OLZt2xZLliyJ5ubmWLRoUdHjlcaKFSuiUqnE9OnTo6mpKfr6+mLVqlWxcOHCokcrTN2Fhtp1dnbGa6+9Fi+88ELRo5ROd3d3LF26NJ599tkYO3Zs0eOUWn9/f3R0dMSdd94ZERFz5syJ1157Le677z6h+ZTHHnssHnroodi4cWPMmDEjXnnllVi2bFlMmjTpuF2nugvNqaeeGk1NTbF3794B+/fu3RunnXZaQVOV1w033BBPPfVUbN68Oc4444yixymdHTt2xL59+2Lu3LmH9vX19cXmzZtj7dq10dvbG01NTQVOWB6nn356nHPOOQP2nX322fH73/++oInK6eabb44VK1bEtddeGxERs2bNirfeeiu6urqO29DU3Ws0zc3Ncf7558emTZsO7evv749NmzbFRRddVOBk5VKtVuOGG26Ixx9/PP785z/H1KlTix6plC699NJ49dVX45VXXjm0dXR0xMKFC+OVV14RmU+ZP3/+oLfI7969O84888yCJiqnjz76KBobBz60NjU1RX9/f0ETFa/uzmgiIpYvXx6LFi2Kjo6OuOCCC2LNmjVx4MCBWLx4cdGjlUZnZ2ds3LgxnnjiiWhtbY09e/ZERER7e3uMGzeu4OnKo7W1ddDrVieeeGKMHz/e61n/46abboqLL7447rzzzvjOd74TW7dujfXr18f69euLHq1Urrrqqli1alVMmTIlZsyYES+//HLcfffdcf311xc9WnGqdeqee+6pTpkypdrc3Fy94IILqlu2bCl6pFKJiCG3Bx98sOjRSu+rX/1qdenSpUWPUUp/+MMfqjNnzqy2tLRUp0+fXl2/fn3RI5VOpVKpLl26tDplypTq2LFjq1/+8perP/nJT6q9vb1Fj1aYuvx3NADUj7p7jQaA+iI0AKQSGgBSCQ0AqYQGgFRCA0Cqug1Nb29v/PSnP43e3t6iRyk9azU81ml4rNPwWav/qtt/R1OpVKK9vT16enqira2t6HFKzVoNj3UaHus0fNbqv+r2jAaA+iA0AKQa8Ytq9vf3x7vvvhutra3R0NBwxL9PpVIZ8F8Oz1oNj3UaHus0fKN9rarVanz44YcxadKkQVes/rQRf43m7bffjsmTJ4/kIQFI1N3d/ZmfdzXiZzStra0REfHiiy/GSSedNNKHP6zZs2cXPQJQIs3NzUWPMMjHH39c9AhD+r/H9cMZ8dD839NlJ5100ucOB/XsaJ4aPp6U9Y2v/vyG7/PWypsBAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCpjig09957b5x11lkxduzYuPDCC2Pr1q3Hei4ARomaQ/Poo4/G8uXL47bbboudO3fGueeeG5dffnns27cvYz4A6lzNobn77rvje9/7XixevDjOOeecuO++++ILX/hC/OY3v8mYD4A6V1NoPv7449ixY0csWLDg//8GjY2xYMGCeOmll4a8T29vb1QqlQEbAMePmkLz/vvvR19fX0ycOHHA/okTJ8aePXuGvE9XV1e0t7cf2nyMM8DxJf1dZytXroyenp5DW3d3d/YhASiRmj7K+dRTT42mpqbYu3fvgP179+6N0047bcj7tLS0REtLy5FPCEBdq+mMprm5Oc4///zYtGnToX39/f2xadOmuOiii475cADUv5rOaCIili9fHosWLYqOjo644IILYs2aNXHgwIFYvHhxxnwA1LmaQ/Pd73433nvvvbj11ltjz549cd5558Uzzzwz6A0CABAR0VCtVqsjecBKpRLt7e3xt7/9LVpbW0fy0J9p6tSpRY/AKNPQ0FD0CHVhhB+Chq2Mry339vYWPcKQenp6oq2t7bBfd60zAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFQ1X1TzWJk9e3ZRhx5SGa+31NhYzr8HlHGtGB5/dhShnI9kAIwaQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQakyRB29oaCjy8AM0Npavubt27Sp6hCHNmTOn6BEGqVarRY8wSBlnKuP3eRnXKSLihBNOKHqEQXp7e4se4YiU77sOgFFFaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFQ1haarqyvmzZsXra2tMWHChLj66qvjjTfeyJoNgFGgptA8//zz0dnZGVu2bIlnn302Pvnkk7jsssviwIEDWfMBUOdq+uCzZ555ZsCvf/vb38aECRNix44dcckllxzTwQAYHY7qEzZ7enoiIuKUU0457G16e3sHfCpcpVI5mkMCUGeO+M0A/f39sWzZspg/f37MnDnzsLfr6uqK9vb2Q9vkyZOP9JAA1KEjDk1nZ2e89tpr8cgjj3zm7VauXBk9PT2Htu7u7iM9JAB16IieOrvhhhviqaeeis2bN8cZZ5zxmbdtaWmJlpaWIxoOgPpXU2iq1WrceOON8fjjj8dzzz0XU6dOzZoLgFGiptB0dnbGxo0b44knnojW1tbYs2dPRES0t7fHuHHjUgYEoL7V9BrNunXroqenJ772ta/F6aeffmh79NFHs+YDoM7V/NQZANTCtc4ASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmO6IPPRqMyXjB07ty5RY8wpJdffrnoEQaZPXt20SMMMnbs2KJHGKS3t7foETgKjY3lOjeoVqvDeuws19QAjDpCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBqTJEHr1arRR6+9Pr6+ooeYUizZ88ueoRBdu3aVfQIg5x77rlFjzBIQ0ND0SMMUtbHgf7+/qJHGGTMmEIfsgepVqvxySeffO7tnNEAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEcVmrvuuisaGhpi2bJlx2gcAEabIw7Ntm3b4v777y/lZ5MAUB5HFJr9+/fHwoUL44EHHoiTTz75WM8EwChyRKHp7OyMK6+8MhYsWPC5t+3t7Y1KpTJgA+D4UfPngj7yyCOxc+fO2LZt27Bu39XVFbfffnvNgwEwOtR0RtPd3R1Lly6Nhx56KMaOHTus+6xcuTJ6enoObd3d3Uc0KAD1qaYzmh07dsS+ffti7ty5h/b19fXF5s2bY+3atdHb2xtNTU0D7tPS0hItLS3HZloA6k5Nobn00kvj1VdfHbBv8eLFMX369PjRj340KDIAUFNoWltbY+bMmQP2nXjiiTF+/PhB+wEgwpUBAEhW87vO/tdzzz13DMYAYLRyRgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQ6qivdXY0Ghoaijz8ANVqtegR6kZzc3PRIwxy3nnnFT3CIH/84x+LHmGQb37zm0WPMEhfX1/RIwypjN/n//nPf4oeYYDhPm46owEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApGqoVqvVkTxgpVKJ9vb2/x68oWEkD/2ZRngZhqVM6/Np1mp4GhvL9/e43bt3Fz3CINOmTSt6hCGNGTOm6BEGOXjwYNEjDKmnpyfa2toO+/Xy/SQAMKoIDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkKrm0Lzzzjtx3XXXxfjx42PcuHExa9as2L59e8ZsAIwCNX3gwgcffBDz58+Pr3/96/H000/HF7/4xfjnP/8ZJ598ctZ8ANS5mkKzevXqmDx5cjz44IOH9k2dOvWYDwXA6FHTU2dPPvlkdHR0xDXXXBMTJkyIOXPmxAMPPPCZ9+nt7Y1KpTJgA+D4UVNo3nzzzVi3bl185StfiT/96U/xgx/8IJYsWRIbNmw47H26urqivb390DZ58uSjHhqA+tFQreED4Jubm6OjoyNefPHFQ/uWLFkS27Zti5deemnI+/T29kZvb++hX1cqlUOxKdPnvNewDCOmTOvzadZqeBoby/emzt27dxc9wiDTpk0reoQhjRlT0ysLI+LgwYNFjzCknp6eaGtrO+zXa/pJOP300+Occ84ZsO/ss8+Of//734e9T0tLS7S1tQ3YADh+1BSa+fPnxxtvvDFg3+7du+PMM888pkMBMHrUFJqbbroptmzZEnfeeWf861//io0bN8b69eujs7Mzaz4A6lxNoZk3b148/vjj8fDDD8fMmTPjjjvuiDVr1sTChQuz5gOgztX0ZoBjoVKpRHt7+38PXqIXcL3APXzWani8GWB4vBlg+I6LNwMAQK2EBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQKpCL+ZTxmtm8fnKeF2xMurr6yt6hEHKeF2xd955p+gRhlTGjz8ZO3Zs0SMMUK1WB3yw5eE4owEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFKNKXqAsmhsLF9zq9Vq0SMMqampqegRBjl48GDRIwwyZkz5frz6+vqKHmGQL33pS0WPMKTXX3+96BEGmTFjRtEjDDDcx6jyPboCMKoIDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgVU2h6evri1tuuSWmTp0a48aNi2nTpsUdd9xR2qsMA1C8mq5jvnr16li3bl1s2LAhZsyYEdu3b4/FixdHe3t7LFmyJGtGAOpYTaF58cUX41vf+lZceeWVERFx1llnxcMPPxxbt25NGQ6A+lfTU2cXX3xxbNq0KXbv3h0REbt27YoXXnghrrjiisPep7e3NyqVyoANgONHTWc0K1asiEqlEtOnT4+mpqbo6+uLVatWxcKFCw97n66urrj99tuPelAA6lNNZzSPPfZYPPTQQ7Fx48bYuXNnbNiwIX7xi1/Ehg0bDnuflStXRk9Pz6Gtu7v7qIcGoH7UdEZz8803x4oVK+Laa6+NiIhZs2bFW2+9FV1dXbFo0aIh79PS0hItLS1HPykAdammM5qPPvooGhsH3qWpqSn6+/uP6VAAjB41ndFcddVVsWrVqpgyZUrMmDEjXn755bj77rvj+uuvz5oPgDpXU2juueeeuOWWW+KHP/xh7Nu3LyZNmhTf//7349Zbb82aD4A611Ad4X/WX6lUor29fSQPOSz/+5RgGZT1igtNTU1FjzDIwYMHix5hkDFjavp73Ijo6+sreoRByvp9/vrrrxc9wiAzZswoeoQBqtVqVKvV6Onpiba2tsPernyPrgCMKkIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkKrQq/41NDQUefgByviZOmVan08r41qdcMIJRY8wSBkv9FnGC1g2NzcXPcKQOjo6ih5hkO3btxc9wgD79++PSy655HNv54wGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBINWYkT5gtVod8v8ZzPoMXxnXqowzlVFZ16mMc+3fv7/oEQY4cOBARHz+WjVUR3g133777Zg8efJIHhKARN3d3XHGGWcc9usjHpr+/v549913o7W1NRoaGo7496lUKjF58uTo7u6Otra2Yzjh6GOthsc6DY91Gr7RvlbVajU+/PDDmDRpUjQ2Hv6VmBF/6qyxsfEzy1ertra2UfkHmMFaDY91Gh7rNHyjea3a29s/9zbeDABAKqEBIFXdhqalpSVuu+22aGlpKXqU0rNWw2Odhsc6DZ+1+q8RfzMAAMeXuj2jAaA+CA0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACk+n8cK4Tgt2gVtgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 480x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(conf_matrix, cmap = plt.cm.gray)\n",
    "# Main diagonal means that digits are classified correctly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "94c17886",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_predict_forest = cross_val_predict(forest_classifier, X_train, y_train, cv = 3)\n",
    "conf_matrix_forest = confusion_matrix(y_train, y_train_predict_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "70d39dd9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[5844,    1,    8,    2,    4,    8,   18,    1,   34,    3],\n",
       "       [   0, 6635,   37,   18,   13,    6,    3,   14,   10,    6],\n",
       "       [  30,   10, 5756,   30,   29,    3,   12,   37,   42,    9],\n",
       "       [   9,   10,   86, 5810,    3,   67,    9,   43,   64,   30],\n",
       "       [   8,   12,   10,    2, 5664,    0,   28,   13,   10,   95],\n",
       "       [  22,   10,   11,   70,    8, 5188,   47,    5,   37,   23],\n",
       "       [  25,   11,    4,    0,    9,   39, 5811,    0,   19,    0],\n",
       "       [   4,   24,   65,    4,   32,    1,    0, 6035,   13,   87],\n",
       "       [  10,   35,   33,   56,   30,   38,   28,    7, 5552,   62],\n",
       "       [  22,    9,   15,   65,   74,   14,    3,   57,   48, 5642]])"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_matrix_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "068f4b4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x14e415850>"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAGkCAYAAAAIduO+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAVuklEQVR4nO3df6jVhf348de9t3m9q3svZVOTNF0Ultd+eS1MaIykiBYLRltg4AzGaLfUhJhtVItmN8cWQjbL2CeEZT9gRC1oIxzpXImmFUUrN4p2V6gFcY8a3OKe8/1j5Hd3V+sevS/f73N9POAQvj3v+371vuee532fc3y/m2q1Wi0AIElz0QMAMLYJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqRo2NA8++GBMnz49xo8fH5dcckls27at6JFKpbe3N+bOnRvt7e0xceLEuPbaa+Odd94peqzSu++++6KpqSmWLVtW9Cil9MEHH8QNN9wQEyZMiLa2tpg9e3a88sorRY9VKoODg3HHHXfEjBkzoq2tLc4888y455574ng+21dDhubJJ5+M5cuXx1133RU7d+6M888/P6688srYu3dv0aOVxqZNm6Knpye2bt0aL7zwQnz++edxxRVXxIEDB4oerbS2b98eDz/8cJx33nlFj1JKn3zyScyfPz++9rWvxfPPPx9vvfVW/OY3v4mTTz656NFKZdWqVbF27dpYs2ZN/P3vf49Vq1bFr371q3jggQeKHq0wTY14Us1LLrkk5s6dG2vWrImIiGq1GlOnTo1bbrklVqxYUfB05fTRRx/FxIkTY9OmTXHZZZcVPU7p7N+/Py666KL47W9/G7/85S/jggsuiNWrVxc9VqmsWLEi/va3v8Vf//rXokcpte985zsxadKk+N3vfndw2fe+971oa2uL3//+9wVOVpyGO6L57LPPYseOHbFgwYKDy5qbm2PBggXx8ssvFzhZufX390dExCmnnFLwJOXU09MTV1999ZDHFUM9++yz0d3dHdddd11MnDgxLrzwwnjkkUeKHqt0Lr300ti4cWPs2rUrIiJef/312LJlS1x11VUFT1acE4oeoF4ff/xxDA4OxqRJk4YsnzRpUrz99tsFTVVu1Wo1li1bFvPnz4+urq6ixymdJ554Inbu3Bnbt28vepRSe/fdd2Pt2rWxfPny+NnPfhbbt2+PJUuWxLhx42LRokVFj1caK1asiEqlEjNnzoyWlpYYHByMlStXxsKFC4serTANFxrq19PTE2+++WZs2bKl6FFKp6+vL5YuXRovvPBCjB8/vuhxSq1arUZ3d3fce++9ERFx4YUXxptvvhkPPfSQ0PyXp556Kh577LHYsGFDzJo1K1577bVYtmxZTJky5bjdTw0XmlNPPTVaWlpiz549Q5bv2bMnJk+eXNBU5XXzzTfHc889F5s3b47TTz+96HFKZ8eOHbF379646KKLDi4bHByMzZs3x5o1a2JgYCBaWloKnLA8TjvttDj33HOHLDvnnHPiD3/4Q0ETldNtt90WK1asiOuvvz4iImbPnh3vv/9+9Pb2Hrehabj3aMaNGxdz5syJjRs3HlxWrVZj48aNMW/evAInK5darRY333xzPP300/GXv/wlZsyYUfRIpXT55ZfHG2+8Ea+99trBW3d3dyxcuDBee+01kfkv8+fPH/YR+V27dsUZZ5xR0ETl9Omnn0Zz89Cn1paWlqhWqwVNVLyGO6KJiFi+fHksWrQouru74+KLL47Vq1fHgQMHYvHixUWPVho9PT2xYcOGeOaZZ6K9vT12794dERGdnZ3R1tZW8HTl0d7ePux9qxNPPDEmTJjg/az/ceutt8all14a9957b3z/+9+Pbdu2xbp162LdunVFj1Yq11xzTaxcuTKmTZsWs2bNildffTXuv//+uPHGG4serTi1BvXAAw/Upk2bVhs3blzt4osvrm3durXokUolIg55e/TRR4serfS+9a1v1ZYuXVr0GKX0xz/+sdbV1VVrbW2tzZw5s7Zu3bqiRyqdSqVSW7p0aW3atGm18ePH1775zW/Wfv7zn9cGBgaKHq0wDfnvaABoHA33Hg0AjUVoAEglNACkEhoAUgkNAKmEBoBUDRuagYGB+MUvfhEDAwNFj1J69tXI2E8jYz+NnH31Hw3772gqlUp0dnZGf39/dHR0FD1OqdlXI2M/jYz9NHL21X807BENAI1BaABIdcxPqlmtVuPDDz+M9vb2aGpqOuKvU6lUhvyXw7OvRsZ+Ghn7aeTG+r6q1Wqxb9++mDJlyrAzVv+3Y/4ezb///e+YOnXqsdwkAIn6+vq+9HpXx/yIpr29PSIitm7dGieddNKx3vxhOSU8wJH54nn9cI55aL54ueykk076yuEAinI0L+1nKeuHhL9qX/kwAACphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEg1RGF5sEHH4zp06fH+PHj45JLLolt27aN9lwAjBF1h+bJJ5+M5cuXx1133RU7d+6M888/P6688srYu3dvxnwANLi6Q3P//ffHj370o1i8eHGce+658dBDD8XXv/71+L//+7+M+QBocHWF5rPPPosdO3bEggUL/v8XaG6OBQsWxMsvv3zIdQYGBqJSqQy5AXD8qCs0H3/8cQwODsakSZOGLJ80aVLs3r37kOv09vZGZ2fnwZvLOAMcX9I/dXb77bdHf3//wVtfX1/2JgEokbou5XzqqadGS0tL7NmzZ8jyPXv2xOTJkw+5Tmtra7S2th75hAA0tLqOaMaNGxdz5syJjRs3HlxWrVZj48aNMW/evFEfDoDGV9cRTUTE8uXLY9GiRdHd3R0XX3xxrF69Og4cOBCLFy/OmA+ABld3aH7wgx/ERx99FHfeeWfs3r07LrjggvjTn/407AMCABAR0VSr1WrHcoOVSiU6OzvjzTffjPb29mO56S91xhlnFD0CUCJNTU1FjzDMMX66HrH+/v7o6Og47N871xkAqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAqrpPqjlaurq6itr0IZXxHELNzeX8PaCM+wpGm8f56CnnMxkAY4bQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQ6ociNNzU1Fbn5Ico0yxfefffdokc4pDPPPLPoERpCrVYreoSGUMafvbJq1MeUIxoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQqq7Q9Pb2xty5c6O9vT0mTpwY1157bbzzzjtZswEwBtQVmk2bNkVPT09s3bo1Xnjhhfj888/jiiuuiAMHDmTNB0CDa6odxZV0Pvroo5g4cWJs2rQpLrvsshGtU6lUorOz8z8bL9EFj8p4QSEXPmtsZXxMlVGZngfKrqyPqf7+/ujo6Djs3x/VFTb7+/sjIuKUU0457H0GBgZiYGDg4J8rlcrRbBKABnPEHwaoVquxbNmymD9/fnR1dR32fr29vdHZ2XnwNnXq1CPdJAAN6IhfOrvpppvi+eefjy1btsTpp59+2Psd6ojmi9iU6ZC5jIekXjprbGV8TJVRmZ4Hyq6sj6mUl85uvvnmeO6552Lz5s1fGpmIiNbW1mhtbT2SzQAwBtQVmlqtFrfccks8/fTT8eKLL8aMGTOy5gJgjKgrND09PbFhw4Z45plnor29PXbv3h0REZ2dndHW1pYyIACNra73aA73Wuqjjz4aP/zhD0f0NXy8eeS8R9PYyviYKqMyPQ+UXVkfU6P6Hk1Z/ycBKC/nOgMgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApDqiC5+NFifp/HJnnXVW0SMc0nvvvVf0CMNMnz696BGGaW4u3+9x1Wq16BEahrNKf7WRPoeX7ycBgDFFaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSnVD0ABze4OBg0SMc0vTp04seYZhdu3YVPcIwZ599dtEjDNPU1FT0CMPUarWiRzikMs7V0tJS9AhD1Gq1qFarX3k/RzQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEg1VGF5r777oumpqZYtmzZKI0DwFhzxKHZvn17PPzww3HeeeeN5jwAjDFHFJr9+/fHwoUL45FHHomTTz55tGcCYAw5otD09PTE1VdfHQsWLPjK+w4MDESlUhlyA+D4UfelnJ944onYuXNnbN++fUT37+3tjbvvvrvuwQAYG+o6ounr64ulS5fGY489FuPHjx/ROrfffnv09/cfvPX19R3RoAA0prqOaHbs2BF79+6Niy666OCywcHB2Lx5c6xZsyYGBgaipaVlyDqtra3R2to6OtMC0HDqCs3ll18eb7zxxpBlixcvjpkzZ8ZPf/rTYZEBgLpC097eHl1dXUOWnXjiiTFhwoRhywEgwpkBAEhW96fO/teLL744CmMAMFY5ogEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIddTnOuP409xcvt9Pzj777KJHGGbr1q1FjzDMvHnzih6hYTQ1NRU9wjCDg4NFj3BEyveMAcCYIjQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqU4oegAaT7VaLXqEhjBv3ryiRxjmvffeK3qEYaZPn170CIdUq9WKHmHMcEQDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUtUdmg8++CBuuOGGmDBhQrS1tcXs2bPjlVdeyZgNgDGgruvRfPLJJzF//vz49re/Hc8//3x84xvfiH/84x9x8sknZ80HQIOrKzSrVq2KqVOnxqOPPnpw2YwZM0Z9KADGjrpeOnv22Weju7s7rrvuupg4cWJceOGF8cgjj3zpOgMDA1GpVIbcADh+1BWad999N9auXRtnnXVW/PnPf46bbroplixZEuvXrz/sOr29vdHZ2XnwNnXq1KMeGoDG0VSr48LY48aNi+7u7njppZcOLluyZEls3749Xn755UOuMzAwEAMDAwf/XKlUxIbjQlNTU9EjDPPee+8VPcIw06dPL3oEjlJ/f390dHQc9u/rOqI57bTT4txzzx2y7Jxzzol//etfh12ntbU1Ojo6htwAOH7UFZr58+fHO++8M2TZrl274owzzhjVoQAYO+oKza233hpbt26Ne++9N/75z3/Ghg0bYt26ddHT05M1HwANrq7QzJ07N55++ul4/PHHo6urK+65555YvXp1LFy4MGs+ABpcXR8GGA2VSiU6OzuP5SahED4MMDI+DND4RvXDAABQL6EBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQ6oSiB+DwmpvL+XtAtVoteoRhynhesWN8GsERKeN5xXbv3l30CIc0efLkokcYpqWlpegRhqjVaiN6PijnMxkAY4bQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSnVD0AGXR1NRU9AjD1Gq1okc4pObm8v1+UsZ9VcbHVBlNnjy56BEO6a233ip6hGG6urqKHmGIkf7cle8ZA4AxRWgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKnqCs3g4GDccccdMWPGjGhra4szzzwz7rnnnlKeOReAcqjrMgGrVq2KtWvXxvr162PWrFnxyiuvxOLFi6OzszOWLFmSNSMADayu0Lz00kvx3e9+N66++uqIiJg+fXo8/vjjsW3btpThAGh8db10dumll8bGjRtj165dERHx+uuvx5YtW+Kqq6467DoDAwNRqVSG3AA4ftR1RLNixYqoVCoxc+bMaGlpicHBwVi5cmUsXLjwsOv09vbG3XfffdSDAtCY6jqieeqpp+Kxxx6LDRs2xM6dO2P9+vXx61//OtavX3/YdW6//fbo7+8/eOvr6zvqoQFoHHUd0dx2222xYsWKuP766yMiYvbs2fH+++9Hb29vLFq06JDrtLa2Rmtr69FPCkBDquuI5tNPP43m5qGrtLS0RLVaHdWhABg76jqiueaaa2LlypUxbdq0mDVrVrz66qtx//33x4033pg1HwANrq7QPPDAA3HHHXfET37yk9i7d29MmTIlfvzjH8edd96ZNR8ADa6pdoz/WX+lUonOzs5juckRaWpqKnqEhlHGfeXsFI2rrN+7t956q+gRhunq6ip6hCFqtVrUarXo7++Pjo6Ow97Puc4ASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKnqOnvzWFbWE/uVURlPqlnGmVynaWT+9xpXZTFr1qyiRxjm7bffLnqEIfbv3x9z5sz5yvuV8zsMwJghNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFKdcKw3WKvVjvUmGWW+h4ymsj6eyjjX/v37ix5hiC/m+ap9dcxDs2/fvmO9SUZZGX8AaVweTyM3Z86cokc4pH379kVnZ+dh/76pdoy/y9VqNT788MNob2+PpqamI/46lUolpk6dGn19fdHR0TGKE4499tXI2E8jYz+N3FjfV7VaLfbt2xdTpkyJ5ubDvxNzzI9ompub4/TTTx+1r9fR0TEmv4EZ7KuRsZ9Gxn4aubG8r77sSOYLPgwAQCqhASBVw4amtbU17rrrrmhtbS16lNKzr0bGfhoZ+2nk7Kv/OOYfBgDg+NKwRzQANAahASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFT/DyJPanHBZHVIAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 480x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(conf_matrix_forest, cmap = plt.cm.gray)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d62acdd1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAGkCAYAAAAIduO+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAWeUlEQVR4nO3df2xVhf3/8XepUNCVThFQQlE0S5Af/kCQKNPNSDRGzUwWNxNcCCZu2YqAZG6wRZ1xWFk2RwIOxWyOZOKPZHE6E10MThlTAlI1EjfBmGj9AejmWkW9sPZ+/1g+fNfdqlzkzbm3Ph7JDeFwL/eVA+mT05Z7G8rlcjkAIMmgogcAMLAJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0Aqeo2NLfddlscf/zxMXTo0JgxY0Zs2rSp6Ek1pb29PaZPnx7Nzc0xatSouPTSS+Oll14qelbNu+WWW6KhoSEWLlxY9JSa9MYbb8QVV1wRI0aMiGHDhsWUKVPimWeeKXpWTenp6Ynrrrsuxo8fH8OGDYsTTzwxbrrppvg8v9pXXYbmvvvui0WLFsUNN9wQHR0dccopp8QFF1wQu3btKnpazXjyySejra0tNm7cGI899ljs3bs3zj///Ni9e3fR02rW5s2b44477oiTTz656Ck16d13342ZM2fG4MGD45FHHokXX3wxfvGLX8SRRx5Z9LSasmzZsli1alWsXLky/va3v8WyZcviZz/7WaxYsaLoaYVpqMcX1ZwxY0ZMnz49Vq5cGRERvb290draGldffXUsXry44HW16e23345Ro0bFk08+Geecc07Rc2rO+++/H1OnTo1f/epX8dOf/jROPfXUWL58edGzasrixYvjr3/9a/zlL38pekpNu/jii2P06NHx61//et+xr3/96zFs2LD43e9+V+Cy4tTdFc2ePXtiy5YtMWvWrH3HBg0aFLNmzYqnn366wGW1raurKyIijjrqqIKX1Ka2tra46KKL+vy9oq+HHnoopk2bFpdddlmMGjUqTjvttLjzzjuLnlVzzjrrrFi3bl1s27YtIiKef/752LBhQ1x44YUFLyvOYUUPqNY777wTPT09MXr06D7HR48eHX//+98LWlXbent7Y+HChTFz5syYPHly0XNqzr333hsdHR2xefPmoqfUtFdeeSVWrVoVixYtih/96EexefPmmD9/fgwZMiTmzJlT9LyasXjx4uju7o4JEyZEY2Nj9PT0xNKlS2P27NlFTytM3YWG6rW1tcXWrVtjw4YNRU+pOZ2dnbFgwYJ47LHHYujQoUXPqWm9vb0xbdq0uPnmmyMi4rTTToutW7fG7bffLjT/5f7774+777471q5dG5MmTYrnnnsuFi5cGGPGjPncnqe6C83RRx8djY2NsXPnzj7Hd+7cGcccc0xBq2rXvHnz4uGHH47169fH2LFji55Tc7Zs2RK7du2KqVOn7jvW09MT69evj5UrV0apVIrGxsYCF9aOY489NiZOnNjn2EknnRS///3vC1pUm6699tpYvHhxXH755RERMWXKlHj11Vejvb39cxuauvsazZAhQ+L000+PdevW7TvW29sb69atizPPPLPAZbWlXC7HvHnz4oEHHojHH388xo8fX/SkmnTeeefFCy+8EM8999y+27Rp02L27Nnx3HPPicx/mTlzZsW3yG/bti2OO+64ghbVpg8++CAGDer7obWxsTF6e3sLWlS8uruiiYhYtGhRzJkzJ6ZNmxZnnHFGLF++PHbv3h1z584telrNaGtri7Vr18aDDz4Yzc3NsWPHjoiIaGlpiWHDhhW8rnY0NzdXfN3qiCOOiBEjRvh61v+45ppr4qyzzoqbb745vvGNb8SmTZti9erVsXr16qKn1ZRLLrkkli5dGuPGjYtJkybFs88+G7feemtceeWVRU8rTrlOrVixojxu3LjykCFDymeccUZ548aNRU+qKRHR7+2uu+4qelrN+8pXvlJesGBB0TNq0h//+Mfy5MmTy01NTeUJEyaUV69eXfSkmtPd3V1esGBBedy4ceWhQ4eWTzjhhPKPf/zjcqlUKnpaYery/9EAUD/q7ms0ANQXoQEgldAAkEpoAEglNACkEhoAUtVtaEqlUvzkJz+JUqlU9JSa51ztH+dp/zhP+8+5+o+6/X803d3d0dLSEl1dXTF8+PCi59Q052r/OE/7x3naf87Vf9TtFQ0A9UFoAEh1yF9Us7e3N958881obm6OhoaGA/59uru7+/zIx3Ou9o/ztH+cp/030M9VuVyO9957L8aMGVPxitX/7ZB/jeb111+P1tbWQ/mUACTq7Oz8xPe7OuRXNM3NzYf6KevWkCFDip7Qr56enqInVPikf00V5aijjip6QoU9e/YUPaFCV1dX0RP6ddVVVxU9ocIdd9xR9IR+fdrH9UMems/y6bLPm1o9V7W4qxY31WL8anFTLf7ZRdTuP/Rq0af9Gdbe3zoABhShASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEh1QKG57bbb4vjjj4+hQ4fGjBkzYtOmTQd7FwADRNWhue+++2LRokVxww03REdHR5xyyilxwQUXxK5duzL2AVDnqg7NrbfeGldddVXMnTs3Jk6cGLfffnscfvjh8Zvf/CZjHwB1rqrQ7NmzJ7Zs2RKzZs36/7/BoEExa9asePrpp/t9TKlUiu7u7j43AD4/qgrNO++8Ez09PTF69Og+x0ePHh07duzo9zHt7e3R0tKy7+ZtnAE+X9K/62zJkiXR1dW179bZ2Zn9lADUkKreyvnoo4+OxsbG2LlzZ5/jO3fujGOOOabfxzQ1NUVTU9OBLwSgrlV1RTNkyJA4/fTTY926dfuO9fb2xrp16+LMM8886OMAqH9VXdFERCxatCjmzJkT06ZNizPOOCOWL18eu3fvjrlz52bsA6DOVR2ab37zm/H222/H9ddfHzt27IhTTz01Hn300YpvEACAiAMITUTEvHnzYt68eQd7CwADkNc6AyCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEh1QK91xqHxhS98oegJ/SqVSkVPqLB79+6iJ1Qol8tFT6jwr3/9q+gJFRoaGoqe0K//fd+tWjBkyJCiJ/RRLpdj7969n3o/VzQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFSHFfXELS0t0dDQUNTTV+jt7S16QoV//OMfRU/o16mnnlr0hAoffvhh0RMqvPXWW0VPqDB+/PiiJ1To7u4uekK/rrjiiqInVPjDH/5Q9IQD4ooGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApKoqNO3t7TF9+vRobm6OUaNGxaWXXhovvfRS1jYABoCqQvPkk09GW1tbbNy4MR577LHYu3dvnH/++bF79+6sfQDUuare+OzRRx/t8/Pf/va3MWrUqNiyZUucc845B3UYAAPDZ3qHza6uroiIOOqooz72PqVSKUql0r6f1+q76QGQ44C/GaC3tzcWLlwYM2fOjMmTJ3/s/drb26OlpWXfrbW19UCfEoA6dMChaWtri61bt8a99977ifdbsmRJdHV17bt1dnYe6FMCUIcO6FNn8+bNi4cffjjWr18fY8eO/cT7NjU1RVNT0wGNA6D+VRWacrkcV199dTzwwAPxxBNPxPjx47N2ATBAVBWatra2WLt2bTz44IPR3NwcO3bsiIiIlpaWGDZsWMpAAOpbVV+jWbVqVXR1dcVXv/rVOPbYY/fd7rvvvqx9ANS5qj91BgDV8FpnAKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUB/TGZwNRb29v0RMqfPnLXy56Qr82bNhQ9IS68IMf/KDoCRVuv/32oidUqNW3GPn3v/9d9IQKJ5xwQtET+ujp6Ynt27d/6v1c0QCQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUh1W1BN/8MEH0dDQUNTTV6ilLf/n5ZdfLnpCvwYPHlz0hAp79+4tekKFX/7yl0VPqDBy5MiiJ1To7u4uekK/Xn311aInVJgxY0bRE/rYs2dPbN++/VPv54oGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApPpMobnllluioaEhFi5ceJDmADDQHHBoNm/eHHfccUecfPLJB3MPAAPMAYXm/fffj9mzZ8edd94ZRx555MHeBMAAckChaWtri4suuihmzZr1qfctlUrR3d3d5wbA50fVb+V87733RkdHR2zevHm/7t/e3h433nhj1cMAGBiquqLp7OyMBQsWxN133x1Dhw7dr8csWbIkurq69t06OzsPaCgA9amqK5otW7bErl27YurUqfuO9fT0xPr162PlypVRKpWisbGxz2Oampqiqanp4KwFoO5UFZrzzjsvXnjhhT7H5s6dGxMmTIgf/vCHFZEBgKpC09zcHJMnT+5z7IgjjogRI0ZUHAeACK8MAECyqr/r7H898cQTB2EGAAOVKxoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVJ/5tc4O1BFHHBENDQ1FPX2Fjz76qOgJFUqlUtET+jV79uyiJ1R4/PHHi55Q4bXXXit6QoVjjjmm6AkVavXt3RcuXFj0hAqTJk0qekIfPT09+3U/VzQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFSHFT2gVjQ0NBQ9ocLhhx9e9IR+rVmzpugJFUaOHFn0hApTp04tekKFjo6OoidUaGxsLHpCv84999yiJ1R48cUXi55wQFzRAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRVh+aNN96IK664IkaMGBHDhg2LKVOmxDPPPJOxDYABoKr3o3n33Xdj5syZce6558YjjzwSI0eOjO3bt8eRRx6ZtQ+AOldVaJYtWxatra1x11137Ts2fvz4gz4KgIGjqk+dPfTQQzFt2rS47LLLYtSoUXHaaafFnXfe+YmPKZVK0d3d3ecGwOdHVaF55ZVXYtWqVfGlL30p/vSnP8V3v/vdmD9//ie+tW97e3u0tLTsu7W2tn7m0QDUj4ZyuVze3zsPGTIkpk2bFk899dS+Y/Pnz4/NmzfH008/3e9jSqVSlEqlfT/v7u6O1tbW+OIXvxgNDQ2fYfrB9d8ba0Wtfu3rzTffLHpChZEjRxY9ocLYsWOLnlCho6Oj6AkVGhsbi57Qr7PPPrvoCRWeeOKJoif0q6urK4YPH/6xv17VFc2xxx4bEydO7HPspJNOitdee+1jH9PU1BTDhw/vcwPg86Oq0MycOTNeeumlPse2bdsWxx133EEdBcDAUVVorrnmmti4cWPcfPPN8fLLL8fatWtj9erV0dbWlrUPgDpXVWimT58eDzzwQNxzzz0xefLkuOmmm2L58uUxe/bsrH0A1Lmq/h9NRMTFF18cF198ccYWAAYgr3UGQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkKrq1zo7WD766KOaeuOzKt7/7ZDp7e0tekK/mpubi55QoRbfPGvr1q1FT6gwaFDt/duyp6en6An9qqWPT//n+9//ftET+iiVSrFixYpPvV/t/a0DYEARGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAqsOKHlArxo4dW/SECu+//37RE/o1ceLEoidU2LhxY9ETKsyaNavoCRU6OjqKnlDhvffeK3pCv/785z8XPaHC9u3bi57QR29v737dzxUNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQqqrQ9PT0xHXXXRfjx4+PYcOGxYknnhg33XRTlMvlrH0A1Lmq3iZg2bJlsWrVqlizZk1MmjQpnnnmmZg7d260tLTE/PnzszYCUMeqCs1TTz0VX/va1+Kiiy6KiIjjjz8+7rnnnti0aVPKOADqX1WfOjvrrLNi3bp1sW3btoiIeP7552PDhg1x4YUXfuxjSqVSdHd397kB8PlR1RXN4sWLo7u7OyZMmBCNjY3R09MTS5cujdmzZ3/sY9rb2+PGG2/8zEMBqE9VXdHcf//9cffdd8fatWujo6Mj1qxZEz//+c9jzZo1H/uYJUuWRFdX175bZ2fnZx4NQP2o6orm2muvjcWLF8fll18eERFTpkyJV199Ndrb22POnDn9PqapqSmampo++1IA6lJVVzQffPBBDBrU9yGNjY3R29t7UEcBMHBUdUVzySWXxNKlS2PcuHExadKkePbZZ+PWW2+NK6+8MmsfAHWuqtCsWLEirrvuuvje974Xu3btijFjxsR3vvOduP7667P2AVDnqgpNc3NzLF++PJYvX540B4CBxmudAZBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSVfWimgfT4MGDo6Ghoainr/DWW28VPaHC4MGDi57Qr3feeafoCRUuvPDCoidUePbZZ4ueUOGf//xn0RMqfOtb3yp6Qr/OPvvsoidU+Pa3v130hAPiigaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEg1WGH+gnL5XKfH2tFre2JqM1NERG9vb1FT6iwd+/eoidUqMXzVIv27NlT9IR+ffjhh0VPqBuf9rGqoXyIP5q9/vrr0draeiifEoBEnZ2dMXbs2I/99UMemt7e3njzzTejubk5GhoaDvj36e7ujtbW1ujs7Izhw4cfxIUDj3O1f5yn/eM87b+Bfq7K5XK89957MWbMmBg06OO/EnPIP3U2aNCgTyxftYYPHz4g/wAzOFf7x3naP87T/hvI56qlpeVT7+ObAQBIJTQApKrb0DQ1NcUNN9wQTU1NRU+pec7V/nGe9o/ztP+cq/845N8MAMDnS91e0QBQH4QGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUv0/+1BVuMLXIWMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 480x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "def plot(conf_matrix):\n",
    "    row_sums = conf_matrix.sum(axis = 1, keepdims = True)\n",
    "    norm_conf_matrix = conf_matrix / row_sums\n",
    "    np.fill_diagonal(norm_conf_matrix, 0)\n",
    "    plt.matshow(norm_conf_matrix, cmap = plt.cm.gray)\n",
    "    \n",
    "plot(conf_matrix)\n",
    "plt.show()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "01820cbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZoAAAGkCAYAAAAIduO+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAWh0lEQVR4nO3df6yWBf3/8TcH4YDucFIMhAmK1oYCKgg6ZVkm0xk43UxzYWO4zNVBQZoLamrO4EgzY5NCcWVsgeLWnGZTYzRFE8YvdVIpOcuOOH647Bx+uCOdc3//aPHtfO6Dnht5c9338fHY7jXv7pvrtQu8n17nHO67T6lUKgUAJKkregAAvZvQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQqmZD87Of/SxOPfXUGDBgQJx//vmxYcOGoidVlebm5pg0aVI0NDTEkCFD4qqrroo33nij6FlV75577ok+ffrEnDlzip5SlbZv3x7XX399DB48OAYOHBjjxo2LTZs2FT2rqnR0dMTtt98eo0aNioEDB8bpp58ed999d3ya3+2rJkOzatWqmDt3btx5552xZcuWOPvss+Oyyy6LXbt2FT2tajz//PPR1NQU69evj9WrV8eBAwfi0ksvjX379hU9rWpt3LgxHnzwwTjrrLOKnlKV3n///Zg8eXL069cvnn766fjzn/8cP/nJT+L4448velpVWbRoUSxdujSWLFkSf/nLX2LRokXx4x//OO6///6ipxWmTy2+qeb5558fkyZNiiVLlkRERGdnZ4wYMSJuvvnmmDdvXsHrqtPu3btjyJAh8fzzz8dFF11U9Jyqs3fv3pgwYUL8/Oc/jx/96EdxzjnnxOLFi4ueVVXmzZsXf/zjH+OFF14oekpVmzZtWgwdOjR+8YtfHLzv6quvjoEDB8avf/3rApcVp+auaD788MPYvHlzTJky5eB9dXV1MWXKlFi3bl2By6pba2trRESccMIJBS+pTk1NTTF16tQuf67o6sknn4yJEyfGNddcE0OGDInx48fHQw89VPSsqnPhhRfGmjVrYtu2bRER8eqrr8aLL74Yl19+ecHLinNM0QMq9d5770VHR0cMHTq0y/1Dhw6N119/vaBV1a2zszPmzJkTkydPjrFjxxY9p+o8+uijsWXLlti4cWPRU6raW2+9FUuXLo25c+fG97///di4cWPccsst0b9//5gxY0bR86rGvHnzoq2tLUaPHh19+/aNjo6OWLBgQUyfPr3oaYWpudBQuaampti6dWu8+OKLRU+pOi0tLTF79uxYvXp1DBgwoOg5Va2zszMmTpwYCxcujIiI8ePHx9atW+OBBx4Qmv/x2GOPxYoVK2LlypUxZsyYeOWVV2LOnDkxfPjwT+15qrnQnHjiidG3b9/YuXNnl/t37twZJ510UkGrqtesWbPiqaeeirVr18bJJ59c9Jyqs3nz5ti1a1dMmDDh4H0dHR2xdu3aWLJkSbS3t0ffvn0LXFg9hg0bFmeeeWaX+84444z4zW9+U9Ci6nTbbbfFvHnz4rrrrouIiHHjxsXbb78dzc3Nn9rQ1Nz3aPr37x/nnnturFmz5uB9nZ2dsWbNmrjgggsKXFZdSqVSzJo1Kx5//PH4wx/+EKNGjSp6UlW65JJL4rXXXotXXnnl4G3ixIkxffr0eOWVV0Tmf0yePLnsR+S3bdsWp5xySkGLqtP+/fujrq7rS2vfvn2js7OzoEXFq7krmoiIuXPnxowZM2LixIlx3nnnxeLFi2Pfvn0xc+bMoqdVjaampli5cmU88cQT0dDQEDt27IiIiMbGxhg4cGDB66pHQ0ND2fetjjvuuBg8eLDvZ/0ft956a1x44YWxcOHCuPbaa2PDhg2xbNmyWLZsWdHTqsoVV1wRCxYsiJEjR8aYMWPi5Zdfjvvuuy9uuOGGoqcVp1Sj7r///tLIkSNL/fv3L5133nml9evXFz2pqkREt7eHH3646GlV74tf/GJp9uzZRc+oSr/97W9LY8eOLdXX15dGjx5dWrZsWdGTqk5bW1tp9uzZpZEjR5YGDBhQOu2000o/+MEPSu3t7UVPK0xN/j0aAGpHzX2PBoDaIjQApBIaAFIJDQCphAaAVEIDQKqaDU17e3v88Ic/jPb29qKnVD3nqmecp55xnnrOufqPmv17NG1tbdHY2Bitra0xaNCgoudUNeeqZ5ynnnGees65+o+avaIBoDYIDQCpjvqbanZ2dsa7774bDQ0N0adPn8P+ddra2rr8L4fmXPWM89QzzlPP9fZzVSqVYs+ePTF8+PCyd6z+X0f9ezTvvPNOjBgx4mgeEoBELS0tH/l5V0f9iqahoeFoH7JHPsnVVZYTTjih6Andam1tLXpCmX79+hU9ocyxxx5b9IQyw4YNK3pCmT/96U9FT+jW1VdfXfSEMk8++WTRE7oolUpx4MCBj31dP+qhqcYX9Ijq3PVRl6JFqsZzVY2bqvH3rxo/yK0af+8iqvM/Xqr1XH3crur7NwGAXkVoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIdVmh+9rOfxamnnhoDBgyI888/PzZs2HCkdwHQS1QcmlWrVsXcuXPjzjvvjC1btsTZZ58dl112WezatStjHwA1ruLQ3HfffXHjjTfGzJkz48wzz4wHHnggjj322PjlL3+ZsQ+AGldRaD788MPYvHlzTJky5f//AnV1MWXKlFi3bl23z2lvb4+2trYuNwA+PSoKzXvvvRcdHR0xdOjQLvcPHTo0duzY0e1zmpubo7Gx8eDNxzgDfLqk/9TZ/Pnzo7W19eCtpaUl+5AAVJGKPsr5xBNPjL59+8bOnTu73L9z58446aSTun1OfX191NfXH/5CAGpaRVc0/fv3j3PPPTfWrFlz8L7Ozs5Ys2ZNXHDBBUd8HAC1r6IrmoiIuXPnxowZM2LixIlx3nnnxeLFi2Pfvn0xc+bMjH0A1LiKQ/O1r30tdu/eHXfccUfs2LEjzjnnnHjmmWfKfkAAACIOIzQREbNmzYpZs2Yd6S0A9ELe6wyAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgVZ9SqVQ6mgdsa2uLxsbGo3nImnXttdcWPaFbW7duLXpCmb179xY9ocy//vWvoieUaW9vL3pCmWHDhhU9oVsffPBB0RPKVNumUqkUe/bsidbW1hg0aNAhH+eKBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQ6piiDvyVr3wl+vXrV9Thy2zcuLHoCWUee+yxoid068orryx6QpmOjo6iJ5RZvXp10RPKnHTSSUVPKHPxxRcXPaFbDQ0NRU8os2rVqqIndNHZ2Rl79uz52Me5ogEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCpKgpNc3NzTJo0KRoaGmLIkCFx1VVXxRtvvJG1DYBeoKLQPP/889HU1BTr16+P1atXx4EDB+LSSy+Nffv2Ze0DoMZV9MFnzzzzTJd//tWvfhVDhgyJzZs3x0UXXXREhwHQO3yiT9hsbW2NiIgTTjjhkI9pb2+P9vb2g//c1tb2SQ4JQI057B8G6OzsjDlz5sTkyZNj7Nixh3xcc3NzNDY2HryNGDHicA8JQA067NA0NTXF1q1b49FHH/3Ix82fPz9aW1sP3lpaWg73kADUoMP60tmsWbPiqaeeirVr18bJJ5/8kY+tr6+P+vr6wxoHQO2rKDSlUiluvvnmePzxx+O5556LUaNGZe0CoJeoKDRNTU2xcuXKeOKJJ6KhoSF27NgRERGNjY0xcODAlIEA1LaKvkezdOnSaG1tjS996UsxbNiwg7dVq1Zl7QOgxlX8pTMAqIT3OgMgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApDqsDz47EjZu3Bh1ddXTuWra8l9vvvlm0RO69bnPfa7oCWWq8cP1VqxYUfSEMk1NTUVPKNPQ0FD0hG5NmjSp6All/va3vxU9oYsDBw7EM88887GPq75XVwB6FaEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASNWnVCqVjuYB29raorGxMQYPHhx1ddXTueOOO67oCWU+/PDDoid0a/fu3UVPKHPgwIGiJ9SEqVOnFj2hzOuvv170hG598MEHRU8os3379qIndPHf1/PW1tYYNGjQIR9XPa/0APRKQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKT6RKG55557ok+fPjFnzpwjNAeA3uawQ7Nx48Z48MEH46yzzjqSewDoZQ4rNHv37o3p06fHQw89FMcff/yR3gRAL3JYoWlqaoqpU6fGlClTPvax7e3t0dbW1uUGwKfHMZU+4dFHH40tW7bExo0be/T45ubmuOuuuyoeBkDvUNEVTUtLS8yePTtWrFgRAwYM6NFz5s+fH62trQdvLS0thzUUgNpU0RXN5s2bY9euXTFhwoSD93V0dMTatWtjyZIl0d7eHn379u3ynPr6+qivrz8yawGoORWF5pJLLonXXnuty30zZ86M0aNHx/e+972yyABARaFpaGiIsWPHdrnvuOOOi8GDB5fdDwAR3hkAgGQV/9TZ//Xcc88dgRkA9FauaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSfeL3Ojtco0ePjmOOKezwZd56662iJ5Tp169f0RO69dJLLxU9ocy0adOKnlBm586dRU8o84UvfKHoCWVeeOGFoid06xvf+EbRE8p8+ctfLnpCF//+97979DhXNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVMcUdeC6urqoq9O5j7J79+6iJ3Rr0qRJRU8oM3To0KInlLnpppuKnlBm3rx5RU+oGevWrSt6QpktW7YUPeGweKUHIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqSoOzfbt2+P666+PwYMHx8CBA2PcuHGxadOmjG0A9AIVfR7N+++/H5MnT46LL744nn766fjsZz8bf/3rX+P444/P2gdAjasoNIsWLYoRI0bEww8/fPC+UaNGHfFRAPQeFX3p7Mknn4yJEyfGNddcE0OGDInx48fHQw899JHPaW9vj7a2ti43AD49KgrNW2+9FUuXLo3Pf/7z8eyzz8a3v/3tuOWWW2L58uWHfE5zc3M0NjYevI0YMeITjwagdlQUms7OzpgwYUIsXLgwxo8fH9/61rfixhtvjAceeOCQz5k/f360trYevLW0tHzi0QDUjopCM2zYsDjzzDO73HfGGWfEP/7xj0M+p76+PgYNGtTlBsCnR0WhmTx5crzxxhtd7tu2bVuccsopR3QUAL1HRaG59dZbY/369bFw4cJ48803Y+XKlbFs2bJoamrK2gdAjasoNJMmTYrHH388HnnkkRg7dmzcfffdsXjx4pg+fXrWPgBqXEV/jyYiYtq0aTFt2rSMLQD0Qt7rDIBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASBVxe91dqRs2rQp+vTpU9Thy0yYMKHoCWUWLlxY9IRu3XTTTUVPKHPBBRcUPaHMx33MOdXtn//8Z9ETymzbtq3oCV3s3bu3R6+drmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUxxR14MbGxqirq57OnXbaaUVPKPO73/2u6Anduvfee4ueUOb3v/990RPKfPOb3yx6Qpnt27cXPaHMunXrip7Qrb///e9FTyizadOmoid0sX///h49rnpe6QHolYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJCqotB0dHTE7bffHqNGjYqBAwfG6aefHnfffXeUSqWsfQDUuIo+JmDRokWxdOnSWL58eYwZMyY2bdoUM2fOjMbGxrjllluyNgJQwyoKzUsvvRRXXnllTJ06NSIiTj311HjkkUdiw4YNKeMAqH0VfenswgsvjDVr1sS2bdsiIuLVV1+NF198MS6//PJDPqe9vT3a2tq63AD49KjoimbevHnR1tYWo0ePjr59+0ZHR0csWLAgpk+ffsjnNDc3x1133fWJhwJQmyq6onnsscdixYoVsXLlytiyZUssX7487r333li+fPkhnzN//vxobW09eGtpafnEowGoHRVd0dx2220xb968uO666yIiYty4cfH2229Hc3NzzJgxo9vn1NfXR319/SdfCkBNquiKZv/+/VFX1/Upffv2jc7OziM6CoDeo6IrmiuuuCIWLFgQI0eOjDFjxsTLL78c9913X9xwww1Z+wCocRWF5v7774/bb789vvOd78SuXbti+PDhcdNNN8Udd9yRtQ+AGldRaBoaGmLx4sWxePHipDkA9Dbe6wyAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkKqiN9U8kkaOHBnHHFPY4cs8++yzRU8o85nPfKboCd366le/WvSEMt/97neLnlBm5syZRU8oU42fcPvTn/606And2rlzZ9ETynz9618vesJhcUUDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEglNACkEhoAUgkNAKmEBoBUQgNAKqEBIJXQAJBKaABIJTQApBIaAFIJDQCphAaAVEIDQCqhASCV0ACQSmgASCU0AKQSGgBSCQ0AqYQGgFRCA0AqoQEgldAAkOqYo33AUqkUEREdHR1H+9AfqbOzs+gJZartHP3X/v37i55QZt++fUVPKFONf6b+++9fNfnggw+KntCt9vb2oifUjI/7c9WndJT/5L3zzjsxYsSIo3lIABK1tLTEySeffMj//6iHprOzM959991oaGiIPn36HPav09bWFiNGjIiWlpYYNGjQEVzY+zhXPeM89Yzz1HO9/VyVSqXYs2dPDB8+POrqDv2dmKP+pbO6urqPLF+lBg0a1Ct/AzM4Vz3jPPWM89RzvflcNTY2fuxj/DAAAKmEBoBUNRua+vr6uPPOO6O+vr7oKVXPueoZ56lnnKeec67+46j/MAAAny41e0UDQG0QGgBSCQ0AqYQGgFRCA0AqoQEgldAAkEpoAEj1/wB5HWDkWH672wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 480x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(conf_matrix_forest)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "f4c560f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ True, False]])"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "y_train_large = (y_train.astype(np.float64) >= 7)\n",
    "y_train_odd = (y_train.astype(np.float64) % 2 == 1)\n",
    "y_multilabel = np.c_[y_train_large, y_train_odd]\n",
    "\n",
    "\n",
    "knn_classifier = KNeighborsClassifier()\n",
    "knn_classifier.fit(X_train, y_multilabel)\n",
    "\n",
    "knn_classifier.predict([X[31313]])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "17b1cd6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train_predict_knn = cross_val_predict(knn_classifier, X_train, y_train, cv = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "153f9643",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9681589850287663"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f1_score(y_train, y_train_predict_knn, average = \"macro\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "2258b440",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "noise = np.random.randint(0, 100, (len(X_train), 784))\n",
    "X_train_mod = X_train + noise\n",
    "\n",
    "noise = np.random.randint(0, 100, (len(X_test), 784))\n",
    "X_test_mod = X_test + noise\n",
    "y_train_mod = X_train\n",
    "y_test_mod = X_test\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "34a57106",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAD1CAYAAADNj/Z6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUVUlEQVR4nO3dSWzV5duH8edQOtB5ANpSSktpKdCiMkoJERRwim5MXGnixmiiiYkuNC40EhI1cdg4RCMLoysTWZgYg1FABRRohZaxlNJRKLSlc+nMeTf/zZuY733giAP39dlep2eG3Pklz30i0Wg0GgAAgFuz/uknAAAA/lkMAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOMcwAACAc7NjveHg4KDsbW1tskciEdnz8/NlP3/+vOyrV6+WfWBgQPa+vj7ZQwihp6dH9sLCQtmzsrJkv3LliuylpaWy//7777LPnTtX9oKCAtlPnz4t+5133il7f3+/7CUlJbIPDQ3JnpOTI3sIIVgLNxsaGmQvKiqSfdYsPV+fPXtW9vXr18uekpIiOwDcDK4MAADgHMMAAADOMQwAAOAcwwAAAM4xDAAA4BzDAAAAzsV8tNA69mYdK2tubpbdOvaWl5cn+6FDh2RPSEiQfcOGDbKHYL8HZWVlslvH2k6cOCH7xMSE7GlpabIXFxfLXltbG9f9t7a2yl5RUSH7/v37Zd+0aZPs1vHTEOzvWWZmpuzZ2dmyj4yMyD42NiZ7XV2d7NZ7AAA3gysDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOMcwAACAcwwDAAA4F/OeAesngK0z+C0tLXH1LVu2yG6xdgCMjo6a95GRkSG7dU4+KSkprp6bmyv78PCw7JOTk7JbZ+yt13/58mXZz507J7v1GZ86dUp2a9dFLPexYsUK2Y8fPy778uXLZZ89W/+Ts76nAHArcGUAAADnGAYAAHCOYQAAAOcYBgAAcI5hAAAA5xgGAABwjmEAAADnYt4zYJ3Dt35Lfvv27bIfPXpUdut34PPy8mTv7u6W3Tr/Hctt5syZI/vMzIzs8+bNk72goED2a9euyW7tAaisrJT99OnTsqekpMhu7QFobW2N6/4PHTokeyysXQhDQ0Oyp6eny75q1SrZm5qaZF+wYIHsAHAzuDIAAIBzDAMAADjHMAAAgHMMAwAAOMcwAACAcwwDAAA4xzAAAIBzMe8ZsM7Il5SUyB6JRGS3zsj39vbKnpOTI7t1hn1qakr2EELYvHmz7NY5/sbGRtnnz58vu3XOf8mSJbL39fXJbp2hr6iokN3a9TA8PCx7YmKi7JYNGzaYt/nll19kt77nCQkJslv7Mqz3IJZ9FwDwV+PKAAAAzjEMAADgHMMAAADOMQwAAOAcwwAAAM4xDAAA4BzDAAAAzsV8qHndunWyW2fgi4qKZLf2FCxevFj2/v5+2VesWCF7bW2t7CGE0NnZKbv1Gq33aGRkRPaJiQnZL1y4IPv69etl7+7ujqtbewyWLl0qu7Vn4I8//pDd+g7EwtoVUVVVJXs0GpXd2mdx5coV2QHgVuDKAAAAzjEMAADgHMMAAADOMQwAAOAcwwAAAM4xDAAA4BzDAAAAzkWi1sHo/zly5IjsaWlpsg8MDMiekpIi+9DQkOxlZWVx3X9LS4vsIYRQXV0te29vr+zZ2dmyJyQkyG69h9evX5d9enpa9szMTNnz8/NlHxsbi+vxr127Jnt7e7vs1vsXQgirVq2S/dixY7JnZGTIXlBQIPvVq1dlt16DtY8DAG4GVwYAAHCOYQAAAOcYBgAAcI5hAAAA5xgGAABwjmEAAADnGAYAAHBudqw3tM5PL1q0SHbrd+KTk5Nlt85/W+ev3377bdkvXbokewghjI6Oym6ds3/mmWdk7+jokP2pp56SfWpqSnbrM+js7JS9u7tb9tzcXNmtPQzDw8OyFxcXy269fyHY79GSJUtkHxkZkd3aV7FixQrZu7q6ZMff7+uvv5b9s88+M+9jwYIFslt7UJ544gnZrf+fy8vLZQe4MgAAgHMMAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOBeJRqPRWG4Y7zKUmZkZ2RsbG2Vfv3697M3NzbK/+uqrsp88eVL2EOzFHdZSImuxUl9fn+zZ2dlxPf6WLVtkr62tlX3lypWyW5+BtVglJycnrr9/9tlnZQ/BXhq0cOFC2QcGBmS3vudJSUmyV1RUyJ6VlSU7/nqLFy+Wva2t7e95IkJmZqbs1rKr2521sOzll18272Pt2rV/1dP5V+LKAAAAzjEMAADgHMMAAADOMQwAAOAcwwAAAM4xDAAA4BzDAAAAzs2O9YaRSET2/Px82VtbW2Xftm2b7IODg7Jb3njjDdnr6urM+7DOmXZ2dsq+a9cu2a33qL29XfaNGzfKbu1SaGhokD01NVX2a9euyT4xMSH71atXZT979mxcjx9CCF9++aXsp06dkn1qakp26zty8OBB2a09BuwZ+PtZ/26tfzch2Of8z5w5I/vx48dl/+mnn2Q/fPiw7IsWLZK9o6ND9nglJibKPnfuXNmtPTjW67f2EITAngEAAHCbYxgAAMA5hgEAAJxjGAAAwDmGAQAAnGMYAADAOYYBAACci3nPgHXO1DojPzk5KXtPT4/sRUVFcd2/dY71+eeflz0E+zlaz+G1116TPSMjQ3brDPx9990n+9GjR2UfHx+Xfd68ebJbewSSkpJkf+mll2Sfnp6WfcGCBbKHEEJCQoLsIyMjsq9evVr2AwcOyF5dXS17dna27Pj7bd26Na4eiwcffDCuv+/v75fd2lNgnaGvra294ed0I5KTk2WvrKyUfdmyZbL39fXJvmTJEtk94MoAAADOMQwAAOAcwwAAAM4xDAAA4BzDAAAAzjEMAADgHMMAAADORaLRaDSWG37//feyW2fQrd+LHh4elt06f239ln1qaqrssfxOvHXO3/rN8ubmZtmt98g6Z2+dJbZ2NVhn7AcHB2WvqamR/c0335T9rbfekt06C/3JJ5/IHkIIixcvlj0zM1N26zO0zktb3+OGhgbZN23aJDtwO9q9e7fsjz/+uOwrV66Uff/+/eZzyM3NNW/zX8aVAQAAnGMYAADAOYYBAACcYxgAAMA5hgEAAJxjGAAAwDmGAQAAnJsd6w3nzp0re0lJiezWGfmUlBTZrd+jtn6n/ty5c7Jbv5cdgr2LwHoM6zUsXbpUdmuPwL333it7S0uL7FVVVbKfOHFCdmsPQV1dnezV1dWyP/3007Ln5+fLHkIIra2tcd2Hta/izJkzspeWlspu7VIAbkfd3d2yP/fcc7Jb63Jef/112W/3HQKx4MoAAADOMQwAAOAcwwAAAM4xDAAA4BzDAAAAzjEMAADgHMMAAADOxbxnYPXq1bJ3dHTIPjU1JXtaWprs8+bNk906g19TUyP71atXZQ8hhIyMDNmt15CXlyf7yMiI7IWFhbIfOHBAduszuHz5suwzMzOyf/PNN7LX19fLfscdd8huvf9z5syRPYQQ5s+fL/vExERcz6GgoED2oaEh2Xt6emQvLi6WHfgv+uijj2S39hBkZ2fLHsseGe+4MgAAgHMMAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHORqPVD0P9jnVHv7OyU/fr167Jb56cvXLggu3WGf3h4WPa2tjbZQwihrKxM9vb2dtmrq6tlz8nJkX3v3r2yr1ixQvb+/n7Zx8bGZLf2KGzYsEH29PR02b/66ivZrTP+c+fOlT2EEJqammS3dilYn2Fzc7PsWVlZsi9atEh2fncd/0UHDx6UfevWrbJPTk7K/vPPP8t+zz33yA6uDAAA4B7DAAAAzjEMAADgHMMAAADOMQwAAOAcwwAAAM4xDAAA4NzsWG9o/db9+fPn9QPN1g/V29sre1FRkewXL16U3fo968TERNlDCKGrq0v2kpKSuB5jfHxc9jVr1shu/aa3tcegp6dH9vfee0/20dFR2R977DHZrT0FIyMjslu7JkIIYWhoSPbS0lLZrddo7Xqor6+P6++B/6LvvvtOdmuPwLZt22Svqam54eeE/48rAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOBfznoFr167Jbv2WvHVG/sCBA7E+lT9l7SHo6OiQvby83HwMa1dCY2Oj7GVlZbL/9ttvcf299RkdO3ZM9qqqKtnr6upkX7ZsmexPPvmk7NZvni9dulT25ORk2UMIoaCgQPbDhw/Lfvfdd8t+7tw52e+8807Zrd9l3759u+zAP2FsbEz2PXv2yG79292xY4fsseyJgcaVAQAAnGMYAADAOYYBAACcYxgAAMA5hgEAAJxjGAAAwDmGAQAAnIt5z0BnZ2dcDzQ1NSX78uXLZU9NTZW9oaFBdusca0tLi+whhDBnzhzZrTPo7e3tsq9evVp26yxvc3Oz7NaehC+++EJ2a4+CtQfAOgtcWloqu/UdTEhIkD2EEAYGBmS3XoPF2iNQX18v+9atW+N6fOCf8M4778h+/Phx2R966CHZN27ceMPPCTeGKwMAADjHMAAAgHMMAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADgX856ByspK2a1z+tYZ+enpadmtM+STk5OyWzsAZmZmZA8hhEOHDsmek5Mje1dXl+xpaWmyW3sOCgoKZN+1a5fsP/74Y1z3//HHH8teUVEhe2ZmpuxNTU2y5+fnyx5CCENDQ7Knp6fL3traKvvChQtlnz9/vuy//PKL7Fu2bJEduBW+/fZb2Xfu3Cl7VlaW7K+99toNPyf8tbgyAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOMcwAACAczHvGTh37pzsubm5slt7Anp7e2W3zpCnpKTIHolEZLf2FIRg7wE4efKk7NY5+8bGxrj+3noNn3/+uezWHoTXX39dduszSk1Nld16/tb7f/bsWdlDsHcZdHd3y15cXCz7yMiI7NevX5c9Go3KDtwKV69elf2FF16Q3doT8/DDD8teU1MjO249rgwAAOAcwwAAAM4xDAAA4BzDAAAAzjEMAADgHMMAAADOMQwAAOBcJBrjwebm5mbZ+/v7ZR8bG5O9urpa9traWtnvuusu2Ts7O2VftWqV7CHYZ8RPnDghe3p6uuzWGfjLly/Lbv2m+A8//CD7hg0bZP/ggw9kT0pKkr2oqEj2vr4+2aempmS3voMhhDAwMCC7tcugrKxM9ra2NtnHx8dlnzdvnuzWrgngz8zMzMhu/duvq6uTvby8XPY9e/bIvmTJEtlx63FlAAAA5xgGAABwjmEAAADnGAYAAHCOYQAAAOcYBgAAcI5hAAAA52bHekPrDLl1Rtw6v93S0iJ7SkqK7B0dHbJnZ2fLfuTIEdlDCGF0dFR264z40NCQ7BcvXjSfg2J9BvPnz5d9x44dshcWFspu7XL49ddfZc/IyJA9NTVV9tLSUtlDCKG+vl72/Px82a3XaO0RiEQisickJMgO3IwLFy7Ibu0RsLz//vuys0fg348rAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOMcwAACAczEvHYpGo7LX1NTI3tXVJXtbW5vs1tIia+FOVVWV7NZSoxBCaG5ulr2/v1/2iooK2a2lQy+++KLs1tKdV155RfapqSnZr1y5Iru1MGdmZkb2wcFB2a3FUQMDA7KHYC+Gsp6D9RkePHhQ9pKSEtkvXboke1lZmezwqb29Xfb7778/rvt/9913ZX/kkUfiun/887gyAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOMcwAACAczHvGbDOsY6Pj8v+xx9/yF5cXCz7smXLZK+rq5Pdev7WGfgQQigqKpI9KSlJ9kgkIvvu3bvj+vtff/1V9p07d8pufQYLFiyQ3Tpjb+2iOHPmjOyjo6OyJyYmyh5CCMPDw7f0MTZt2iR7T0+P7Pn5+bIDf+bTTz+V3fr/z7J582bZrf+b8O/HlQEAAJxjGAAAwDmGAQAAnGMYAADAOYYBAACcYxgAAMA5hgEAAJyLec/A5OSk7NYZ+61bt8peX18ve1dXl+yVlZWynzx5UvaNGzfKHkII+/fvl3358uWy79u3T/aGhgbZExISZK+qqpK9sbFR9rVr18re29sb199PTEzIbu0xuHjxouxXrlyRPYQQVq5cKXtycrLstbW1sl+6dEl2a49AXl6e7PDpwIEDsn/44Yd/0zPB7YorAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMMAwAAOBfznoFoNCq79TvwTU1Nsqelpck+NjYmu3XGPCMjQ/ahoSHZQwihurpa9lmz9Gx16NAh2a1z9tYZ90WLFsluPT9rl8Pw8LDshYWFslvPf3BwUPbS0lLZs7KyZA8hhAsXLsiempoq+wMPPCD71NSU7NY+DWsPAXw6ePCg7Na/TUt5ebns6enpcd0//v24MgAAgHMMAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADjHMAAAgHMx7xm4fv267Nb57MTERNl7enpkt86gWyorK2W39hSEYJ9zt87RW+fgrV0Njz76qOzPP/+87FVVVbL39fXJnpeXJ7v1m+s5OTmyW3sSrLPU1i6KEELIzc2V3XoPrO9hUlKS7MuWLZO9v79fduszAP7MXXfdJfvevXtlt/7d4L+PKwMAADjHMAAAgHMMAwAAOMcwAACAcwwDAAA4xzAAAIBzDAMAADgXiUaj0Vhu2NzcLLt1hv7kyZOyW+dgp6enZW9qapJ9YmJCdusMfgghnD17VvZZs/RsZb1Ga8/A+Pi47N3d3bLn5+fLXlhYKHtLS4vs1hl866uWnJws+5o1a2Tfv3+/7CGEsHnzZtn37dsn+7p162S/ePGi7NaegWPHjsm+du1a2QHgZnBlAAAA5xgGAABwjmEAAADnGAYAAHCOYQAAAOcYBgAAcI5hAAAA52LeMwAAAG5PXBkAAMA5hgEAAJxjGAAAwDmGAQAAnGMYAADAOYYBAACcYxgAAMA5hgEAAJxjGAAAwLn/A4k8a9aNKCgrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 640x480 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(121); plot_digit(X_test_mod[0])\n",
    "plt.subplot(122); plot_digit(y_test_mod[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "4425501d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAGFCAYAAAASI+9IAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAHFUlEQVR4nO3csY5NaxiAYUu2WiESjU6j0I8KiWjcgQvgMiZRqUSlE4lCo6OjUBI1ElcgotWZ7HW6tzknx/n3MXtts5+n/7K+TGbyztf80zzP8ykAOHXq1OmlFwBgd4gCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAGS19AL7YJqm4ZnDw8ONvnXx4sXhmUuXLg3PXLt2bXgG2H0uBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIF5J3YL3798PzxwcHBzDJv9sk1dcb926dQybLOvGjRvDM9++fRueefTo0fDMps6ePTs88+DBg+GZe/fuDc+wm1wKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAg0zzP89JL8Hfv3r3baO7ly5fDM58+fRqeefXq1fDMpjZ5sO8k/lrv8s/h7du3wzPXr1///Yvwv7kUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBAVksvwD+7evXqVufYfUdHR8Mzd+/eHZ55+vTp8MyPHz+GZ9hNLgUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABAP4sEfYrUa/3Pd5KG6aZqGZzg5XAoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAECmeZ7npZcAjscmL55evnx5eObz58/DM+wmlwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAMhq6QWA/+bDhw/DM5s8iHfnzp3hGU4OlwIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAMg0z/O89BLAr50+vZ3/4b5//z48c+7cuWPYhCW4FACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQFZLLwD75tmzZ1v71pMnT4ZnPG6331wKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAg0zzP89JLwD6Zpmlrc+v1eqNvsb9cCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQFZLLwB/ssPDw6196+vXr1v7FvvLpQBARAGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFADLN8zwvvQTsgo8fPw7PXLlyZXhm04ftLly4sNEcjHApABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGArJZeAHbF/fv3h2emaRqe8bAdu8ylAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUA4kE8TqSHDx8Oz7x48WJ45suXL8MzsMtcCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoARBQAiCgAEFEAINM8z/PSS8C/OTo6Gp45c+bMMWzyd2/evBmeuXnz5jFsAr+HSwGAiAIAEQUAIgoARBQAiCgAEFEAIKIAQEQBgIgCABEFACIKAEQUAMhq6QXgV86fP7+V7xwcHAzPePGUk8alAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUA4kE8tubnz58bzd2+fXt45vnz58Mzjx8/Hp6Bk8alAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAMs3zPC+9BPthmqad/tZ6vT6GTeDP4lIAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgBZLb0A/Momj9u9fv36GDaBk8+lAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoAZJrneV56CfbDJq+dbjq3Xq83+hbsO5cCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFADIaukF2B/eXoTd51IAIKIAQEQBgIgCABEFACIKAEQUAIgoABBRACCiAEBEAYCIAgARBQAiCgBEFACIKAAQUQAgogBARAGAiAIAEQUAIgoA5C+h8ZIarXIImQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "knn_classifier.fit(X_train_mod, y_train_mod)\n",
    "clean_digit = knn_classifier.predict([X_test_mod[0]])\n",
    "plot_digit(clean_digit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "7d2d42be",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-6 {color: black;background-color: white;}#sk-container-id-6 pre{padding: 0;}#sk-container-id-6 div.sk-toggleable {background-color: white;}#sk-container-id-6 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-6 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-6 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-6 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-6 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-6 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-6 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-6 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-6 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-6 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-6 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-6 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-6 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-6 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-6 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-6 div.sk-item {position: relative;z-index: 1;}#sk-container-id-6 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-6 div.sk-item::before, #sk-container-id-6 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-6 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-6 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-6 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-6 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-6 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-6 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-6 div.sk-label-container {text-align: center;}#sk-container-id-6 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-6 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-6\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>GridSearchCV(cv=3, estimator=KNeighborsClassifier(),\n",
       "             param_grid=[{&#x27;n_neighbors&#x27;: [3, 4, 5, 6],\n",
       "                          &#x27;weights&#x27;: [&#x27;uniform&#x27;, &#x27;distance&#x27;]}])</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item sk-dashed-wrapped\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-10\" type=\"checkbox\" ><label for=\"sk-estimator-id-10\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">GridSearchCV</label><div class=\"sk-toggleable__content\"><pre>GridSearchCV(cv=3, estimator=KNeighborsClassifier(),\n",
       "             param_grid=[{&#x27;n_neighbors&#x27;: [3, 4, 5, 6],\n",
       "                          &#x27;weights&#x27;: [&#x27;uniform&#x27;, &#x27;distance&#x27;]}])</pre></div></div></div><div class=\"sk-parallel\"><div class=\"sk-parallel-item\"><div class=\"sk-item\"><div class=\"sk-label-container\"><div class=\"sk-label sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-11\" type=\"checkbox\" ><label for=\"sk-estimator-id-11\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">estimator: KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier()</pre></div></div></div><div class=\"sk-serial\"><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-12\" type=\"checkbox\" ><label for=\"sk-estimator-id-12\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier()</pre></div></div></div></div></div></div></div></div></div></div>"
      ],
      "text/plain": [
       "GridSearchCV(cv=3, estimator=KNeighborsClassifier(),\n",
       "             param_grid=[{'n_neighbors': [3, 4, 5, 6],\n",
       "                          'weights': ['uniform', 'distance']}])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "param_grid = [{\"n_neighbors\" : [3, 4, 5, 6], \"weights\" : [\"uniform\", \"distance\"]}]\n",
    "\n",
    "grid_search = GridSearchCV(knn_classifier, param_grid, cv = 3)\n",
    "grid_search.fit(X_train[:10000], y_train[:10000])\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "b78d8346",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'n_neighbors': 4, 'weights': 'distance'}"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "9c0f6100",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>#sk-container-id-7 {color: black;background-color: white;}#sk-container-id-7 pre{padding: 0;}#sk-container-id-7 div.sk-toggleable {background-color: white;}#sk-container-id-7 label.sk-toggleable__label {cursor: pointer;display: block;width: 100%;margin-bottom: 0;padding: 0.3em;box-sizing: border-box;text-align: center;}#sk-container-id-7 label.sk-toggleable__label-arrow:before {content: \"▸\";float: left;margin-right: 0.25em;color: #696969;}#sk-container-id-7 label.sk-toggleable__label-arrow:hover:before {color: black;}#sk-container-id-7 div.sk-estimator:hover label.sk-toggleable__label-arrow:before {color: black;}#sk-container-id-7 div.sk-toggleable__content {max-height: 0;max-width: 0;overflow: hidden;text-align: left;background-color: #f0f8ff;}#sk-container-id-7 div.sk-toggleable__content pre {margin: 0.2em;color: black;border-radius: 0.25em;background-color: #f0f8ff;}#sk-container-id-7 input.sk-toggleable__control:checked~div.sk-toggleable__content {max-height: 200px;max-width: 100%;overflow: auto;}#sk-container-id-7 input.sk-toggleable__control:checked~label.sk-toggleable__label-arrow:before {content: \"▾\";}#sk-container-id-7 div.sk-estimator input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-label input.sk-toggleable__control:checked~label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 input.sk-hidden--visually {border: 0;clip: rect(1px 1px 1px 1px);clip: rect(1px, 1px, 1px, 1px);height: 1px;margin: -1px;overflow: hidden;padding: 0;position: absolute;width: 1px;}#sk-container-id-7 div.sk-estimator {font-family: monospace;background-color: #f0f8ff;border: 1px dotted black;border-radius: 0.25em;box-sizing: border-box;margin-bottom: 0.5em;}#sk-container-id-7 div.sk-estimator:hover {background-color: #d4ebff;}#sk-container-id-7 div.sk-parallel-item::after {content: \"\";width: 100%;border-bottom: 1px solid gray;flex-grow: 1;}#sk-container-id-7 div.sk-label:hover label.sk-toggleable__label {background-color: #d4ebff;}#sk-container-id-7 div.sk-serial::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: 0;}#sk-container-id-7 div.sk-serial {display: flex;flex-direction: column;align-items: center;background-color: white;padding-right: 0.2em;padding-left: 0.2em;position: relative;}#sk-container-id-7 div.sk-item {position: relative;z-index: 1;}#sk-container-id-7 div.sk-parallel {display: flex;align-items: stretch;justify-content: center;background-color: white;position: relative;}#sk-container-id-7 div.sk-item::before, #sk-container-id-7 div.sk-parallel-item::before {content: \"\";position: absolute;border-left: 1px solid gray;box-sizing: border-box;top: 0;bottom: 0;left: 50%;z-index: -1;}#sk-container-id-7 div.sk-parallel-item {display: flex;flex-direction: column;z-index: 1;position: relative;background-color: white;}#sk-container-id-7 div.sk-parallel-item:first-child::after {align-self: flex-end;width: 50%;}#sk-container-id-7 div.sk-parallel-item:last-child::after {align-self: flex-start;width: 50%;}#sk-container-id-7 div.sk-parallel-item:only-child::after {width: 0;}#sk-container-id-7 div.sk-dashed-wrapped {border: 1px dashed gray;margin: 0 0.4em 0.5em 0.4em;box-sizing: border-box;padding-bottom: 0.4em;background-color: white;}#sk-container-id-7 div.sk-label label {font-family: monospace;font-weight: bold;display: inline-block;line-height: 1.2em;}#sk-container-id-7 div.sk-label-container {text-align: center;}#sk-container-id-7 div.sk-container {/* jupyter's `normalize.less` sets `[hidden] { display: none; }` but bootstrap.min.css set `[hidden] { display: none !important; }` so we also need the `!important` here to be able to override the default hidden behavior on the sphinx rendered scikit-learn.org. See: https://github.com/scikit-learn/scikit-learn/issues/21755 */display: inline-block !important;position: relative;}#sk-container-id-7 div.sk-text-repr-fallback {display: none;}</style><div id=\"sk-container-id-7\" class=\"sk-top-container\"><div class=\"sk-text-repr-fallback\"><pre>KNeighborsClassifier(n_neighbors=4, weights=&#x27;distance&#x27;)</pre><b>In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. <br />On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.</b></div><div class=\"sk-container\" hidden><div class=\"sk-item\"><div class=\"sk-estimator sk-toggleable\"><input class=\"sk-toggleable__control sk-hidden--visually\" id=\"sk-estimator-id-13\" type=\"checkbox\" checked><label for=\"sk-estimator-id-13\" class=\"sk-toggleable__label sk-toggleable__label-arrow\">KNeighborsClassifier</label><div class=\"sk-toggleable__content\"><pre>KNeighborsClassifier(n_neighbors=4, weights=&#x27;distance&#x27;)</pre></div></div></div></div></div>"
      ],
      "text/plain": [
       "KNeighborsClassifier(n_neighbors=4, weights='distance')"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_model = grid_search.best_estimator_\n",
    "\n",
    "final_model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "fa01391d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.score(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "fe8e2d4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9714"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid_search.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e99b5c3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}