digit.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy\n",
    "from scipy import ndimage\n",
    "from keras.layers import Conv2D,MaxPool2D,LeakyReLU,Flatten,Dense,Input,Dropout,BatchNormalization\n",
    "from keras.models import Model\n",
    "import keras.backend as K\n",
    "from keras.losses import categorical_crossentropy\n",
    "from keras.optimizers import Nadam\n",
    "from keras.regularizers import l2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_one_hot(Y, C):\n",
    "    Y = Y.astype('int')\n",
    "    Y = np.eye(C)[Y.reshape(-1)].T\n",
    "    return Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(42000, 28, 28, 1)\n",
      "(42000, 10)\n",
      "(28000, 28, 28, 1)\n"
     ]
    }
   ],
   "source": [
    "data1 = pd.read_csv('train.csv')\n",
    "data2 = pd.read_csv('test.csv')\n",
    "data1 = np.array(data1)\n",
    "data2 = np.array(data2)\n",
    "X_train = data1[:,1::]/255\n",
    "X_test = data2[:,:]/255\n",
    "n1 = X_train.shape[0]\n",
    "n2 = X_test.shape[0]\n",
    "X_train = np.reshape(X_train,(n1,28,28,1))\n",
    "X_test = np.reshape(X_test,(n2,28,28,1))\n",
    "Y_train = data1[:,np.newaxis,0]\n",
    "Y_train = convert_to_one_hot(Y_train,10).T\n",
    "print(X_train.shape)\n",
    "print(Y_train.shape)\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MODEL():\n",
    "    X_in = Input(shape=(28,28,1),name='image_in')\n",
    "    \n",
    "    X = Conv2D(filters=32,kernel_size=3)(X_in)\n",
    "    X = BatchNormalization()(X)\n",
    "    X = LeakyReLU()(X)\n",
    "    X = Conv2D(filters=32,kernel_size=3)(X)\n",
    "    X = BatchNormalization()(X)\n",
    "    X = LeakyReLU()(X)\n",
    "    X = MaxPool2D()(X)\n",
    "    X = Dropout(0.4)(X)\n",
    "    \n",
    "    X = Conv2D(filters=64,kernel_size=3)(X)\n",
    "    X = BatchNormalization()(X)\n",
    "    X = LeakyReLU()(X)\n",
    "    X = Conv2D(filters=64,kernel_size=3)(X)\n",
    "    X = BatchNormalization()(X)\n",
    "    X = LeakyReLU()(X)\n",
    "    X = MaxPool2D()(X)\n",
    "    X = Dropout(0.4)(X)\n",
    "    \n",
    "    X = Flatten()(X)\n",
    "    \n",
    "    X = Dense(128)(X)\n",
    "    X = BatchNormalization()(X)\n",
    "    X = LeakyReLU()(X)\n",
    "    X = Dropout(0.4)(X)\n",
    "    \n",
    "    X = Dense(10,activation='softmax')(X)\n",
    "    \n",
    "    model = Model(inputs=X_in,outputs=X)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cat_loss(target, output, axis=-1):\n",
    "    output_dimensions = list(range(len(output.shape)))\n",
    "    output /= np.sum(output, axis)[:,None]\n",
    "    output = np.clip(output, K.epsilon(), 1. - K.epsilon())\n",
    "    return -np.mean(np.sum(target * np.log(output), axis))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def np_count(t, val):\n",
    "    elements_equal_to_value = np.equal(t, val)\n",
    "    as_ints = elements_equal_to_value.astype('int32')\n",
    "    count = np.sum(as_ints)\n",
    "    return count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0812 03:50:22.025803 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:517: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
      "\n",
      "W0812 03:50:22.038653 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:74: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
      "\n",
      "W0812 03:50:22.039813 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4138: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
      "\n",
      "W0812 03:50:22.063959 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:174: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
      "\n",
      "W0812 03:50:22.064640 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:181: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "W0812 03:50:22.119818 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1834: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
      "\n",
      "W0812 03:50:22.277062 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3976: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
      "\n",
      "W0812 03:50:22.281810 139822823315264 deprecation.py:506] From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0812 03:50:22.575020 139822823315264 deprecation_wrapper.py:119] From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "W0812 03:50:24.125295 139822823315264 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/25\n",
      "loss:  0.3242098718250098\n",
      "accuracy:  0.8994285714285715\n",
      "Epoch 2/25\n",
      "loss:  0.042437039355052456\n",
      "accuracy:  0.9860238095238095\n",
      "Epoch 3/25\n",
      "loss:  0.03174645437089109\n",
      "accuracy:  0.9900238095238095\n",
      "Epoch 4/25\n",
      "loss:  0.03157360568068273\n",
      "accuracy:  0.9902857142857143\n",
      "Epoch 5/25\n",
      "loss:  0.02124843103081334\n",
      "accuracy:  0.9932142857142857\n",
      "Epoch 6/25\n",
      "loss:  0.022035358601884294\n",
      "accuracy:  0.9926428571428572\n",
      "Epoch 7/25\n",
      "loss:  0.020102457681804006\n",
      "accuracy:  0.9935952380952381\n",
      "Epoch 8/25\n",
      "loss:  0.037136256465065694\n",
      "accuracy:  0.9883809523809524\n",
      "Epoch 9/25\n",
      "loss:  0.014930227841204423\n",
      "accuracy:  0.9953095238095238\n",
      "Epoch 10/25\n",
      "loss:  0.0177657240653505\n",
      "accuracy:  0.994452380952381\n",
      "Epoch 11/25\n",
      "loss:  0.013067151268744791\n",
      "accuracy:  0.9960476190476191\n",
      "Epoch 12/25\n",
      "loss:  0.013301449806587774\n",
      "accuracy:  0.9960238095238095\n",
      "Epoch 13/25\n",
      "loss:  0.011197312943970247\n",
      "accuracy:  0.9965238095238095\n",
      "Epoch 14/25\n",
      "loss:  0.013104387573912036\n",
      "accuracy:  0.9959761904761905\n",
      "Epoch 15/25\n",
      "loss:  0.011759610689138797\n",
      "accuracy:  0.9963095238095238\n",
      "Epoch 16/25\n",
      "loss:  0.012534730517723861\n",
      "accuracy:  0.9957857142857143\n",
      "Epoch 17/25\n",
      "loss:  0.012164903188395918\n",
      "accuracy:  0.9959285714285714\n",
      "Epoch 18/25\n",
      "loss:  0.00887552759086926\n",
      "accuracy:  0.9973333333333333\n",
      "Epoch 19/25\n",
      "loss:  0.010262696221701428\n",
      "accuracy:  0.9969523809523809\n",
      "Epoch 20/25\n",
      "loss:  0.00834436190268094\n",
      "accuracy:  0.9975238095238095\n",
      "Epoch 21/25\n",
      "loss:  0.007741484275715565\n",
      "accuracy:  0.9978809523809524\n",
      "Epoch 22/25\n",
      "loss:  0.007556295891869489\n",
      "accuracy:  0.9980476190476191\n",
      "Epoch 23/25\n",
      "loss:  0.0073828437101757\n",
      "accuracy:  0.9977380952380952\n",
      "Epoch 24/25\n",
      "loss:  0.009365213002137874\n",
      "accuracy:  0.997\n",
      "Epoch 25/25\n",
      "loss:  0.0068503289566867985\n",
      "accuracy:  0.9979047619047619\n"
     ]
    }
   ],
   "source": [
    "num_epochs = 25\n",
    "batch_step = -1\n",
    "min_test_loss=99.0\n",
    "loss_lst = []\n",
    "acc_lst = []\n",
    "indeces = np.array(range(X_train.shape[0]))\n",
    "num_batches = int(np.ceil(X_train.shape[0]/batch_size))\n",
    "model = MODEL()\n",
    "model.compile(optimizer=Nadam(),loss=categorical_crossentropy)\n",
    "for epoch in range(num_epochs):\n",
    "    np.random.shuffle(indeces)\n",
    "    X_train_s = np.copy(X_train[indeces])\n",
    "    Y_train_s = np.copy(Y_train[indeces])\n",
    "    print('Epoch {}/{}'.format(epoch+1,num_epochs))\n",
    "    for bi in range(num_batches):\n",
    "        batch_step += 1\n",
    "        a = bi*batch_size\n",
    "        b = (bi+1)*batch_size\n",
    "        batch_loss = model.train_on_batch(X_train_s[a:b],Y_train_s[a:b])\n",
    "    train_pred = model.predict(X_train)\n",
    "    loss = cat_loss(Y_train,train_pred)\n",
    "    y_hat = np.argmax(train_pred,1)\n",
    "    yy = np.argmax(Y_train,1)\n",
    "    correct_pred = np.equal(y_hat,yy)\n",
    "    cnt = np_count(correct_pred, True)\n",
    "    acc = cnt/X_train.shape[0]\n",
    "    print('loss: ',loss)\n",
    "    print('accuracy: ',acc)\n",
    "    loss_lst.append(loss)\n",
    "    acc_lst.append(acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f2a38041518>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHklJREFUeJzt3X2QXXWd5/H39z72vZ1+uEk6IeQRSJgljBikDWM5g0+g0Z0i7goYdtzBGnaDszDqurs1sDOLU8xapejA6hYKqKlBZzUy6mpqNlOYRRjHVTQdeUxiSBPz1EI6pNPpTvrpPnz3j3u6c9N0p293unM753xeVbfuuefh5ne4xef8+nfO+R5zd0REJDpitW6AiIicXwp+EZGIUfCLiESMgl9EJGIU/CIiEaPgFxGJGAW/iEjEKPhFRCJGwS8iEjGJWjdgtPnz5/uKFStq3QwRkQvKjh07Xnf3lmrWnXXBv2LFCtra2mrdDBGRC4qZHah2XQ31iIhEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxCj4RUQiRsEvIhIxoQn+3oE8D257mecOdde6KSIis1pogr9Ugi8+uZcdB47XuikiIrNaaIK/oS5BzKC7b6jWTRERmdVCE/yxmNGUSdLdl691U0REZrXQBD9AczbFcfX4RUTOKmTBn+REv3r8IiJnE67gzyTV4xcRmUCogj+XTWmMX0RkAqEK/qasTu6KiEwkVMGfy6Y4OVggXyzVuikiIrNWqIK/OZsEUK9fROQsQhb8KQBO9OsEr4jIeMIV/Jlyj/+4evwiIuOqKvjNbJ2Z7TGzdjO7e4zlHzOzF83sOTP7qZmtrlh2T7DdHjN733Q2frRc0OPXUI+IyPgmDH4ziwMPAe8HVgO3VgZ74Fvu/iZ3XwPcDzwQbLsa2ABcCawDvhx834wYHuPXtfwiIuOrpse/Fmh3933uPgRsBtZXruDuPRUf6wEPptcDm9190N1/A7QH3zcjhoP/hHr8IiLjSlSxzmLgUMXnw8C1o1cyszuBTwEp4N0V2z4zatvFY2y7EdgIsGzZsmraPaY56QSJmKnHLyJyFtN2ctfdH3L3y4A/B/5ykts+6u6t7t7a0tIy5TaYGc3ZJN2q1yMiMq5qgr8DWFrxeUkwbzybgQ9OcdtzVi7NrB6/iMh4qgn+7cAqM7vEzFKUT9ZuqVzBzFZVfPyXwN5geguwwczSZnYJsAr45bk3e3yq1yMicnYTjvG7e8HM7gKeAOLAJnffaWb3AW3uvgW4y8yuB/LAceC2YNudZvY4sAsoAHe6e3GG9gUon+Dt6B6YyX9CROSCVs3JXdx9K7B11Lx7K6Y/cZZtPwN8ZqoNnKzmbIpdv+2ZeEURkYgK1Z27MFyTX0M9IiLjCV3w5+pT9OeLDORndERJROSCFbrgbwrq9egRjCIiYwtd8Ktej4jI2YUu+FWvR0Tk7EIb/Orxi4iMLYTBPzzUox6/iMhYQhf8ueEev07uioiMKXTBn0nGScVjGuMXERlH6IJ/uEKnavKLiIwtdMEP5RO86vGLiIwtpMGvCp0iIuMJZ/Bnkgp+EZFxhDL4c9kU3f0a6hERGUsog788xp/H3SdeWUQkYkIa/CmGCiUG8qVaN0VEZNYJafCrXo+IyHhCGfw51esRERlXKIO/KaN6PSIi4wll8OfqVa9HRGQ8oQz+5qDHrzF+EZE3Cmfwa4xfRGRcVQW/ma0zsz1m1m5md4+x/FNmtsvMXjCzJ81secWyopk9F7y2TGfjx1OXjFOXjGmMX0RkDImJVjCzOPAQcANwGNhuZlvcfVfFas8Cre7eZ2Z/CtwPfDhY1u/ua6a53RPKqV6PiMiYqunxrwXa3X2fuw8Bm4H1lSu4+1Pu3hd8fAZYMr3NnLymTPnuXREROVM1wb8YOFTx+XAwbzy3A/9Y8bnOzNrM7Bkz++BYG5jZxmCdtqNHj1bRpInlsilOqF6PiMgbTDjUMxlm9hGgFXhHxezl7t5hZpcCPzazF939lcrt3P1R4FGA1tbWaSmw05xNsrfz5HR8lYhIqFTT4+8AllZ8XhLMO4OZXQ/8BXCjuw8Oz3f3juB9H/A0cPU5tLdqqskvIjK2aoJ/O7DKzC4xsxSwATjj6hwzuxp4hHLod1bMz5lZOpieD7wdqDwpPGOas0m6+4ZUoVNEZJQJh3rcvWBmdwFPAHFgk7vvNLP7gDZ33wJ8HpgD/L2ZARx09xuBK4BHzKxE+SDz2VFXA82YXDZJoeScGioyJz2tI1oiIhe0qhLR3bcCW0fNu7di+vpxtvsZ8KZzaeBUjdy9e2pIwS8iUiGUd+7C6bt3T6hej4jIGUIc/KrXIyIyltAGv2ryi4iMLbTB3zQS/Orxi4hUCm3wN488jEU9fhGRSqEN/lQiRn0qroexiIiMEtrgh/IJXp3cFRE5U8iDP8kJDfWIiJwh1MGfU49fROQNQh38TdmkxvhFREYJdfDnskld1SMiMkqog785k6K7b4hSSRU6RUSGhTv4s0lKDr2DhVo3RURk1gh58Jdv4tKVPSIip4U6+Ifr9ejKHhGR00Id/MOlmXVlj4jIaSEP/uF6Perxi4gMC3fwZ1SaWURktFAHf1NGY/wiIqOFOvgT8RgNdQn1+EVEKoQ6+KFcr0dj/CIip4U++JtVr0dE5AxVBb+ZrTOzPWbWbmZ3j7H8U2a2y8xeMLMnzWx5xbLbzGxv8LptOhtfjXJNfgW/iMiwCYPfzOLAQ8D7gdXArWa2etRqzwKt7n4V8F3g/mDbucCngWuBtcCnzSw3fc2fWHMmyQkN9YiIjKimx78WaHf3fe4+BGwG1leu4O5PuXtf8PEZYEkw/T5gm7t3uftxYBuwbnqaXp1cNqkev4hIhWqCfzFwqOLz4WDeeG4H/nGK2067pmyKnoE8RVXoFBEBIDGdX2ZmHwFagXdMcruNwEaAZcuWTWeTyGWTuENPf55cfWpav1tE5EJUTY+/A1ha8XlJMO8MZnY98BfAje4+OJlt3f1Rd29199aWlpZq214V1esRETlTNcG/HVhlZpeYWQrYAGypXMHMrgYeoRz6nRWLngDea2a54KTue4N5501zptzL1927IiJlEw71uHvBzO6iHNhxYJO77zSz+4A2d98CfB6YA/y9mQEcdPcb3b3LzP6a8sED4D5375qRPRnHcI9fNflFRMqqGuN3963A1lHz7q2Yvv4s224CNk21gedquEKnevwiImWhv3N3+GEsqtcjIlIW+uBvqEtippr8IiLDQh/88ZjRlFG9HhGRYaEPfiiXbdDduyIiZdEIfpVmFhEZEZHgT+rkrohIIBLBn8um6O5Xj19EBCIS/E2ZJN2n1OMXEYGIBH8um6J3sEC+WKp1U0REai4SwT9StkGXdIqIRCv4dYJXRCQywV+u16NLOkVEIhL8qtcjInJaJIJfNflFRE6LRvDX6+SuiMiwSAR/QzpBPGbq8YuIEJHgNzOaMyrbICICEQl+gCbV6xERASIU/KrXIyJSFpngb84kOa56PSIiEQr+bEpX9YiIEKngT+qqHhERqgx+M1tnZnvMrN3M7h5j+XVm9iszK5jZTaOWFc3sueC1ZboaPlm5bJK+oSKDhWKtmiAiMiskJlrBzOLAQ8ANwGFgu5ltcfddFasdBD4K/OcxvqLf3ddMQ1vPSVNQr+dEX54FjfEat0ZEpHaq6fGvBdrdfZ+7DwGbgfWVK7j7fnd/AZi1Be9H6vVonF9EIq6a4F8MHKr4fDiYV606M2szs2fM7IOTat00GqnXc0rj/CISbRMO9UyD5e7eYWaXAj82sxfd/ZXKFcxsI7ARYNmyZTPSiGb1+EVEgOp6/B3A0orPS4J5VXH3juB9H/A0cPUY6zzq7q3u3trS0lLtV0/K6YexqMcvItFWTfBvB1aZ2SVmlgI2AFVdnWNmOTNLB9PzgbcDu86+1czIjTyMRT1+EYm2CYPf3QvAXcATwG7gcXffaWb3mdmNAGb2VjM7DNwMPGJmO4PNrwDazOx54Cngs6OuBjpvsqk4ybhxXMEvIhFX1Ri/u28Fto6ad2/F9HbKQ0Cjt/sZ8KZzbOO0MLPg7l0N9YhItEXmzl1QvR4REYhY8KtCp4hIxIJfNflFRCIW/DkFv4hItIK/OZtShU4RibyIBX+SwUKJgbwqdIpIdEUr+Ifr9ajXLyIRFqngH6nQqXF+EYmwSAV/UxD86vGLSJRFKvhzFQ9jERGJqkgFf/NIj1/BLyLRFangH6nQqbt3RSTCIhX8dck46URMJ3dFJNIiFfwQ1OvRyV0RibDIBX9zNqkxfhGJtEgGv67qEZEoi17wZ1SvR0SiLXLBn6tP0t2vHr+IRFfkgr8pUz656+61boqISE1ELvhz2ST5otM3pAqdIhJNkQv+ZtXrEZGIi2DwB3fv6soeEYmoqoLfzNaZ2R4zazezu8dYfp2Z/crMCmZ206hlt5nZ3uB123Q1fKqaMyrNLCLRNmHwm1kceAh4P7AauNXMVo9a7SDwUeBbo7adC3wauBZYC3zazHLn3uypy9WrXo+IRFs1Pf61QLu773P3IWAzsL5yBXff7+4vAKVR274P2ObuXe5+HNgGrJuGdk/ZcI9fd++KSFRVE/yLgUMVnw8H86pxLtvOiOGHsZzQyV0RiahZcXLXzDaaWZuZtR09enRG/610Ik42FdcYv4hEVjXB3wEsrfi8JJhXjaq2dfdH3b3V3VtbWlqq/Oqpy2VTGuoRkciqJvi3A6vM7BIzSwEbgC1Vfv8TwHvNLBec1H1vMK+mmjJJTujkrohE1ITB7+4F4C7Kgb0beNzdd5rZfWZ2I4CZvdXMDgM3A4+Y2c5g2y7grykfPLYD9wXzaipXr9LMIhJdiWpWcvetwNZR8+6tmN5OeRhnrG03AZvOoY3TrjmT4tcnemrdDBGRmpgVJ3fPt+ZsUid3RSSyohv8/XlV6BSRSIpk8OeyKYolp3ewUOumiIicd5EM/qbM8E1cGu4RkeiJZPDnggqdKs0sIlEUyeAfrsmvE7wiEkURDX71+EUkuiIa/MEYvx66LiIRFM3gHy7NfErBLyLRE8ngT8RjNKQTehiLiERSJIMfoLled++KSDRFN/gzKbp1cldEIii6wZ9VhU4RiaYIB39KV/WISCRFNvhz2aSu4xeRSIps8Ddnkpzoz1MqqUKniERLdIM/m8IdegY03CMi0RLh4Fe9HhGJpsgGvyp0ikhURTb4m4Z7/LqyR0QiJrLBP9zj101cIhI1kQ3+4UJtGuMXkaipKvjNbJ2Z7TGzdjO7e4zlaTP7TrD8F2a2Ipi/wsz6zey54PXw9DZ/6hozSczQ3bsiEjmJiVYwszjwEHADcBjYbmZb3H1XxWq3A8fdfaWZbQA+B3w4WPaKu6+Z5nafs3jMaKxLckJDPSISMdX0+NcC7e6+z92HgM3A+lHrrAceC6a/C7zHzGz6mjkzcqrXIyIRVE3wLwYOVXw+HMwbcx13LwAngHnBskvM7Fkz+ycz+4NzbO+0asqmdFWPiETOhEM95+hVYJm7HzOza4AfmNmV7t5TuZKZbQQ2AixbtmyGm3RaLpuk65SGekQkWqrp8XcASys+LwnmjbmOmSWAJuCYuw+6+zEAd98BvAJcPvofcPdH3b3V3VtbWlomvxdT1JzRw1hEJHqqCf7twCozu8TMUsAGYMuodbYAtwXTNwE/dnc3s5bg5DBmdimwCtg3PU0/d83ZlO7cFZHImXCox90LZnYX8AQQBza5+04zuw9oc/ctwNeBb5pZO9BF+eAAcB1wn5nlgRLwMXfvmokdmYrmbJLegQKFYolEPLK3NIhIxFQ1xu/uW4Gto+bdWzE9ANw8xnbfA753jm2cMfPmpAH40pN72fiOy5iTnulTHiIitRfpbu76NRez7sqL+NKP27nu/qf42j/vYyBfrHWzRERmVKSDv7EuycP/9hp+eOfbufLiRv77/9nNu77wNN/+5UEKxVKtmyciMiMiHfzD3ry0mW/efi3f+vfXclFTHfd8/0VuePAnbHn+t3pCV4Wdvz3BPd9/kc7egVo3RUTOgbnPrmBrbW31tra2mv377s6Tuzv5wo/28OvXerliUSP/5X2X867fWcAFcDPyjHli52t8cvNz9OeLXLGoke/c8Xs01iVr3SwRCZjZDndvrWZd9fhHMTOuX72QrR//A764YQ19QwX+5G/buOnhn/PMvmO1bt555+58+el27vjmDn7nogYeuOXNtHf28u8ea9P5EJELlHr8E8gXSzzedogvPbmXIz2DXHd5Cx96y2ISsRgxg1jMiJmVp82w4D1mRixWnk7EjN9d3ERdMl7r3ZmUwUKRe77/It//VQc3vvli7r/pKuqScbY8/1s+sflZrr9iIV/5o7foUliRWWAyPX4Ff5UG8kW+8fP9fPnpV6Z0t+/CxjT/4Z0r2bB2KenE7D8AvH5ykI99cwdtB47zn264nLvevfKMoa5v/Hw/9/5wJ7e0LuFzH7oq0sNgIrOBgn8G9Q0V6DjeT8mh5E7JHQ+miyWn5OXhkcrlPf15Nv10P7/c38WipjrufNdKbmldSioxO3vKv36th9v/to1jpwZ54JY1fOBNi8Zc78FtL/PFJ/dyxzsu5Z73X3GeWykilSYT/LpjaZKyqQSrFjZMerv3XXkR/6/9GA9s28Nf/uAlvvL0K/zZu1fyoWuWkJxFQyVP7j7Cx7/9LHPqEjx+x9u4aknzuOt+8vpVdJ0a4pF/2se8+hQbr7vsPLZURKZKPf7zzN35yd7XeWDbyzx/qJulczP82btX8a+vXlzTsXJ35+s//Q2f2bqbKy9u5Gt//FYuaqqbcLtiyfnE5mf5hxde5fM3XcXNrUsn3EZEpp+Gei4A7s5Tezp5YNvLvNTRw4p5WT7+nlWsX7OYeOz8jpcPFUr8tx+8xHfaDvGBN13E39y8hkyq+vMQQ4UStz+2nZ+9coyHP3INN6xeOIOtFZGxKPgvIO7Otl1HePD/7mX3qz1c2lLPJ96zij+86uI3HABKJWegUGQgX6I/X6R/qMhAvjgynYgZLQ1pFjTU0ZhJVHXCtevUEB/7ux388jddfPzdK/nk9ZcTm8KB59RggX/z1Wf49Wu9fONP1nLtpfMm3khEpo2C/wJUKjk/2vUaD27by54jvSxqqiOTjJdDPQj2wUL1ZSRS8RgtDekzXguGp+ekWdBYR7FU4j9+53le6xng8zddxfo1ox+sNjldp4a4+eGf0dkzyHfueBurL248p+8Tkeop+C9gpZKz9aVX+YfnXyURNzLJOJlUnEwyTl0wXZeIld+T8TOWDxVLHO0dLL9ODnK0J3jvHaSzd3DMp43Nn5Pmq398DVcvy01L+zu6+7npKz8jX3S+96dvY/m8+mn5XhE5OwW/jClfLHHs5BCdvQMcDQ4E113ewsLGiU/iTkZ7Zy83PfxzGuuSfPdjb2PBBN9/arDAgWN9HOw6xf5jfRzs6mNBQ5rW5XO5elkz9SqXLTIhBb/U3LMHj/NHX/sFy+fVs3nj7+HuHDjWx/5jpzh4rI/9x/o4cOwUB7r6ONo7eMa2zdkkJ/rzuEPMYPXFjbQun8s1y3O0rsixqClTo70Smb0U/DIr/OTlo9z+2HYMY2hUmeuLGutYPi/Linn1LAvel8/Lsmxelsa6JD0DeZ492M2O/V20HTjOswe76Q9qAy1uzowcBK5ZnuNfXNR43q+EEpltFPwya/zk5aM8ufsIS+dmWTY3y4r59Sybm5103aJCscTuV3vZvr+LHQeO03agiyM95b8U5qQTXHlxI6lE7PQd1KXgbmp3SqXye7HEyHSp5CTjMRY0prmosY6FjXUsbKpjYUOai5rKn+fPSU94QHF3uvvyI+dSzjjHEgyn1afjNGWSNNYlacwkaRrn1ZhJvuHfyxdLI1duDQyVRk72n55XfneHRNxIxGIk4kYybsRjMZIxIxGPEY8NzzOS8RjpRIxFTZlZe/e4TJ6CX0LP3Tl8vH/kILDrtz2UHOIxIx4UyIsHBfTiI4X0jHjF/MFCic6eAY4EJ8GLo569EDNoaSgfGBY01rGwMU2h6GcE++snB8kX3/j/UDpRvqoql03Rny9yoj/Pif48QxNcmdWQTpBOxhkMgr0wg8+DiMeM5fOyrGyZw6qFc1i5YA4rWxq4bEE92ZTOq1xoFPwik1QsOcdODvJacCA40jMw8nqtZzA4QAyQiMdomXPmZbJv+NyQpiE99n0UAxUHgZ7gffRrIF8KrtaKUZeIn3EFV93w/IoruuoSccygUHIKRSdfLFEsOYVSiXyx/BdQvliiUPTyOqUS/UNFDhzro73zJO1HT7L/9VNnHGQWN2dYuWAOqxYEB4QFc1jYWEfvQCFo5xAn+vN095Xb3D3c/r7T+9HdN0RyjMuKK/97LWioo6UhTWNddfedzGYD+SKdPYN09g7Q2TvIqcECi5oyLMllWNRcN+PFGVWrR2SS4jFjQdCzn0l1QXhP95VU5ypfLHHg2CnaO0+y90j5YNDeeZJf/OYYA/mz/5WSiBnN2dPDWPPnpLispZ6mTJJ8yUcuJ9539BRHewffcL4HIJUoH1DnN6SpT8VJJcrDUalEnPTIdIx04vSy4VcyHhspjx6PDZdHL5dKj1dOB+vYqOnKvwbNziyzPlxeHcr3qRztHTwj3CunewcK4/43MoMFDWmW5LIsbi4fDBbnMizJZcvTzZnzWrZdPX4RGVep5HR099PeeZKjvYM0ZhI0ZpI0Z1I0ZZM0Z5JkU/Gqe+vuTk9/gaMny2E51n0nA/kiQ4USg4XSyHv5VRyZV2vpRPn80IKGuuCGyPINkgsa6mgJpuekE7x6YoDDx/s5fLyPjuP95enuPl7tHnjDMN78OWnedtk8/uetV0+pTerxi8i0iMWMpXOzLJ2bnZbvMzOaskmasklWLph8lVsoHzyGiqcPCsNDW2OVRy+fyD9dIr3kBOueWTq9cp3xyqznskkWNKZpaairemhqvBsYiyXnSM8AHd3lg8Lhrn46uvuZW5+a0n+Tyaoq+M1sHfBFIA58zd0/O2p5GvgGcA1wDPiwu+8Plt0D3A4UgY+7+xPT1noRiRwzI52Ik07Emdqho/biMePi5gwXN2d464q55/3fn/BaLjOLAw8B7wdWA7ea2epRq90OHHf3lcCDwOeCbVcDG4ArgXXAl4PvExGRGqnmIt61QLu773P3IWAzsH7UOuuBx4Lp7wLvsfLfQeuBze4+6O6/AdqD7xMRkRqpJvgXA4cqPh8O5o25jrsXgBPAvCq3xcw2mlmbmbUdPXq0+taLiMikzYrb9tz9UXdvdffWlpaWWjdHRCTUqgn+DqDyeXpLgnljrmNmCaCJ8knearYVEZHzqJrg3w6sMrNLzCxF+WTtllHrbAFuC6ZvAn7s5RsEtgAbzCxtZpcAq4BfTk/TRURkKia8nNPdC2Z2F/AE5cs5N7n7TjO7D2hz9y3A14Fvmlk70EX54ECw3uPALqAA3OnuxRnaFxERqYLu3BURCYELukibmR0FDpzDV8wHXp+m5lxotO/RFeX9j/K+w+n9X+7uVV0dM+uC/1yZWVu1R72w0b5Hc98h2vsf5X2Hqe3/rLicU0REzh8Fv4hIxIQx+B+tdQNqSPseXVHe/yjvO0xh/0M3xi8iImcXxh6/iIicRWiC38zWmdkeM2s3s7tr3Z7zzcz2m9mLZvacmYX6Rggz22RmnWb2UsW8uWa2zcz2Bu+5WrZxJo2z/39lZh3B7/+cmX2glm2cKWa21MyeMrNdZrbTzD4RzA/973+WfZ/0bx+KoZ6gxv/LwA2UK4BuB2519101bdh5ZGb7gVZ3D/31zGZ2HXAS+Ia7/24w736gy90/Gxz4c+7+57Vs50wZZ///Cjjp7l+oZdtmmpktAha5+6/MrAHYAXwQ+Cgh//3Psu+3MMnfPiw9/mqeGSAh4e4/oVwapFLlMyEeo/w/RCiNs/+R4O6vuvuvguleYDflUu+h//3Psu+TFpbgr6ruf8g58CMz22FmG2vdmBpY6O6vBtOvAQtr2ZgaucvMXgiGgkI31DGama0ArgZ+QcR+/1H7DpP87cMS/AK/7+5vofyIzDuD4YBICirDXvhjmJPzFeAyYA3wKvA3tW3OzDKzOcD3gE+6e0/lsrD//mPs+6R/+7AEf+Tr/rt7R/DeCfxvoveIyyPBGOjwWGhnjdtzXrn7EXcvunsJ+Coh/v3NLEk5+P6Xu38/mB2J33+sfZ/Kbx+W4K/mmQGhZWb1wckezKweeC/w0tm3Cp3KZ0LcBvywhm0574ZDL/CvCOnvHzzL++vAbnd/oGJR6H//8fZ9Kr99KK7qAQguYfofnH5mwGdq3KTzxswupdzLh/IzFr4V5v03s28D76RclfAI8GngB8DjwDLK1V1vcfdQngAdZ//fSflPfQf2A3dUjHmHhpn9PvDPwItAKZj9XymPdYf69z/Lvt/KJH/70AS/iIhUJyxDPSIiUiUFv4hIxCj4RUQiRsEvIhIxCn4RkYhR8IuIRIyCX0QkYhT8IiIR8/8BaB5VqHk+dgwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f29e879a8d0>]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAHoFJREFUeJzt3X2UXHWd5/H3t6u6+jndne4mQJ6DQY2IEJooKCbj2YEwMwcEVxd0Z0DnmDkqe3bOGdyFcUbcuAyzM7jrcWV1UFnBJxYZRdxhFxnAh1Fw00DCo4EQIEkHkk5S1Ul3VXc9ffePut2pdPqh0qmkmns/r3Pq9K3fvVX9uyn41K9/997vNXdHRESio67WHRARkZNLwS8iEjEKfhGRiFHwi4hEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxCj4RUQiJl7rDkzU3d3ty5Ytq3U3RETeVJ544ol97t5TybZzLviXLVtGX19frbshIvKmYmavVbqtpnpERCJmxuA3szvMbK+ZPTvFejOzr5jZNjN72sxWl627xsxeCh7XVLPjIiIyO5WM+L8NrJ9m/aXAyuCxAfgagJnNB24C3g2sAW4ys87j6ayIiBy/GYPf3X8JHJhmk8uBu7zkcaDDzE4DLgEecvcD7p4EHmL6LxARETkJqjHHvxDYWfZ8V9A2VftRzGyDmfWZWd/AwEAVuiQiIlOZEwd33f12d+91996enorORhIRkVmqRvD3A4vLni8K2qZqFxGRGqrGefz3A9eZ2d2UDuQOuvvrZvYg8DdlB3QvBm6swu8TkYjLFYoMjeQ5NJJnOJsnmy+SLRSP/Bk8coXDbaNBW7zOaG6I09oQozkRp7UhTnMiRktDsNwQo7UhTlN9DDM74fszmi+w80CGkVyBsxa2n/DfN2Pwm9kPgHVAt5ntonSmTj2Au38deAD4A2AbkAY+Hqw7YGZfBDYFb7XR3ac7SCwic5S7c2g0T0siTqyuOkFYKDoHhrPsGxodf+wfynIwk+PgSJ6h0TyHRnLBzzxDI/mgPcdIrliVPszEDFoScVoaYpzS1sjpHY2c3tHEwo4mTu9o4rT2RhZ2NNHd2kDdDP8u2XyRHQfSvLZ/mFf2DfPq/mFe25/mlX3D7E5lKDqcs7iD+z7z3hO/X3PtZuu9vb2uK3dFauvQSI4tOwd5akeSJ3ckeWpnilQ6B0BrQ5x5jXHaGuuZ11T62dYYZ97Yz6bSz7bGenL5YlmwZxk4dDjkDwxnKU4SP2al39HWUHqP1sbSKLz0nkFb8Ly1IU5LQ5yGeB2JeB2JWB31wc/xtngd9bHD6xOxOgrupEcLDGXzpEdLXzLpbIGh0TzDo3mGswWGR8fWFRgazbHn4Ci7Uxn6UxnS2cIRfa6PGae1N41/MZze3kRbY5xdyQyvBkE/Fu5j2hrjLO9uYVlXC8u6mlnW3cJbTmnl7EUds/rMzOwJd++tZNs5V7JBZC4YyRU4MJzlwHCWZDo7vpyI19Hd2kB3a4Kulga6WhO0NsSrNh3g7rgz4+ixmtyd7fuGefK1JE/uSPHUjiRb9xxibEy48pRWLll1Kit6WkhnCxwayXNwJMehkRwHM3n2Hhph297S6PzQSJ78JGneWD/279bAos5mzl3SMf68p63h8L9pawNtDfETvv91GO3NdbQ31x/za92dg5k8/akMu1MZdg9m2J0aKS2nMjz+8n7eODhC0Q+H++olnVx57kKWdbewtKuF5d0tdDbXn5RppMko+OVNbXg0T99rSZ7ZlSJfdAzDDIzSyHHsf6xSW2kdlNbnCkUODOdIprPsH86SLAv6iSO66STidXS3JOhua6CrpRReXa0JulsaaG+uZzRfZHi0NFUxNqIcKnscsS5boOhOR1M981sOf7mUlks/57c20N2SYH7QPr85QTxWR6Ho5ArF4FFaHpvjzhd9fDlXcIZH8zzbP3jUaL6tMc45izu45B2nsnppJ+cs7qC9qfJwdHcyueDLIZMjHqujp62BlsTJmSs/GcyM9uZ62pvrWXX6vEm3yReKDGcLzGus3qCgmhT8csK5e9X+489kC/S9doDHt+/nsZf38/SuwUlHmJVqbYjT2VLP/OYE3a0JVi5oZX5zgs6xkA0enc0JOpvryRW8NBc9nGXfoVH2D5fmpfcNZdk/PMrA0Ci/e+MQ+4eyZAtHzkPXGbQEUxgtDfHxKYwFbY3jy60NceoMDqSz7B8qfSG9tHdo/AtpqpnZOmPSaZOZjI3mVy/tYPWSTs7oaT2u0baZ0ZyI05yIs2Be46zf580uHqujvWlOnC0/KQW/nDCFovPZH27hn555nYWdTSyd38zSrhaWzG9myfxmlnY1s3h+M431sSnfYyRX4MnXkjy2fT+Pb9/P5p0pcgUnVmecvaidT75/BRes6OK8pZ001cdwgukSwB0cPyIsy9viMaMhPvXvnsqp7TMH2tjB0NRwjsZEXVXOECkUnVQw7bRvaGz6qfQlVCg69bE64jErzXMHjyOf23h7Q30dZy5oO6bRvISHgl9OCHfnr3/yLD96qp/Lzzmd0VzpjIZNryYZGs0fse2CeQ0snd/Ckq7SF8KiziZe3Z8uBf2OFNlCkTqDdy7q4BPvW84FK7roXTaf1oap/vOt/Z/WZsa8xnrmNVYvWGN1FkwjNbByQdXeViJIwS8nxH976EW+/9sdfOb3zuCzl7xtvN29dArfjgPp4NS20s8d+9P8y0v7eOPgCFCaujhrYTvXvndZEPSdtFUxREWiTMEfIe7OnoOjbN83xPaBYfpTGda/41TetXh2p49N5du/foWvPLKNq85fzPUXv/WIdWaHR63nLjm6WOtIrsCuZIaetgZNQ4icIAr+OSaVzvKdx17jn3+3l3mN8fHT3I469a3t8NkcEx0ayfHqvjTb9w3x8kDpHOLtA0O8sm/4qLNVvvWrV/ibK9/Jvz5vUVX6/5PN/Xzhp89z8aoF/OcPnnXMc9qN9THeckprVfoiIpNT8M8R/akM3/rVK9y9aQfpbIHzlnZyaCTP9oFh9g2NMpo/+kpFM4KzUUpfBPmC88q+YfYeGj1im0WdTSzvbuX8ZfNZ0dPCiu5Wlve00FQf47rvP8n1P9zC1jcOcsOlbz+uqzJ/8eIAf3HPFt69fD5fufrcSb+URKT2FPw1tvWNQ/zDL17m/i27AbjsXaezYe0K3nbq4fOD3Z2h0Tz7hoLL2w+VThss/Tx8ybsB7z+zh+XdLZzR08Ly7laWdk1/1sydn1jDF//383zjV6/w0t4hvnL1ubM6ILl5Z4pPffcJVi5o4xvX9E77O0WktlSyoQbcnf/3ygG+/ouXeXTrAM2JGFedv4Q/vWg5CzuaatKn7/32NW76yXMs7Wrmm9ecz/Lulopfu23vEB/++m9oa6zn3k9dwClt0T1/W6RWVLJhjioWnZ89v4ev/+JlNu9M0dWS4C9+/0z++IKldDQnatq3j717KSu6W/n0957gg7f9mts+upr3reye8XWvD2b4k2/9llhdHd/50zUKfZE3AY34T4LRfIEfP9nP7b/czvZ9wyyZ38wn37+CD5+3aM5NiezYn+aTd/WxbWCIv/7Dt3PNhcumPECbHM7y4X94jD2DI/xgw3tOSjlZEZmcRvwn0OadKfpePUA6WyCTK5DJFkhn82RyRTLZPJlcobQuWF8qalUqI/vOhe3c9tHVrD/r1KqVtq22JV3N/OOnL+TP797MF376PFv3HOI/XXYWifiRB2rT2TyfuHMTOw6kufPjaxT6Im8iCv4KDWZy/Jf/+zu+/9sd422JWB2N9XVBbZIYjfUxmhOlGzj0tDbQlCg9b07E+cDbTuHCM7rmZMGmiVob4tz+x+fxpYe2ctujL/Py3mG+9m9X09XaAJSKm33qu0+yZWeK//Gx1VxwRleNeywix0LBPwN354Fn3uALP32O/UOjfPKi5Xxq3VuY1xgP9emKdXXGZy95G2cuaOM/3Ps0l33113zzml7euqCNz/5wC794cYBbrnwn6886rdZdFZFjpOCfRn8qw+fve5aHf7eXsxbO439ee37kpjQuP2chy7pa2PCdPj70td/w3rd089Dze/jsJW/l6jVLat09EZkFBf8kCkXnzt+8yq0/24o7/NUfvp1rL1wW6hH+dN61uIP7r3sfG77zBA89v4drL1zGp9edUetuicgsKfgneG73IDf+6Bme3jXIurf28MXLz2Lx/OZad6vmFsxr5H9teA+bXj3Ae8/oflMcqxCRySn4A5lsgS//84t8819eobO5nv9+9bn80dmnKeDKNNbHuGhlT627ISLHScFPqcbMX933DDsPZLjq/MXccOnban5BlYjIiRLp4E9n8/zlj57hvs27WdHTwt0b3sN7VujURBEJt0gH/0827+a+zbv5zO+dwb/7wMo5dxWtiMiJEOngHwjKF//5vzqT+oiesSMi0RPptEums7Q2xBX6IhIpkU68VDpHR7Nu7yci0RLp4E+ms3Tq7B0RiZhIB79G/CISRREP/qzO1xeRyIl08CfTOTo14heRiIls8BeKzsGRnEb8IhI5FQW/ma03s61mts3Mbphk/VIze9jMnjazn5vZorJ1f2dmz5nZC2b2FZsjxW8OZnK4Q0eTRvwiEi0zBr+ZxYDbgEuBVcDVZrZqwma3Ane5+9nARuCW4LUXAu8FzgbOAs4H1lat98chmc4C0Nmi4BeRaKlkxL8G2Obu2909C9wNXD5hm1XAI8Hyo2XrHWgEEkADUA/sOd5OV0MynQPQVI+IRE4lwb8Q2Fn2fFfQVm4LcGWwfAXQZmZd7v4YpS+C14PHg+7+wsRfYGYbzKzPzPoGBgaOdR9mZTBTGvFrqkdEoqZaB3evB9aa2VOUpnL6gYKZvQV4O7CI0pfFB8zsookvdvfb3b3X3Xt7ek5OvffkcGnErwu4RCRqKinS1g8sLnu+KGgb5+67CUb8ZtYKfMjdU2b2SeBxdx8K1v0f4ALgV1Xo+3EZn+NX8ItIxFQy4t8ErDSz5WaWAK4C7i/fwMy6zWzsvW4E7giWd1D6SyBuZvWU/ho4aqqnFgYzOeoM2hojXaBURCJoxuB39zxwHfAgpdC+x92fM7ONZnZZsNk6YKuZvQgsAG4O2u8FXgaeoXQcYIu7/7S6uzA7yXSW9qZ66urmxNmlIiInTUXDXXd/AHhgQtvny5bvpRTyE19XAP7sOPt4QpSu2tU0j4hET2Sv3B1UgTYRiajIBn9SBdpEJKIiG/wqySwiURXZ4NdNWEQkqiIZ/KP5AulsQVftikgkRTL4B8fq9LRoxC8i0RPJ4B8r0KabsIhIFEUy+FMq1yAiERbJ4B8b8bdrjl9EIiiSwT8+4tccv4hEUDSDP6M5fhGJrkgGfzKdJRGro6k+VuuuiIicdJEM/tRw6ardOXLfdxGRkyqSwa+rdkUkyiIZ/KlMjnbN74tIREUz+NNZHdgVkciKZPDrJiwiEmWRC353D27CouAXkWiKXPCnswWyhaJq8YtIZEUu+JPjdXoU/CISTZEL/tRYSWZN9YhIREU3+FWgTUQiKnLBn1SBNhGJuMgF/1hlTh3cFZGoimDwj031aMQvItEUueBPpnO0JGIk4pHbdRERIILBn0pndUaPiERa9II/k6OzRfP7IhJdkQv+ZDqr+X0RibTIBX8qndMZPSISaREMft2ERUSiraLgN7P1ZrbVzLaZ2Q2TrF9qZg+b2dNm9nMzW1S2bomZ/czMXjCz581sWfW6f2yKRWcwoxG/iETbjMFvZjHgNuBSYBVwtZmtmrDZrcBd7n42sBG4pWzdXcDfu/vbgTXA3mp0fDYOjuQouur0iEi0VTLiXwNsc/ft7p4F7gYun7DNKuCRYPnRsfXBF0Tc3R8CcPchd09XpeezkAwu3lJlThGJskqCfyGws+z5rqCt3BbgymD5CqDNzLqAM4GUmf3IzJ4ys78P/oKoCZVrEBGp3sHd64G1ZvYUsBboBwpAHLgoWH8+sAK4duKLzWyDmfWZWd/AwECVunQ0lWQWEaks+PuBxWXPFwVt49x9t7tf6e7nAp8L2lKU/jrYHEwT5YH7gNUTf4G73+7uve7e29PTM8tdmdnhm7Ao+EUkuioJ/k3ASjNbbmYJ4Crg/vINzKzbzMbe60bgjrLXdpjZWJp/AHj++Ls9OynN8YuIzBz8wUj9OuBB4AXgHnd/zsw2mtllwWbrgK1m9iKwALg5eG2B0jTPw2b2DGDAN6q+FxVKpbOYQVujgl9EoiteyUbu/gDwwIS2z5ct3wvcO8VrHwLOPo4+Vk0ynaO9qZ5YndW6KyIiNROpK3dTmZzm90Uk8qIV/Oks7brXrohEXKSCP5nO6sCuiERetIJ/WFM9IiKRCv7BTI52jfhFJOIiE/zZfJGh0bxG/CISeZEJ/lRm7KpdjfhFJNoiE/yDqtMjIgJEKPiT48GvEb+IRFuEgl8F2kREIELBP6gRv4gIEKHgT47fhEUjfhGJtggFf476mNGSqNkNwERE5oTIBH8qnaWjOYGZKnOKSLRFKPhzdKhAm4hIdIK/VKBN8/siIpEJ/lQ6pzN6RESIUvBnNOIXEYGIBL+7k9SIX0QEiEjwZ3IFsvmizuEXESEiwZ8KrtpVZU4RkYgE/+GrdhX8IiKRCP6USjKLiIyLRPCrMqeIyGGRCH7N8YuIHBaR4C+N+HWjdRGRiAR/Mp2jORGjIa7KnCIikQj+VDqn+X0RkUBEgj9LuypziogAEQn+ZDpLZ4uCX0QEIhL8pcqcmuoREYEKg9/M1pvZVjPbZmY3TLJ+qZk9bGZPm9nPzWzRhPXzzGyXmX21Wh0/FqmMbsIiIjJmxuA3sxhwG3ApsAq42sxWTdjsVuAudz8b2AjcMmH9F4FfHn93j12x6KR0ExYRkXGVjPjXANvcfbu7Z4G7gcsnbLMKeCRYfrR8vZmdBywAfnb83T12h0byFF11ekRExlQS/AuBnWXPdwVt5bYAVwbLVwBtZtZlZnXAl4Drj7ejs5XKqFyDiEi5ah3cvR5Ya2ZPAWuBfqAAfBp4wN13TfdiM9tgZn1m1jcwMFClLpUkxwu0acQvIgIQr2CbfmBx2fNFQds4d99NMOI3s1bgQ+6eMrMLgIvM7NNAK5AwsyF3v2HC628Hbgfo7e312e7MZA6XZNaIX0QEKgv+TcBKM1tOKfCvAj5avoGZdQMH3L0I3AjcAeDuHyvb5lqgd2Lon2iDKtAmInKEGad63D0PXAc8CLwA3OPuz5nZRjO7LNhsHbDVzF6kdCD35hPU32OmEb+IyJEqGfHj7g8AD0xo+3zZ8r3AvTO8x7eBbx9zD49TMp3DDJVsEBEJhP7K3VQ6y7zGemJ1VuuuiIjMCREI/pzO6BERKRP64E+ms5rfFxEpE/rgL9Xi14hfRGRM+IM/ozo9IiLlwh/8wzmd0SMiUibUwZ8rFDk0mteIX0SkTKiDfzATXLWru2+JiIwLdfCngqt2NdUjInJYqIM/OV6nR1M9IiJjwh38w6rFLyIyUaiDP5VRLX4RkYnCHfzjlTkV/CIiY0Id/Ml0jnid0dpQURFSEZFICHXwlwq0JTBTZU4RkTEhD/6spnlERCYIdfAn01kVaBMRmSDUwT821SMiIoeFP/h11a6IyBFCHfzJdJbOFo34RUTKhTb4M9kCo/miDu6KiEwQ2uBPZYKLt5o04hcRKRfa4E8OjxVo04hfRKRcaIP/cLkGjfhFRMqFN/h1ExYRkUmFNviTac3xi4hMJrTBn0qrJLOIyGRCHPxZmupjNNbHat0VEZE5JbTBn0znNNoXEZlEaIO/VJlT8/siIhOFNviT6ZzO4RcRmURFwW9m681sq5ltM7MbJlm/1MweNrOnzeznZrYoaD/HzB4zs+eCdf+m2jswFdXiFxGZ3IzBb2Yx4DbgUmAVcLWZrZqw2a3AXe5+NrARuCVoTwN/4u7vANYDXzazjmp1fjoqySwiMrlKRvxrgG3uvt3ds8DdwOUTtlkFPBIsPzq23t1fdPeXguXdwF6gpxodn467k8poqkdEZDKVBP9CYGfZ811BW7ktwJXB8hVAm5l1lW9gZmuABPDy7LpauUOjeQpFp1MjfhGRo1Tr4O71wFozewpYC/QDhbGVZnYa8B3g4+5enPhiM9tgZn1m1jcwMHDcnUkFBdradRMWEZGjVBL8/cDisueLgrZx7r7b3a9093OBzwVtKQAzmwf8E/A5d398sl/g7re7e6+79/b0HP9M0Fi5Bo34RUSOVknwbwJWmtlyM0sAVwH3l29gZt1mNvZeNwJ3BO0J4MeUDvzeW71uT08F2kREpjZj8Lt7HrgOeBB4AbjH3Z8zs41mdlmw2Tpgq5m9CCwAbg7aPwK8H7jWzDYHj3OqvRMTjZVkbleBNhGRo8Qr2cjdHwAemND2+bLle4GjRvTu/l3gu8fZx2OWHB6b6tGIX0RkolBeuZtM6+CuiMhUQhn8g5kcbY1x4rFQ7p6IyHEJZTIm01md0SMiMoWQBr+u2hURmUoog39QJZlFRKYUyuDXTVhERKYW0uDXHL+IyFRCF/z5QpFDI3mN+EVEphC64B8MyjV06Bx+EZFJhS74xy7e6mzRVI+IyGRCF/xjdXp0Vo+IyORCGPzBiF9z/CIikwpd8I/V4u9QZU4RkUmFLvjHRvwdqsUvIjKp8AV/JkuszmhrqKjitIhI5IQu+JPpHB1N9ZhZrbsiIjInhS74U+msLt4SEZlGCIM/p3INIiLTCF3wq0CbiMj0Qhf8KZVkFhGZVuiCv1SZUyN+EZGphCr4R3IFRnJFjfhFRKYRquAfv3hLI34RkSmFKvjHyjXorB4RkamFKvg14hcRmVnIgl8F2kREZhKq4D98ExaN+EVEphKq4E9lNMcvIjKTcAV/OkdDvI7G+lituyIiMmeFKviTw1mN9kVEZhCu4FedHhGRGVUU/Ga23sy2mtk2M7thkvVLzexhM3vazH5uZovK1l1jZi8Fj2uq2fmJBjMa8YuIzGTG4DezGHAbcCmwCrjazFZN2OxW4C53PxvYCNwSvHY+cBPwbmANcJOZdVav+0fSiF9EZGaVjPjXANvcfbu7Z4G7gcsnbLMKeCRYfrRs/SXAQ+5+wN2TwEPA+uPv9uRUmVNEZGaVBP9CYGfZ811BW7ktwJXB8hVAm5l1VfjaqnD34CYsGvGLiEynWgd3rwfWmtlTwFqgHyhU+mIz22BmfWbWNzAwMKsODI3myRddUz0iIjOoJPj7gcVlzxcFbePcfbe7X+nu5wKfC9pSlbw22PZ2d+91996enp5j3IWSQtH5o7NP462nzpvV60VEoqKS4N8ErDSz5WaWAK4C7i/fwMy6zWzsvW4E7giWHwQuNrPO4KDuxUFb1XU0J/jqR1ez9szZfXGIiETFjMHv7nngOkqB/QJwj7s/Z2YbzeyyYLN1wFYzexFYANwcvPYA8EVKXx6bgI1Bm4iI1Ii5e637cITe3l7v6+urdTdERN5UzOwJd++tZNtQXbkrIiIzU/CLiESMgl9EJGIU/CIiEaPgFxGJGAW/iEjEzLnTOc1sAHjtON6iG9hXpe682WjfoyvK+x/lfYfD+7/U3Su6gnXOBf/xMrO+Ss9lDRvtezT3HaK9/1Hed5jd/muqR0QkYhT8IiIRE8bgv73WHagh7Xt0RXn/o7zvMIv9D90cv4iITC+MI34REZlGaILfzNab2VYz22ZmN9S6Pyebmb1qZs+Y2WYzC3V5UzO7w8z2mtmzZW3zzewhM3sp+NlZyz6eSFPs/xfMrD/4/Deb2R/Uso8nipktNrNHzex5M3vOzP590B76z3+afT/mzz4UUz1mFgNeBH6f0n19NwFXu/vzNe3YSWRmrwK97h7685nN7P3AEHCXu58VtP0dcMDd/zb44u909/9Yy36eKFPs/xeAIXe/tZZ9O9HM7DTgNHd/0szagCeADwLXEvLPf5p9/wjH+NmHZcS/Btjm7tvdPQvcDVxe4z7JCeLuvwQm3tDncuDOYPlOSv9DhNIU+x8J7v66uz8ZLB+idHOohUTg859m349ZWIJ/IbCz7PkuZvkP8ibmwM/M7Akz21DrztTAAnd/PVh+g9Kd4KLmOjN7OpgKCt1Ux0Rmtgw4F/gtEfv8J+w7HONnH5bgF3ifu68GLgU+E0wHRJKX5i/f/HOYx+ZrwBnAOcDrwJdq250Ty8xagX8E/tzdD5avC/vnP8m+H/NnH5bg7wcWlz1fFLRFhrv3Bz/3Aj+mNP0VJXuCOdCxudC9Ne7PSeXue9y94O5F4BuE+PM3s3pKwfc9d/9R0ByJz3+yfZ/NZx+W4N8ErDSz5WaWAK4C7q9xn04aM2sJDvZgZi3AxcCz078qdO4HrgmWrwF+UsO+nHRjoRe4gpB+/mZmwLeAF9z9v5atCv3nP9W+z+azD8VZPQDBKUxfBmLAHe5+c427dNKY2QpKo3yAOPD9MO+/mf0AWEepKuEe4CbgPuAeYAml6q4fcfdQHgCdYv/XUfpT34FXgT8rm/MODTN7H/Ar4BmgGDT/JaW57lB//tPs+9Uc42cfmuAXEZHKhGWqR0REKqTgFxGJGAW/iEjEKPhFRCJGwS8iEjEKfhGRiFHwi4hEjIJfRCRi/j/m7kPdbhJyrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(acc_lst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y_test = model.predict(X_test)\n",
    "Y_test = np.argmax(Y_test,1)\n",
    "Y_test=Y_test.reshape(28000,1)\n",
    "c = np.array([[i+1 for i in range(28000)]]).T\n",
    "ans=np.concatenate((c,Y_test),axis=1)\n",
    "df = pd.DataFrame(ans,columns=['ImageId','Label'])\n",
    "df.to_csv('ans.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}