grapevine-leaves-livedemo-final.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "lyric-upper",
   "metadata": {
    "papermill": {
     "duration": 0.011307,
     "end_time": "2021-05-08T11:34:11.883685",
     "exception": false,
     "start_time": "2021-05-08T11:34:11.872378",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Pytorch Grapevine Leaf Classifier\n",
    "\n",
    "The start notebook was taken with very little modification (just path and use pyplot) from Maxim Vlah\n",
    "\n",
    "Dataset and code (c) 2021 Maxim Vlah (maximvlah@gmail.com), Licensed CC-BY-SA-NC 4.0 and used with permission.\n",
    "\n",
    "Dataset: https://www.kaggle.com/maximvlah/grapevine-leaves\n",
    "\n",
    "Original notebook: https://www.kaggle.com/maximvlah/pytorch-resnet18-94-f1\n",
    "\n",
    "**Important notice from Thomas**: I picked this notebook because I liked the dataset and the code was reasonable and protypical of a lot of code I see in my work. Creating, curating and releasing a useful dataset (from your own images, too) is a great accomplishment, and a demonstration of the dataset with simple demo code is cool, too. I do not think of this code as primarily _unoptimized_ but rather as primarily _worth optimizing_. Thank you Maxim!\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "theoretical-carbon",
   "metadata": {
    "_kg_hide-input": false,
    "_kg_hide-output": true,
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:11.916438Z",
     "iopub.status.busy": "2021-05-08T11:34:11.915919Z",
     "iopub.status.idle": "2021-05-08T11:34:16.622168Z",
     "shell.execute_reply": "2021-05-08T11:34:16.621538Z"
    },
    "papermill": {
     "duration": 4.728604,
     "end_time": "2021-05-08T11:34:16.622318",
     "exception": false,
     "start_time": "2021-05-08T11:34:11.893714",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "'''Load librarires'''\n",
    "import pickle\n",
    "import time\n",
    "import random\n",
    "import ntpath\n",
    "import glob\n",
    "import os\n",
    "from copy import deepcopy\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "from tqdm import tqdm\n",
    "from pathlib import Path\n",
    "from collections import defaultdict\n",
    "\n",
    "from matplotlib import pyplot\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset,DataLoader\n",
    "import torchvision\n",
    "from torchvision import models, transforms, utils\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3a735130",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:16.655371Z",
     "iopub.status.busy": "2021-05-08T11:34:16.654741Z",
     "iopub.status.idle": "2021-05-08T11:34:19.158662Z",
     "shell.execute_reply": "2021-05-08T11:34:19.159326Z"
    },
    "papermill": {
     "duration": 2.526336,
     "end_time": "2021-05-08T11:34:19.159477",
     "exception": false,
     "start_time": "2021-05-08T11:34:16.633141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are using -> cuda:0\n"
     ]
    }
   ],
   "source": [
    "class CFG:\n",
    "\n",
    "    '''Store all hyperparameters here.''' \n",
    "\n",
    "    SEED = 420\n",
    "    TEST_SIZE = 0.2\n",
    "    VAL_SIZE = 0.25\n",
    "    CLASSES = None #Need to update manually\n",
    "    OUTPUT_FEATURES = None #Need to update manually\n",
    "    \n",
    "    #transforms\n",
    "    TRAIN_TRANSFORMS_PRESCALE = transforms.Compose([\n",
    "        transforms.Resize((256,256)),\n",
    "        #Convert a PIL Image or numpy.ndarray to tensor.\n",
    "        transforms.ToTensor(),\n",
    "    ])\n",
    "    TRAIN_TRANSFORMS = transforms.Compose([\n",
    "        #Convert a PIL Image or numpy.ndarray to tensor.\n",
    "        transforms.ToTensor(),\n",
    "    ])\n",
    "    TRAIN_TRANSFORMS_GPU = transforms.Compose([\n",
    "        #Rotate the image by given angle.\n",
    "        transforms.RandomRotation(5),\n",
    "        #Crop the given PIL Image to random size and aspect ratio.\n",
    "        transforms.Resize((224,224)),\n",
    "        #Horizontally flip the given PIL Image randomly with a given probability.\n",
    "        transforms.RandomHorizontalFlip(p = 0.2),\n",
    "        #Normalize a tensor image with mean and standard deviation\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                             std=[0.229, 0.224, 0.225])\n",
    "    ])\n",
    "\n",
    "    VAL_TRANSFORMS = TRAIN_TRANSFORMS\n",
    "    VAL_TRANSFORMS_GPU = TRAIN_TRANSFORMS_GPU\n",
    "    \n",
    "    TEST_TRANSFORMS = transforms.Compose([\n",
    "        #Crop the given PIL Image to random size and aspect ratio.\n",
    "        transforms.Resize((224,224)),\n",
    "        #Convert a PIL Image or numpy.ndarray to tensor.\n",
    "        transforms.ToTensor(),\n",
    "        #Normalize a tensor image with mean and standard deviation\n",
    "        transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                             std=[0.229, 0.224, 0.225])\n",
    "    ])\n",
    "    \n",
    "    #models\n",
    "    \n",
    "    # MODEL1 = {\n",
    "    #     'name': 'custom_cnn',\n",
    "    #     'transfer': False,\n",
    "    #     'architecture': nn.Sequential(\n",
    "    #       nn.Conv2d(1,20,5),\n",
    "    #       nn.ReLU(),\n",
    "    #       nn.Conv2d(20,64,5),\n",
    "    #       nn.ReLU()\n",
    "    #     ),\n",
    "    #     'criterion': nn.CrossEntropyLoss(),\n",
    "    #     'optimizer': optim.SGD,\n",
    "    #     'momentum': 0.9,\n",
    "    #     'lr': 0.003,\n",
    "    #     'history': None\n",
    "    # }\n",
    "\n",
    "    MODEL2 = {\n",
    "        'name': 'resnet18',\n",
    "        'transfer': True,\n",
    "        'architecture': models.resnet18(pretrained=True), # ResNet18\n",
    "        'criterion': nn.CrossEntropyLoss(),\n",
    "        'optimizer': optim.SGD,\n",
    "        'momentum': 0.9,\n",
    "        'lr': 0.002,\n",
    "        'history': None\n",
    "    }\n",
    "\n",
    "    # MODELS =[MODEL1,MODEL2]\n",
    "\n",
    "    BATCH_SIZE = 64\n",
    "    EPOCHS = 120\n",
    "    \n",
    "    DEVICE = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print('You are using ->', DEVICE)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "composite-bracelet",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:16.655371Z",
     "iopub.status.busy": "2021-05-08T11:34:16.654741Z",
     "iopub.status.idle": "2021-05-08T11:34:19.158662Z",
     "shell.execute_reply": "2021-05-08T11:34:19.159326Z"
    },
    "papermill": {
     "duration": 2.526336,
     "end_time": "2021-05-08T11:34:19.159477",
     "exception": false,
     "start_time": "2021-05-08T11:34:16.633141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1009/1009 [00:20<00:00, 48.72it/s]\n"
     ]
    }
   ],
   "source": [
    "paths = glob.glob('./wine-data//*/*' )\n",
    "for img_path in tqdm(paths):\n",
    "    img = Image.open(img_path)\n",
    "    img = CFG.TRAIN_TRANSFORMS_PRESCALE(img)\n",
    "    img_path_new = img_path.rsplit('/', 2)\n",
    "    img_path_new[0] += '-256'\n",
    "    img_path_new = '/'.join(img_path_new)\n",
    "    os.makedirs(os.path.dirname(img_path_new), exist_ok=True)\n",
    "    torchvision.utils.save_image(img, img_path_new)\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "swedish-girlfriend",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.187740Z",
     "iopub.status.busy": "2021-05-08T11:34:19.187112Z",
     "iopub.status.idle": "2021-05-08T11:34:19.192431Z",
     "shell.execute_reply": "2021-05-08T11:34:19.191996Z"
    },
    "papermill": {
     "duration": 0.020998,
     "end_time": "2021-05-08T11:34:19.192538",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.171540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed):\n",
    "    '''Make the results reproducible'''\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "    torch.backends.cudnn.benchmark = True \n",
    "\n",
    "seed_everything(CFG.SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "engaging-ranch",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.219882Z",
     "iopub.status.busy": "2021-05-08T11:34:19.219376Z",
     "iopub.status.idle": "2021-05-08T11:34:19.347056Z",
     "shell.execute_reply": "2021-05-08T11:34:19.347469Z"
    },
    "papermill": {
     "duration": 0.143883,
     "end_time": "2021-05-08T11:34:19.347610",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.203727",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>./wine-data-256/Muller Thurgau/IMG_2087.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>./wine-data-256/Muller Thurgau/IMG_2006.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>./wine-data-256/Muller Thurgau/IMG_2046.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>./wine-data-256/Muller Thurgau/IMG_2111.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>./wine-data-256/Muller Thurgau/IMG_2036.JPEG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           path  label\n",
       "0  ./wine-data-256/Muller Thurgau/IMG_2087.JPEG      5\n",
       "1  ./wine-data-256/Muller Thurgau/IMG_2006.JPEG      5\n",
       "2  ./wine-data-256/Muller Thurgau/IMG_2046.JPEG      5\n",
       "3  ./wine-data-256/Muller Thurgau/IMG_2111.JPEG      5\n",
       "4  ./wine-data-256/Muller Thurgau/IMG_2036.JPEG      5"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''Store image paths and their labels in pandas dataframe. Will be used to create pytorch datasets. '''\n",
    "\n",
    "paths = glob.glob('./wine-data-256/*/*' )\n",
    "meta = pd.DataFrame([(path, ntpath.basename(ntpath.dirname(path))) for path in paths], columns = ['path','label'])\n",
    "#get class mappings\n",
    "classes = dict(enumerate(meta.label.astype('category').cat.categories))\n",
    "CFG.CLASSES = classes\n",
    "CFG.OUTPUT_FEATURES = len(CFG.CLASSES)\n",
    "\n",
    "meta.label = meta.label.astype('category').cat.codes\n",
    "meta.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "valid-scanning",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.375694Z",
     "iopub.status.busy": "2021-05-08T11:34:19.374979Z",
     "iopub.status.idle": "2021-05-08T11:34:19.378018Z",
     "shell.execute_reply": "2021-05-08T11:34:19.378390Z"
    },
    "papermill": {
     "duration": 0.019042,
     "end_time": "2021-05-08T11:34:19.378513",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.359471",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0: 'Auxerrois',\n",
       " 1: 'Cabernet Franc',\n",
       " 2: 'Cabernet Sauvignon',\n",
       " 3: 'Chardonnay',\n",
       " 4: 'Merlot',\n",
       " 5: 'Muller Thurgau',\n",
       " 6: 'Pinot Noir',\n",
       " 7: 'Riesling',\n",
       " 8: 'Sauvignon Blanc',\n",
       " 9: 'Syrah',\n",
       " 10: 'Tempranillo'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "analyzed-student",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.409007Z",
     "iopub.status.busy": "2021-05-08T11:34:19.408493Z",
     "iopub.status.idle": "2021-05-08T11:34:19.418535Z",
     "shell.execute_reply": "2021-05-08T11:34:19.418993Z"
    },
    "papermill": {
     "duration": 0.028516,
     "end_time": "2021-05-08T11:34:19.419140",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.390624",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train length -> 605\n",
      "Val length -> 202\n",
      "Test length -> 202\n"
     ]
    }
   ],
   "source": [
    "'''Split data into train, validation and test sets'''\n",
    "\n",
    "X = list(meta.path)\n",
    "y = list(meta.label)\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, \n",
    "                                                    y, \n",
    "                                                    test_size=CFG.TEST_SIZE, \n",
    "                                                    random_state=CFG.SEED, \n",
    "                                                    stratify=y) #stratified split\n",
    "\n",
    "X_train, X_val, y_train, y_val = train_test_split(X_train,\n",
    "                                                  y_train,\n",
    "                                                  test_size=CFG.VAL_SIZE,\n",
    "                                                  random_state=CFG.SEED,\n",
    "                                                  stratify=y_train) #stratified split\n",
    "\n",
    "print(f'Train length -> {len(X_train)}')\n",
    "print(f'Val length -> {len(X_val)}')\n",
    "print(f'Test length -> {len(X_test)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "elder-turkish",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.453050Z",
     "iopub.status.busy": "2021-05-08T11:34:19.452507Z",
     "iopub.status.idle": "2021-05-08T11:34:19.455373Z",
     "shell.execute_reply": "2021-05-08T11:34:19.454956Z"
    },
    "papermill": {
     "duration": 0.023129,
     "end_time": "2021-05-08T11:34:19.455469",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.432340",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "'''Custom pytorch dataset implementation.'''\n",
    "class LeafDataset(Dataset):\n",
    "    def __init__(self,X,y, transform=None):\n",
    "        self.X = X\n",
    "        self.y = torch.tensor(y, dtype=torch.long)\n",
    "        self.transform = transform\n",
    "\n",
    "        assert len(self.X) == len(self.y), f'X and y have different lengths -> {len(self.X)} != {len(self.y)} '\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.X)\n",
    "\n",
    "    def __getitem__(self,idx):\n",
    "        img_path = self.X[idx]\n",
    "        img = Image.open(img_path)\n",
    "        if self.transform is not None:\n",
    "            img = self.transform(img)\n",
    "        label = self.y[idx]\n",
    "        return (img,label) \n",
    "\n",
    "    def show_img(self,idx):\n",
    "        '''Plot image'''\n",
    "        img,label = self.__getitem__(idx)\n",
    "        img = img.numpy().transpose((1, 2, 0))\n",
    "        pyplot.figure(figsize=(16, 8))\n",
    "        pyplot.axis('off')\n",
    "        pyplot.imshow(img)\n",
    "        pyplot.title(CFG.CLASSES[int(label)]) #using CFG.CLASSES dict\n",
    "        pyplot.pause(0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "personalized-inside",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.488750Z",
     "iopub.status.busy": "2021-05-08T11:34:19.488073Z",
     "iopub.status.idle": "2021-05-08T11:34:19.490869Z",
     "shell.execute_reply": "2021-05-08T11:34:19.490439Z"
    },
    "papermill": {
     "duration": 0.021416,
     "end_time": "2021-05-08T11:34:19.490986",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.469570",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 605/605 [00:00<00:00, 1858.17it/s]\n",
      "100%|██████████| 202/202 [00:00<00:00, 2414.53it/s]\n"
     ]
    }
   ],
   "source": [
    "'''Instantiate pytorch train, validation and test sets'''\n",
    "TRAIN = LeafDataset(X_train,y_train, CFG.TRAIN_TRANSFORMS)\n",
    "VAL = LeafDataset(X_val,y_val, CFG.VAL_TRANSFORMS)\n",
    "TEST = LeafDataset(X_test,y_test, CFG.TEST_TRANSFORMS)\n",
    "\n",
    "def ds_to_tensords(ds):\n",
    "    allimgs = []\n",
    "    alllab = []\n",
    "    for img, lab in tqdm(ds):\n",
    "        allimgs.append(img)\n",
    "        alllab.append(lab)\n",
    "    allimgs = torch.stack(allimgs).cuda()\n",
    "    alllab = torch.tensor(alllab).cuda()\n",
    "    return torch.utils.data.TensorDataset(allimgs, alllab)\n",
    "\n",
    "TRAIN_FAST = ds_to_tensords(TRAIN)\n",
    "VAL_FAST = ds_to_tensords(VAL)\n",
    "\n",
    "'''Instantiate Dataloaders'''\n",
    "TRAIN_LOADER = DataLoader(TRAIN_FAST,CFG.BATCH_SIZE)\n",
    "VAL_LOADER = DataLoader(VAL_FAST,CFG.BATCH_SIZE)\n",
    "TEST_LOADER = DataLoader(TEST,CFG.BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "sorted-teens",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.525117Z",
     "iopub.status.busy": "2021-05-08T11:34:19.524065Z",
     "iopub.status.idle": "2021-05-08T11:34:19.562116Z",
     "shell.execute_reply": "2021-05-08T11:34:19.561647Z"
    },
    "papermill": {
     "duration": 0.057173,
     "end_time": "2021-05-08T11:34:19.562226",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.505053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Net(nn.Module):\n",
    "    '''\n",
    "    ========================\n",
    "          NEURAL NET\n",
    "    ========================\n",
    "    \n",
    "    Args:\n",
    "        model_dict(dict): configuration dict containing the model architecture\n",
    "        output_features(int): length of output tensor; for classification equals to number of classes\n",
    "    '''\n",
    "    def __init__(self, model_dict, output_features):\n",
    "        super().__init__()\n",
    "        self.__dict__.update(model_dict) #unpack model dict from CFG into this class\n",
    "        \n",
    "        if self.transfer:\n",
    "            model = self.architecture\n",
    "            num_ftrs = model.fc.in_features\n",
    "            model.fc = nn.Linear(num_ftrs, output_features)\n",
    "            self.model = model\n",
    "        else:\n",
    "            self.model = self.architecture\n",
    "        \n",
    "        self.output_features = output_features\n",
    "        #optimizer\n",
    "        self.optimizer = self.optimizer(self.model.parameters(),self.lr)\n",
    "        #path where to save model\n",
    "        self.save_path = 'models'\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "\n",
    "    def fit(self,\n",
    "            train_loader,\n",
    "            val_loader,\n",
    "            epochs = 5,\n",
    "            batch_size = 32,\n",
    "            device = 'cpu'):\n",
    "        '''\n",
    "        =============================\n",
    "            OPTIMIZATION LOOP\n",
    "        =============================\n",
    "\n",
    "        Args:\n",
    "            train_loader(torch dataloader)\n",
    "            val_loader(torch dataloader)\n",
    "            epochs(int)\n",
    "            batch_size(int)\n",
    "            device(str)\n",
    "\n",
    "            \n",
    "        Output style inspired by the skorch fit() method\n",
    "\n",
    "        '''\n",
    "        #may be changed if lrscheduler is used???\n",
    "        lr = deepcopy(self.lr)\n",
    "\n",
    "        #get model training history\n",
    "        history = self.history\n",
    "        if history == None:\n",
    "            history = defaultdict(list)\n",
    "        else:\n",
    "            pass\n",
    "        #get train and val sizes\n",
    "        train_size = len(train_loader.dataset)\n",
    "        val_size = len(val_loader.dataset)\n",
    "        #stuff for printing epoch metrics as a beautiful table\n",
    "        headers = ['epoch','train_loss','val_loss','val_acc','cp','lr','dur']\n",
    "        template = '{:<10} {:<10} {:<10} {:<10} {:<10} {:<10} {:<10}'\n",
    "        print(template.format(*headers))\n",
    "        print(template.replace(':', ':-').format('','','','','','',''))\n",
    "        cyan = \"\\033[96m{:<10}\\033[00m\" #cyan\n",
    "        purple = \"\\033[95m{:<10}\\033[00m\" #purple\n",
    "        green = \"\\033[92m{:<10}\\033[00m\" #green\n",
    "        white = \"\\033[0m{:<10}\\033[0m\" #white\n",
    "        #set model into train mode\n",
    "        self.model.train()\n",
    "        #send model to device\n",
    "        self.model.to(device)\n",
    "        #training loop\n",
    "        for epoch in range(epochs):\n",
    "            start_time = time.time()\n",
    "            train_loss = 0.0\n",
    "            val_loss, val_acc = 0.0, 0.0\n",
    "            self.model.train()\n",
    "            #optimization  loop\n",
    "            for (X,y) in train_loader:\n",
    "                #Send training data to device\n",
    "                X,y = X.to(device), y.to(device)\n",
    "                X = CFG.TRAIN_TRANSFORMS_GPU(X)\n",
    "                #Forward propagation\n",
    "                pred = self.model(X)\n",
    "                loss = self.criterion(pred,y)\n",
    "                #Backpropagation\n",
    "                self.optimizer.zero_grad()\n",
    "                loss.backward()\n",
    "                self.optimizer.step()\n",
    "            #update loss\n",
    "            train_loss = loss.item()\n",
    "            #validation loop\n",
    "            self.model.eval()\n",
    "            with torch.no_grad():\n",
    "                for X, y in val_loader:\n",
    "                    X,y = X.to(device),y.to(device)\n",
    "                    X = CFG.VAL_TRANSFORMS_GPU(X)\n",
    "                    pred = self.model(X)\n",
    "                    val_loss = self.criterion(pred,y)\n",
    "                    val_acc = val_acc + (pred.argmax(1) == y).type(torch.float).sum()\n",
    "            val_loss = val_loss.item()\n",
    "            val_acc = val_acc.item()\n",
    "            #calculate validation accuracy after the epoch\n",
    "            val_acc /= val_size\n",
    "            #append epoch results\n",
    "            history['epoch'].append(epoch+1)\n",
    "            history['train_loss'].append(train_loss)\n",
    "            history['val_loss'].append(val_loss)\n",
    "            history['val_acc'].append(val_acc)\n",
    "            \n",
    "            #colorize epoch's output if it improves\n",
    "            colortemp = template.split(' ')\n",
    "            # colorize train loss if it decreases\n",
    "            if history['train_loss'][-1] == min(history['train_loss']):\n",
    "                colortemp[1] = cyan\n",
    "            else:\n",
    "                colortemp[1] = white\n",
    "            #colorize validation loss if it decreases\n",
    "            if history['val_loss'][-1] == min(history['val_loss']):\n",
    "                colortemp[2] = purple\n",
    "            else:\n",
    "                colortemp[2] = white\n",
    "            # colorize validation accuracy & save best weights if it increases\n",
    "            if history['val_acc'][-1] == max(history['val_acc']):\n",
    "                #colorize       \n",
    "                colortemp[3] = green\n",
    "                #checkpoint\n",
    "                cp = '+'\n",
    "                if not os.path.exists(self.save_path):\n",
    "                    os.mkdir(self.save_path)\n",
    "                torch.save(self.model.state_dict(), Path(self.save_path,f'best_{self.name}.pth'))\n",
    "            else:\n",
    "                colortemp[3] = white\n",
    "                cp = '-'\n",
    "            colortemp = ' '.join(colortemp)\n",
    "\n",
    "            #calculate epoch duration (in seconds)\n",
    "            end_time = time.time()\n",
    "            dur = end_time - start_time\n",
    "            #append the rest of epoch results\n",
    "            history['cp'].append(cp)\n",
    "            history['lr'].append(lr)\n",
    "            history['dur'].append(dur)\n",
    "            #display the epoch results\n",
    "            print(colortemp.format(*f'{epoch+1}/{epochs} {train_loss:.4f} {val_loss:.4f} {val_acc:.2f} {cp} {lr} {dur:.2f}'.split(' ')))\n",
    "        #update epoch number of the entire training history\n",
    "        history['epoch'] = [e+1 for e in range(len(history['epoch']))]\n",
    "        #update model's training history\n",
    "        self.history = history\n",
    "        #save training history as csv\n",
    "        self.save_history()\n",
    "\n",
    "    def eval_model(self,dataloader,avg=None,device ='cpu'):\n",
    "        '''\n",
    "        ==================================\n",
    "           ACCURACY PRECISION RECALL F1\n",
    "        ==================================\n",
    "        '''\n",
    "        labels = [l for l in range(self.output_features)]\n",
    "        loader_size = len(dataloader)\n",
    "        dataset_size = len(dataloader.dataset)\n",
    "\n",
    "        acc =0\n",
    "        precision = 0\n",
    "        recall = 0\n",
    "        f1 = 0\n",
    "\n",
    "        #set model to evaluation mode\n",
    "        self.model.eval()\n",
    "        #model to device, default cpu\n",
    "        self.model.to(device)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            for X, y in dataloader:\n",
    "                pred = self.model(X)\n",
    "                #accuracy\n",
    "                acc += (pred.argmax(1) == y).type(torch.float).sum().item()\n",
    "\n",
    "                pred = pred.argmax(1)\n",
    "                pred,y = list(pred), list(y)\n",
    "                #precision\n",
    "                p = precision_score(y, pred, labels = labels, zero_division = 1, average = avg)\n",
    "                precision+=p\n",
    "                #recall\n",
    "                r = recall_score(y, pred, labels = labels, zero_division = 1,  average = avg)\n",
    "                recall+=r\n",
    "                #f1 score\n",
    "                f = f1_score(y, pred, labels = labels, zero_division = 1,  average = avg)\n",
    "                f1 += f\n",
    "\n",
    "        acc /= dataset_size\n",
    "        precision /= loader_size\n",
    "        recall /= loader_size\n",
    "        f1 /= loader_size\n",
    "\n",
    "        print(f\" Accuracy: {(100*acc):>0.1f}%\")\n",
    "        print(f\"Precision: {(100*np.mean(precision)):>0.1f}%\")\n",
    "        print(f\"   Recall: {(100*np.mean(recall)):>0.1f}%\")\n",
    "        print(f\" F1 Score: {(100*np.mean(f1)):>0.1f}%\")\n",
    "\n",
    "        \n",
    "    def plot_loss_history(self):\n",
    "        '''\n",
    "        Plot loss history\n",
    "        '''\n",
    "        assert self.history != None, 'No history to plot -> the model has not been trained yet!'\n",
    "        \n",
    "        df = pd.DataFrame(self.history)\n",
    "        fig = pyplot.figure()\n",
    "        pyplot.plot(df.epoch, df.train_loss, label='train loss')\n",
    "        pyplot.plot(df.epoch, df.val_loss, label='val loss')\n",
    "        pyplot.title('Loss History')\n",
    "        pyplot.legend()\n",
    "        pyplot.show()\n",
    "\n",
    "    def save_history(self):\n",
    "        '''Save model's training history'''\n",
    "        assert self.history != None, 'No history to save -> the model has not been trained yet!'\n",
    "        #save as csv\n",
    "        pd.DataFrame(self.history).to_csv(Path(f'models/{self.name}_history.csv')) \n",
    "        #save as pickle file\n",
    "        with open(Path(f'models/{self.name}_history.pkl'), 'wb') as f:\n",
    "            pickle.dump(self.history, f, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "    def save_model(self):\n",
    "        torch.save(self.model.state_dict(), Path(self.save_path,f'latest_{self.name}.pth'))\n",
    "\n",
    "    def load_model(self,path = 'models'):\n",
    "        '''Load model'''\n",
    "        try:\n",
    "            #load model weights\n",
    "            p = Path(path,f'best_{self.name}.pth')\n",
    "            self.model.load_state_dict(torch.load(p))\n",
    "            #load model training history\n",
    "            with open(Path(path,f'{self.name}_history.pkl'), 'rb') as h: \n",
    "                self.history = pickle.load(h)\n",
    "        except:\n",
    "            print('No model to load!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "5e2f2a27",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.600420Z",
     "iopub.status.busy": "2021-05-08T11:34:19.599609Z",
     "iopub.status.idle": "2021-05-08T11:34:19.602614Z",
     "shell.execute_reply": "2021-05-08T11:34:19.602163Z"
    },
    "papermill": {
     "duration": 0.023493,
     "end_time": "2021-05-08T11:34:19.602721",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.579228",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#instantiate model and send to device\n",
    "Resnet18 = Net(CFG.MODEL2,CFG.OUTPUT_FEATURES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "norman-national",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.600420Z",
     "iopub.status.busy": "2021-05-08T11:34:19.599609Z",
     "iopub.status.idle": "2021-05-08T11:34:19.602614Z",
     "shell.execute_reply": "2021-05-08T11:34:19.602163Z"
    },
    "papermill": {
     "duration": 0.023493,
     "end_time": "2021-05-08T11:34:19.602721",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.579228",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "wooden-baptist",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T11:34:19.640153Z",
     "iopub.status.busy": "2021-05-08T11:34:19.639534Z",
     "iopub.status.idle": "2021-05-08T13:39:36.915991Z",
     "shell.execute_reply": "2021-05-08T13:39:36.916582Z"
    },
    "papermill": {
     "duration": 7517.298847,
     "end_time": "2021-05-08T13:39:36.916788",
     "exception": false,
     "start_time": "2021-05-08T11:34:19.617941",
     "status": "completed"
    },
    "scrolled": false,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      train_loss val_loss   val_acc    cp         lr         dur       \n",
      "---------- ---------- ---------- ---------- ---------- ---------- ----------\n",
      "1/120      \u001b[96m2.0570    \u001b[00m \u001b[95m1.9608    \u001b[00m \u001b[92m0.43      \u001b[00m +          0.002      0.53      \n",
      "2/120      \u001b[96m1.5361    \u001b[00m \u001b[95m1.5334    \u001b[00m \u001b[92m0.71      \u001b[00m +          0.002      0.51      \n",
      "3/120      \u001b[96m1.1406    \u001b[00m \u001b[95m1.2613    \u001b[00m \u001b[92m0.75      \u001b[00m +          0.002      0.51      \n",
      "4/120      \u001b[96m0.8781    \u001b[00m \u001b[95m1.0661    \u001b[00m \u001b[92m0.83      \u001b[00m +          0.002      0.51      \n",
      "5/120      \u001b[96m0.6887    \u001b[00m \u001b[95m0.8865    \u001b[00m \u001b[92m0.89      \u001b[00m +          0.002      0.51      \n",
      "6/120      \u001b[96m0.6303    \u001b[00m \u001b[95m0.8080    \u001b[00m \u001b[92m0.90      \u001b[00m +          0.002      0.51      \n",
      "7/120      \u001b[96m0.4416    \u001b[00m \u001b[95m0.7212    \u001b[00m \u001b[92m0.91      \u001b[00m +          0.002      0.51      \n",
      "8/120      \u001b[96m0.4360    \u001b[00m \u001b[95m0.6616    \u001b[00m \u001b[92m0.93      \u001b[00m +          0.002      0.51      \n",
      "9/120      \u001b[96m0.3484    \u001b[00m \u001b[0m0.6692    \u001b[0m \u001b[0m0.91      \u001b[0m -          0.002      0.47      \n",
      "10/120     \u001b[96m0.2711    \u001b[00m \u001b[95m0.6375    \u001b[00m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "11/120     \u001b[96m0.2398    \u001b[00m \u001b[95m0.5502    \u001b[00m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "12/120     \u001b[96m0.2046    \u001b[00m \u001b[0m0.5778    \u001b[0m \u001b[92m0.94      \u001b[00m +          0.002      0.52      \n",
      "13/120     \u001b[96m0.1784    \u001b[00m \u001b[95m0.5203    \u001b[00m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "14/120     \u001b[96m0.1647    \u001b[00m \u001b[95m0.4630    \u001b[00m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "15/120     \u001b[96m0.1558    \u001b[00m \u001b[0m0.5234    \u001b[0m \u001b[92m0.94      \u001b[00m +          0.002      0.53      \n",
      "16/120     \u001b[96m0.1430    \u001b[00m \u001b[0m0.4894    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "17/120     \u001b[0m0.1788    \u001b[0m \u001b[0m0.4990    \u001b[0m \u001b[92m0.94      \u001b[00m +          0.002      0.53      \n",
      "18/120     \u001b[96m0.1141    \u001b[00m \u001b[0m0.4749    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "19/120     \u001b[0m0.1871    \u001b[0m \u001b[95m0.4617    \u001b[00m \u001b[0m0.90      \u001b[0m -          0.002      0.48      \n",
      "20/120     \u001b[0m0.1360    \u001b[0m \u001b[95m0.4130    \u001b[00m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "21/120     \u001b[96m0.1025    \u001b[00m \u001b[0m0.4216    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "22/120     \u001b[96m0.0859    \u001b[00m \u001b[0m0.4357    \u001b[0m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "23/120     \u001b[96m0.0842    \u001b[00m \u001b[0m0.4873    \u001b[0m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "24/120     \u001b[96m0.0727    \u001b[00m \u001b[0m0.4339    \u001b[0m \u001b[92m0.94      \u001b[00m +          0.002      0.53      \n",
      "25/120     \u001b[96m0.0678    \u001b[00m \u001b[0m0.4324    \u001b[0m \u001b[92m0.94      \u001b[00m +          0.002      0.53      \n",
      "26/120     \u001b[96m0.0671    \u001b[00m \u001b[0m0.4548    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "27/120     \u001b[96m0.0638    \u001b[00m \u001b[0m0.4458    \u001b[0m \u001b[92m0.94      \u001b[00m +          0.002      0.53      \n",
      "28/120     \u001b[0m0.0667    \u001b[0m \u001b[95m0.3769    \u001b[00m \u001b[92m0.96      \u001b[00m +          0.002      0.52      \n",
      "29/120     \u001b[0m0.0960    \u001b[0m \u001b[95m0.3530    \u001b[00m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "30/120     \u001b[96m0.0569    \u001b[00m \u001b[95m0.3384    \u001b[00m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "31/120     \u001b[96m0.0516    \u001b[00m \u001b[0m0.3575    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "32/120     \u001b[96m0.0499    \u001b[00m \u001b[0m0.4062    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "33/120     \u001b[96m0.0498    \u001b[00m \u001b[0m0.3558    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "34/120     \u001b[96m0.0474    \u001b[00m \u001b[0m0.4033    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "35/120     \u001b[96m0.0468    \u001b[00m \u001b[0m0.3724    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "36/120     \u001b[96m0.0453    \u001b[00m \u001b[0m0.3669    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "37/120     \u001b[96m0.0415    \u001b[00m \u001b[0m0.3580    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "38/120     \u001b[0m0.0447    \u001b[0m \u001b[0m0.4330    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "39/120     \u001b[96m0.0388    \u001b[00m \u001b[0m0.4513    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "40/120     \u001b[0m0.0579    \u001b[0m \u001b[0m0.3507    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "41/120     \u001b[96m0.0373    \u001b[00m \u001b[0m0.3732    \u001b[0m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "42/120     \u001b[0m0.0556    \u001b[0m \u001b[0m0.4071    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "43/120     \u001b[96m0.0357    \u001b[00m \u001b[95m0.3253    \u001b[00m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "44/120     \u001b[0m0.0358    \u001b[0m \u001b[95m0.3239    \u001b[00m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "45/120     \u001b[96m0.0316    \u001b[00m \u001b[95m0.3048    \u001b[00m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "46/120     \u001b[0m0.0381    \u001b[0m \u001b[0m0.3130    \u001b[0m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "47/120     \u001b[96m0.0314    \u001b[00m \u001b[0m0.3318    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "48/120     \u001b[96m0.0292    \u001b[00m \u001b[0m0.3580    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "49/120     \u001b[0m0.0293    \u001b[0m \u001b[95m0.3032    \u001b[00m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "50/120     \u001b[0m0.0333    \u001b[0m \u001b[0m0.3172    \u001b[0m \u001b[0m0.96      \u001b[0m -          0.002      0.48      \n",
      "51/120     \u001b[96m0.0269    \u001b[00m \u001b[0m0.4652    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "52/120     \u001b[96m0.0252    \u001b[00m \u001b[0m0.4599    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "53/120     \u001b[0m0.0482    \u001b[0m \u001b[0m0.4937    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "54/120     \u001b[0m0.0259    \u001b[0m \u001b[0m0.3600    \u001b[0m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "55/120     \u001b[96m0.0250    \u001b[00m \u001b[0m0.3674    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "56/120     \u001b[0m0.0261    \u001b[0m \u001b[0m0.3414    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "57/120     \u001b[96m0.0235    \u001b[00m \u001b[0m0.3196    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "58/120     \u001b[0m0.0407    \u001b[0m \u001b[0m0.4024    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "59/120     \u001b[96m0.0222    \u001b[00m \u001b[95m0.2555    \u001b[00m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "60/120     \u001b[0m0.0246    \u001b[0m \u001b[0m0.3995    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "61/120     \u001b[96m0.0210    \u001b[00m \u001b[0m0.4721    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "62/120     \u001b[0m0.0234    \u001b[0m \u001b[0m0.3341    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "63/120     \u001b[0m0.0213    \u001b[0m \u001b[0m0.4466    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "64/120     \u001b[0m0.0222    \u001b[0m \u001b[0m0.4005    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "65/120     \u001b[96m0.0196    \u001b[00m \u001b[0m0.3036    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "66/120     \u001b[0m0.0199    \u001b[0m \u001b[0m0.3251    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "67/120     \u001b[96m0.0193    \u001b[00m \u001b[0m0.3142    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "68/120     \u001b[96m0.0188    \u001b[00m \u001b[0m0.3023    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "69/120     \u001b[0m0.0195    \u001b[0m \u001b[0m0.4274    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "70/120     \u001b[96m0.0185    \u001b[00m \u001b[0m0.3232    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "71/120     \u001b[0m0.0379    \u001b[0m \u001b[0m0.2966    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "72/120     \u001b[0m0.0356    \u001b[0m \u001b[0m0.4828    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "73/120     \u001b[96m0.0164    \u001b[00m \u001b[0m0.3269    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "74/120     \u001b[0m0.0182    \u001b[0m \u001b[95m0.2517    \u001b[00m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "75/120     \u001b[0m0.0324    \u001b[0m \u001b[0m0.3067    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "76/120     \u001b[0m0.0178    \u001b[0m \u001b[0m0.4413    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "77/120     \u001b[96m0.0159    \u001b[00m \u001b[0m0.4453    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "78/120     \u001b[0m0.0169    \u001b[0m \u001b[0m0.4144    \u001b[0m \u001b[0m0.96      \u001b[0m -          0.002      0.48      \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79/120     \u001b[96m0.0155    \u001b[00m \u001b[0m0.3808    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "80/120     \u001b[0m0.0192    \u001b[0m \u001b[0m0.4252    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "81/120     \u001b[96m0.0153    \u001b[00m \u001b[0m0.3302    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "82/120     \u001b[0m0.0159    \u001b[0m \u001b[0m0.3604    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "83/120     \u001b[0m0.0156    \u001b[0m \u001b[0m0.3351    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "84/120     \u001b[0m0.0254    \u001b[0m \u001b[0m0.4883    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "85/120     \u001b[0m0.0288    \u001b[0m \u001b[0m0.3099    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "86/120     \u001b[0m0.0158    \u001b[0m \u001b[0m0.3345    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "87/120     \u001b[96m0.0145    \u001b[00m \u001b[0m0.3033    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "88/120     \u001b[0m0.0243    \u001b[0m \u001b[0m0.4266    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "89/120     \u001b[96m0.0144    \u001b[00m \u001b[0m0.3117    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "90/120     \u001b[0m0.0239    \u001b[0m \u001b[0m0.4503    \u001b[0m \u001b[0m0.96      \u001b[0m -          0.002      0.48      \n",
      "91/120     \u001b[96m0.0136    \u001b[00m \u001b[0m0.3343    \u001b[0m \u001b[0m0.92      \u001b[0m -          0.002      0.48      \n",
      "92/120     \u001b[0m0.0162    \u001b[0m \u001b[0m0.3189    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "93/120     \u001b[0m0.0233    \u001b[0m \u001b[0m0.3541    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "94/120     \u001b[0m0.0140    \u001b[0m \u001b[0m0.3967    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "95/120     \u001b[96m0.0128    \u001b[00m \u001b[0m0.4280    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "96/120     \u001b[0m0.0131    \u001b[0m \u001b[0m0.2761    \u001b[0m \u001b[0m0.96      \u001b[0m -          0.002      0.48      \n",
      "97/120     \u001b[96m0.0122    \u001b[00m \u001b[0m0.3444    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "98/120     \u001b[0m0.0204    \u001b[0m \u001b[0m0.3234    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "99/120     \u001b[96m0.0115    \u001b[00m \u001b[0m0.4404    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "100/120    \u001b[0m0.0119    \u001b[0m \u001b[0m0.2993    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "101/120    \u001b[0m0.0133    \u001b[0m \u001b[0m0.4757    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "102/120    \u001b[0m0.0241    \u001b[0m \u001b[0m0.3435    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "103/120    \u001b[0m0.0121    \u001b[0m \u001b[0m0.3345    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "104/120    \u001b[0m0.0136    \u001b[0m \u001b[0m0.2900    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "105/120    \u001b[96m0.0108    \u001b[00m \u001b[0m0.3207    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "106/120    \u001b[96m0.0104    \u001b[00m \u001b[0m0.4713    \u001b[0m \u001b[0m0.93      \u001b[0m -          0.002      0.48      \n",
      "107/120    \u001b[0m0.0112    \u001b[0m \u001b[0m0.3652    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "108/120    \u001b[0m0.0134    \u001b[0m \u001b[0m0.3085    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "109/120    \u001b[0m0.0105    \u001b[0m \u001b[0m0.3113    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "110/120    \u001b[0m0.0112    \u001b[0m \u001b[0m0.3158    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "111/120    \u001b[0m0.0105    \u001b[0m \u001b[0m0.2892    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "112/120    \u001b[96m0.0101    \u001b[00m \u001b[95m0.2468    \u001b[00m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "113/120    \u001b[96m0.0097    \u001b[00m \u001b[0m0.3782    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "114/120    \u001b[0m0.0104    \u001b[0m \u001b[0m0.3649    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "115/120    \u001b[0m0.0198    \u001b[0m \u001b[0m0.3329    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "116/120    \u001b[0m0.0173    \u001b[0m \u001b[0m0.3336    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "117/120    \u001b[0m0.0100    \u001b[0m \u001b[0m0.2613    \u001b[0m \u001b[0m0.96      \u001b[0m -          0.002      0.48      \n",
      "118/120    \u001b[0m0.0105    \u001b[0m \u001b[0m0.4482    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "119/120    \u001b[0m0.0105    \u001b[0m \u001b[0m0.2805    \u001b[0m \u001b[0m0.95      \u001b[0m -          0.002      0.48      \n",
      "120/120    \u001b[0m0.0119    \u001b[0m \u001b[0m0.4980    \u001b[0m \u001b[0m0.94      \u001b[0m -          0.002      0.48      \n",
      "58.0130573080387\n"
     ]
    }
   ],
   "source": [
    "t0 = time.perf_counter()\n",
    "Resnet18.fit(TRAIN_LOADER,\n",
    "         VAL_LOADER,\n",
    "         CFG.EPOCHS,\n",
    "         CFG.BATCH_SIZE,\n",
    "         CFG.DEVICE)\n",
    "t1 = time.perf_counter()\n",
    "print(t1 - t0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "powerful-nevada",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T13:39:37.009864Z",
     "iopub.status.busy": "2021-05-08T13:39:37.009314Z",
     "iopub.status.idle": "2021-05-08T13:39:37.901738Z",
     "shell.execute_reply": "2021-05-08T13:39:37.901249Z"
    },
    "papermill": {
     "duration": 0.940682,
     "end_time": "2021-05-08T13:39:37.901891",
     "exception": false,
     "start_time": "2021-05-08T13:39:36.961209",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "renewable-secretariat",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T13:39:37.996367Z",
     "iopub.status.busy": "2021-05-08T13:39:37.995804Z",
     "iopub.status.idle": "2021-05-08T13:40:34.091138Z",
     "shell.execute_reply": "2021-05-08T13:40:34.091944Z"
    },
    "papermill": {
     "duration": 56.145498,
     "end_time": "2021-05-08T13:40:34.092196",
     "exception": false,
     "start_time": "2021-05-08T13:39:37.946698",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Model:\n",
      "--------------------\n",
      " Accuracy: 90.1%\n",
      "Precision: 91.0%\n",
      "   Recall: 90.5%\n",
      " F1 Score: 90.3%\n",
      "Current Model:\n",
      "--------------------\n",
      " Accuracy: 91.6%\n",
      "Precision: 92.3%\n",
      "   Recall: 91.6%\n",
      " F1 Score: 91.5%\n"
     ]
    }
   ],
   "source": [
    "#evaluate best model's performance on the test set\n",
    "print('Best Model:')\n",
    "print('-'*20)\n",
    "best_model = deepcopy(Resnet18)\n",
    "best_model.load_model()\n",
    "best_model.eval_model(TEST_LOADER)\n",
    "#evaluate current model's performance on the test set\n",
    "print('Current Model:')\n",
    "print('-'*20)\n",
    "Resnet18.eval_model(TEST_LOADER)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eleven-iceland",
   "metadata": {
    "papermill": {
     "duration": 0.09136,
     "end_time": "2021-05-08T13:40:34.273030",
     "exception": false,
     "start_time": "2021-05-08T13:40:34.181670",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# What's next?\n",
    "\n",
    "- Train function as model class method\n",
    "- LR Scheduler\n",
    "- Train Custom Model\n",
    "- Confusion Matrix\n",
    "- Random & Grid Search\n",
    "- Cross-Validation\n",
    "- Ensemble Learning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "postal-lindsay",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-08T13:40:34.408154Z",
     "iopub.status.busy": "2021-05-08T13:40:34.407271Z",
     "iopub.status.idle": "2021-05-08T13:40:34.409472Z",
     "shell.execute_reply": "2021-05-08T13:40:34.409832Z"
    },
    "papermill": {
     "duration": 0.05594,
     "end_time": "2021-05-08T13:40:34.409985",
     "exception": false,
     "start_time": "2021-05-08T13:40:34.354045",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def cross_val():\n",
    "    '''\n",
    "    ======================\n",
    "       CROSS VALIDATION\n",
    "    ======================\n",
    "    '''\n",
    "    pass"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 7591.635951,
   "end_time": "2021-05-08T13:40:37.120459",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-05-08T11:34:05.484508",
   "version": "2.3.3"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "16fccae819c14f5a9f2abe761fe94b11": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "465c3752ef80471888972aa4ddc64f18": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6c82535f3c844dbc8f4226ec61de6da3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "6ef956557a4640398a518d156c53d449": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_16fccae819c14f5a9f2abe761fe94b11",
       "placeholder": "​",
       "style": "IPY_MODEL_da26b7aad3c94c2bb7bfc018fe0806b9",
       "value": " 44.7M/44.7M [00:01&lt;00:00, 51.3MB/s]"
      }
     },
     "93b1f22c1e11400291e4b743e807a33a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ae20fbe41b234a4487c246bd621d6705",
       "placeholder": "​",
       "style": "IPY_MODEL_d3b1752519fa4bf2bae6d77524177170",
       "value": "100%"
      }
     },
     "971520e5e9d14d9580fd40b15687ea83": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ffe343c87abd4354adf2e215ba8ff5c7",
       "max": 46827520,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_6c82535f3c844dbc8f4226ec61de6da3",
       "value": 46827520
      }
     },
     "9e145c8cd29b4639bcd97058c7d8ba14": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_93b1f22c1e11400291e4b743e807a33a",
        "IPY_MODEL_971520e5e9d14d9580fd40b15687ea83",
        "IPY_MODEL_6ef956557a4640398a518d156c53d449"
       ],
       "layout": "IPY_MODEL_465c3752ef80471888972aa4ddc64f18"
      }
     },
     "ae20fbe41b234a4487c246bd621d6705": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d3b1752519fa4bf2bae6d77524177170": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "da26b7aad3c94c2bb7bfc018fe0806b9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ffe343c87abd4354adf2e215ba8ff5c7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}