PyTorch Part 5 Code Refactoring Training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.7"
    },
    "colab": {
      "name": "PyTorch Part 5 Code Refactoring Training.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "KunYp1e_SKFS",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "af26c3d6-82ad-45bf-d3ae-03f35664620d"
      },
      "source": [
        "import torch.utils.tensorboard\n",
        "%load_ext tensorboard"
      ],
      "execution_count": 103,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The tensorboard extension is already loaded. To reload it, use:\n",
            "  %reload_ext tensorboard\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "g_WNc7NCSKFc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import torch.optim as optim\n",
        "\n",
        "import torchvision\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "torch.set_printoptions(linewidth=120)\n",
        "torch.set_grad_enabled(True)\n",
        "\n",
        "from torch.utils.tensorboard import SummaryWriter"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "9H5aOs7ASKFj",
        "colab_type": "code",
        "outputId": "4af244b2-9ebc-45be-ad80-db78b0185190",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "print(torch.__version__)\n",
        "print(torchvision.__version__)"
      ],
      "execution_count": 105,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.4.0\n",
            "0.5.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "AY6Zn_E1SKFr",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def accuracy_nums(preds,labels):\n",
        "    return (preds.argmax(dim=1).eq(labels).sum().item())\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "PKWb10qySKFw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class Network(nn.Module): #Line 1\n",
        "    def __init__(self):\n",
        "        super().__init__() # Line 3\n",
        "        \n",
        "        self.conv1=nn.Conv2d(in_channels=1, out_channels=6, kernel_size=5)\n",
        "        self.conv2=nn.Conv2d(in_channels=6, out_channels=12, kernel_size=5)\n",
        "        \n",
        "        self.fc1=nn.Linear(in_features=12*4*4, out_features=120)# 12 comnes from output of 12 channel in conv2\n",
        "        self.fc2=nn.Linear(in_features=120, out_features=60)\n",
        "        \n",
        "        self.out=nn.Linear(in_features=60, out_features=10)\n",
        "        \n",
        "    def forward(self, t):\n",
        "        # (1) Input Layer\n",
        "        #t=t\n",
        "        \n",
        "        # (2) hidden conv1 Layer\n",
        "        t=self.conv1(t)\n",
        "        t=F.relu(t)\n",
        "        t=F.max_pool2d(t, kernel_size=2, stride=2)\n",
        "        \n",
        "        # (3) hidden conv2 Layer\n",
        "        t=self.conv2(t)\n",
        "        t=F.relu(t)\n",
        "        t=F.max_pool2d(t, kernel_size=2, stride=2)\n",
        "        \n",
        "        # (4) hidden linear1 layer\n",
        "        t=t.reshape(-1, 12*4*4) # Need to flatten before using FC layer \n",
        "        t=self.fc1(t)\n",
        "        t=F.relu(t)\n",
        "        \n",
        "        # (5) Hidden linear2 layer\n",
        "        t=self.fc2(t)\n",
        "        t=F.relu(t)\n",
        "        \n",
        "        # (6) Output layer\n",
        "        t=self.out(t)\n",
        "        # Avoiding softmax because we will use cross entropy loss fucntion which has built in softmax fucntion.\n",
        "        #t=F.softmax(t, dim=1) \n",
        "        # THhis alllows the network to not compute the additiional operation while network is been infrenced. But computaion will of course work.\n",
        "        \n",
        "        \n",
        "        \n",
        "        return t\n",
        "network=Network()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "id": "js4DVkx1SKF2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_set=torchvision.datasets.FashionMNIST(\n",
        "    root='./data/FashionMINIST',\n",
        "    train=True,\n",
        "    download=True,\n",
        "    transform=transforms.Compose( # For composition of transforamtion here onlu one transformation is sufficient \n",
        "        [transforms.ToTensor()]\n",
        "        )\n",
        ")\n",
        "train_loader=torch.utils.data.DataLoader(\n",
        "    train_set, \n",
        "    batch_size=100, #Default batach size =1\n",
        "    shuffle=True\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Gtro2lMklsma",
        "colab_type": "text"
      },
      "source": [
        "## Hyperparameter Tuning & Experimentation auto mode"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uPKFJc3SBqZS",
        "colab_type": "text"
      },
      "source": [
        "* Earlier process of training was inefficent, now we will make them effcient using runs\n",
        "### Objective:\n",
        "\n",
        "1. We want automatic addition of parameters to set just by defining them.\n",
        "2. We want program to unpack the parameter by itself"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cVZe5oZXpxWH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import  OrderedDict\n",
        "from collections import namedtuple\n",
        "from itertools import product"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqbeaWekDHnZ",
        "colab_type": "code",
        "outputId": "7d1a1fb7-263f-40b2-ac01-c3b982030658",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "parameters=dict(\n",
        "    lr=[0.01,0.001,0.0001,0.00001],\n",
        "    batch_size=[100,1000,10000],\n",
        "    shuffle=[False]\n",
        ")\n",
        "parameters.keys()"
      ],
      "execution_count": 110,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['lr', 'batch_size', 'shuffle'])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 110
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B75qbC_PDWez",
        "colab_type": "code",
        "outputId": "da003659-4313-4d36-e1fe-5233d17c0424",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "parameters.values()"
      ],
      "execution_count": 111,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_values([[0.01, 0.001, 0.0001, 1e-05], [100, 1000, 10000], [False]])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 111
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VDw4e7A2Dkdp",
        "colab_type": "code",
        "outputId": "fa3653f3-21b7-4d47-e769-35a9f3b327b2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "# namedtuple help us access the tuple by name also, rather then by only index\n",
        "# Example:\n",
        "Point = namedtuple('Point', ['x', 'y'])\n",
        "p=Point(11,12)\n",
        "p # Named tuple behave as tuple but also as function, they can accept parameters"
      ],
      "execution_count": 112,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Point(x=11, y=12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 112
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wtTsqVbyGfKH",
        "colab_type": "code",
        "outputId": "13a9f5f6-30e8-42a2-e1f2-e2b84eaf094b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "p.x+p.y"
      ],
      "execution_count": 113,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "23"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 113
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8_7GIk6MGY1I",
        "colab_type": "code",
        "outputId": "cf1f9fbc-ef7f-4ad9-b9d8-6ef2b5570231",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "Run=namedtuple('Run',parameters.keys())\n",
        "Run"
      ],
      "execution_count": 114,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "__main__.Run"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 114
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5VnGY47vD57-",
        "colab_type": "code",
        "outputId": "76917214-c5aa-4e8b-e8f6-4447f52540bc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 230
        }
      },
      "source": [
        "runs=[]\n",
        "for v in product(*parameters.values()):\n",
        "  runs.append(Run(*v))# Star \"*\" helps us to pass the value as paramters rather then the tuple itself\n",
        "runs"
      ],
      "execution_count": 115,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Run(lr=0.01, batch_size=100, shuffle=False),\n",
              " Run(lr=0.01, batch_size=1000, shuffle=False),\n",
              " Run(lr=0.01, batch_size=10000, shuffle=False),\n",
              " Run(lr=0.001, batch_size=100, shuffle=False),\n",
              " Run(lr=0.001, batch_size=1000, shuffle=False),\n",
              " Run(lr=0.001, batch_size=10000, shuffle=False),\n",
              " Run(lr=0.0001, batch_size=100, shuffle=False),\n",
              " Run(lr=0.0001, batch_size=1000, shuffle=False),\n",
              " Run(lr=0.0001, batch_size=10000, shuffle=False),\n",
              " Run(lr=1e-05, batch_size=100, shuffle=False),\n",
              " Run(lr=1e-05, batch_size=1000, shuffle=False),\n",
              " Run(lr=1e-05, batch_size=10000, shuffle=False)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 115
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3n7QGI8eHVgA",
        "colab_type": "text"
      },
      "source": [
        "#### Making class of above steps we have:\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r2Y0-plyE_U0",
        "colab_type": "text"
      },
      "source": [
        "* The RunBuilder class encapsulates the get_runs function. \n",
        "* This is considered to be better design than simply having the function sitting inside the global scope of the program. \n",
        "* This becomes especially true if there are multiple functions that logically belong together.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ebrqiMtPBXUj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class RunBuilder():\n",
        "  @staticmethod \n",
        "  # This allows us to call the function without calling the class, hence saving memory. \n",
        "  # This is done becuase we do not want global functions and neither extra class objects.  \n",
        "  def get_runs(parameters):\n",
        "    Run=namedtuple(\"Run\", parameters.keys())\n",
        "    runs=[]\n",
        "    for v in product(*parameters.values()):\n",
        "      runs.append(Run(*v))\n",
        "    return runs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kyhHndHaOgaQ",
        "colab_type": "text"
      },
      "source": [
        "### Making Run Manager Class to declutter training loop\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SfWrmWkTOwmR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import pandas as pd\n",
        "from IPython.display import clear_output\n",
        "import json\n",
        "\n",
        "class RunManager():\n",
        "  def __init__(self):\n",
        "    \n",
        "    self.epoch_count=0\n",
        "    self.epoch_loss=0\n",
        "    self.epoch_num_correct=0\n",
        "    self.epoch_start_time=None\n",
        "\n",
        "    self.run_params=None # From run builder class\n",
        "    self.run_count=0\n",
        "    self.run_data=[]\n",
        "    self.run_start_time=None\n",
        "\n",
        "    self.network=None\n",
        "    self.loader=None\n",
        "    self.tb=None\n",
        "\n",
        "  def begin_run(self, run, network, loader):\n",
        "    self.run_start_time=time.time()\n",
        "\n",
        "    self.run_params=run\n",
        "    self.run_count+=1\n",
        "\n",
        "    self.network=network\n",
        "    self.loader=loader\n",
        "    self.tb=SummaryWriter(comment=str(run))\n",
        "    \n",
        "    images, labels=next(iter(self.loader))\n",
        "    grid=torchvision.utils.make_grid(images)\n",
        "\n",
        "    self.tb.add_image('images', grid)\n",
        "    self.tb.add_graph(self.network, images)\n",
        "\n",
        "  def end_run(self):\n",
        "    self.tb.close()\n",
        "    self.epoch_count=0\n",
        "\n",
        "  def begin_epoch(self):\n",
        "    self.epoch_start_time=time.time()\n",
        "\n",
        "    self.epoch_count+=1\n",
        "    self.epoch_loss=0\n",
        "    self.epoch_num_correct=0\n",
        "\n",
        "\n",
        "  def end_epoch(self):\n",
        "    epoch_duration=time.time()-self.epoch_start_time\n",
        "    run_duration=time.time()-self.run_start_time\n",
        "\n",
        "    loss=self.epoch_loss/len(self.loader.dataset)\n",
        "    accuracy=self.epoch_num_correct/len(self.loader.dataset)\n",
        "\n",
        "    self.tb.add_scalar(\"Loss\", loss, self.epoch_count)\n",
        "    self.tb.add_scalar(\"Accuracy\", accuracy, self.epoch_count)\n",
        "    \n",
        "    for name, param in self.network.named_parameters():\n",
        "        self.tb.add_histogram(name, param,self.epoch_count)\n",
        "        self.tb.add_histogram(str(name)+\".grad\", param.grad, self.epoch_count)\n",
        "  \n",
        "    results=OrderedDict()\n",
        "    results[\"run\"]=self.run_count\n",
        "    results[\"epoch\"]=self.epoch_count\n",
        "    results[\"loss\"]=loss\n",
        "    results[\"accuracy\"]=accuracy\n",
        "    results[\"epoch duration\"]=epoch_duration\n",
        "    results[\"run duration\"]=run_duration\n",
        "\n",
        "    for index, value in self.run_params._asdict().items(): results[index]=value\n",
        "    self.run_data.append(results)\n",
        "    df=pd.DataFrame.from_dict(self.run_data, orient='columns')\n",
        "\n",
        "    clear_output(wait=True)\n",
        "    display(df)\n",
        "    \n",
        "  def track_loss(self, loss):\n",
        "    self.epoch_loss +=loss.item()*self.loader.batch_size\n",
        "\n",
        "  def track_num_correct(self, preds, labels):\n",
        "    self.epoch_num_correct+=self._get_num_correct(preds, labels)\n",
        "\n",
        "  def _get_num_correct(self, preds,labels): # _ here tells that this function is internal\n",
        "    return preds.argmax(dim=1).eq(labels).sum().item()\n",
        "\n",
        "  def save(self, fileName):\n",
        "    pd.DataFrame.from_dict(\n",
        "        self.run_data,\n",
        "        orient='columns'\n",
        "\n",
        "    ).to_csv(str(fileName)+\".csv\")\n",
        "\n",
        "    with open(str(fileName)+\".json\", 'w', encoding='utf8') as f:\n",
        "      json.dump(self.run_data, f, ensure_ascii=False, indent=4)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "trusted": true,
        "colab_type": "code",
        "outputId": "22f302a2-c928-4fbf-cfd3-d65f4861fffd",
        "id": "NzyXYoJwge_K",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "parameters=dict(\n",
        "    lr=[0.01,0.001],\n",
        "    batch_size=[100,10000],\n",
        "    shuffle=[False]\n",
        ")\n",
        "\n",
        "m=RunManager()\n",
        "\n",
        "for run in RunBuilder.get_runs(parameters):\n",
        "  # comment=str(run)\n",
        "\n",
        "  lr=run.lr,\n",
        "  batch_size=run.batch_size\n",
        "  shuffle=run.shuffle\n",
        "\n",
        "  network=Network()\n",
        "\n",
        "  train_loader=torch.utils.data.DataLoader(\n",
        "      train_set,\n",
        "      batch_size=batch_size,\n",
        "      shuffle=shuffle,\n",
        "    )\n",
        "  \n",
        "  optimizer=optim.Adam(network.parameters(), lr=run.lr)\n",
        "\n",
        "\n",
        "  m.begin_run(run,network, train_loader)\n",
        "\n",
        "  for epoch in range(10):\n",
        "      m.begin_epoch()\n",
        "\n",
        "      for batch in train_loader:\n",
        "          images, labels =batch\n",
        "\n",
        "          preds=network(images) # Predict from Batch\n",
        "          loss=F.cross_entropy(preds, labels)# This has the all the data to for back prop\n",
        "\n",
        "          optimizer.zero_grad() # Needed becuase Pytorch makes gradiesnt cumulative, which we do not want\n",
        "          loss.backward() # Back-prop to find gradients\n",
        "          optimizer.step() # Updating weights\n",
        "\n",
        "          m.track_loss(loss)\n",
        "          m.track_num_correct(preds, labels)\n",
        "\n",
        "      m.end_epoch()\n",
        "  m.end_run()\n",
        "m.save('results')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>run</th>\n",
              "      <th>epoch</th>\n",
              "      <th>loss</th>\n",
              "      <th>accuracy</th>\n",
              "      <th>epoch duration</th>\n",
              "      <th>run duration</th>\n",
              "      <th>lr</th>\n",
              "      <th>batch_size</th>\n",
              "      <th>shuffle</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0.563334</td>\n",
              "      <td>0.788967</td>\n",
              "      <td>15.688773</td>\n",
              "      <td>15.819127</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0.390641</td>\n",
              "      <td>0.856783</td>\n",
              "      <td>15.785280</td>\n",
              "      <td>31.714363</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0.355015</td>\n",
              "      <td>0.869167</td>\n",
              "      <td>15.630134</td>\n",
              "      <td>47.457055</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0.336299</td>\n",
              "      <td>0.876067</td>\n",
              "      <td>15.671265</td>\n",
              "      <td>63.260104</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>0.327058</td>\n",
              "      <td>0.879283</td>\n",
              "      <td>15.596707</td>\n",
              "      <td>78.971243</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>0.320862</td>\n",
              "      <td>0.882500</td>\n",
              "      <td>15.632622</td>\n",
              "      <td>94.723215</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>0.315740</td>\n",
              "      <td>0.884567</td>\n",
              "      <td>15.769847</td>\n",
              "      <td>110.610630</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>0.312779</td>\n",
              "      <td>0.886300</td>\n",
              "      <td>15.951600</td>\n",
              "      <td>126.688384</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>0.307433</td>\n",
              "      <td>0.885633</td>\n",
              "      <td>15.827248</td>\n",
              "      <td>142.636922</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "      <td>10</td>\n",
              "      <td>0.305354</td>\n",
              "      <td>0.887000</td>\n",
              "      <td>15.886459</td>\n",
              "      <td>158.647893</td>\n",
              "      <td>0.010</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2.116418</td>\n",
              "      <td>0.236300</td>\n",
              "      <td>14.187933</td>\n",
              "      <td>21.908264</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1.416323</td>\n",
              "      <td>0.471967</td>\n",
              "      <td>14.195595</td>\n",
              "      <td>36.230755</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>1.030913</td>\n",
              "      <td>0.600100</td>\n",
              "      <td>14.231872</td>\n",
              "      <td>50.585569</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>0.872122</td>\n",
              "      <td>0.668333</td>\n",
              "      <td>14.171459</td>\n",
              "      <td>64.882834</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>0.782402</td>\n",
              "      <td>0.697233</td>\n",
              "      <td>14.058575</td>\n",
              "      <td>79.077844</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>0.715307</td>\n",
              "      <td>0.720083</td>\n",
              "      <td>14.151730</td>\n",
              "      <td>93.353997</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>2</td>\n",
              "      <td>7</td>\n",
              "      <td>0.664138</td>\n",
              "      <td>0.740250</td>\n",
              "      <td>14.123559</td>\n",
              "      <td>107.609992</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0.621640</td>\n",
              "      <td>0.754383</td>\n",
              "      <td>14.125620</td>\n",
              "      <td>121.861334</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>2</td>\n",
              "      <td>9</td>\n",
              "      <td>0.587274</td>\n",
              "      <td>0.768200</td>\n",
              "      <td>14.209242</td>\n",
              "      <td>136.191067</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>0.566268</td>\n",
              "      <td>0.777283</td>\n",
              "      <td>14.189445</td>\n",
              "      <td>150.510084</td>\n",
              "      <td>0.010</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0.765995</td>\n",
              "      <td>0.710717</td>\n",
              "      <td>15.946132</td>\n",
              "      <td>16.076290</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0.489363</td>\n",
              "      <td>0.819233</td>\n",
              "      <td>16.030597</td>\n",
              "      <td>32.232774</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>0.412751</td>\n",
              "      <td>0.851500</td>\n",
              "      <td>16.021864</td>\n",
              "      <td>48.379370</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>0.371405</td>\n",
              "      <td>0.866183</td>\n",
              "      <td>16.045956</td>\n",
              "      <td>64.557679</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "      <td>0.345033</td>\n",
              "      <td>0.874267</td>\n",
              "      <td>16.092530</td>\n",
              "      <td>80.772673</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>3</td>\n",
              "      <td>6</td>\n",
              "      <td>0.324508</td>\n",
              "      <td>0.881150</td>\n",
              "      <td>16.002841</td>\n",
              "      <td>96.900486</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>3</td>\n",
              "      <td>7</td>\n",
              "      <td>0.308175</td>\n",
              "      <td>0.886883</td>\n",
              "      <td>15.999892</td>\n",
              "      <td>113.040430</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>3</td>\n",
              "      <td>8</td>\n",
              "      <td>0.294865</td>\n",
              "      <td>0.892717</td>\n",
              "      <td>16.043195</td>\n",
              "      <td>129.213759</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>3</td>\n",
              "      <td>9</td>\n",
              "      <td>0.283298</td>\n",
              "      <td>0.896400</td>\n",
              "      <td>16.021567</td>\n",
              "      <td>145.385956</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>3</td>\n",
              "      <td>10</td>\n",
              "      <td>0.273086</td>\n",
              "      <td>0.899700</td>\n",
              "      <td>16.007265</td>\n",
              "      <td>161.522504</td>\n",
              "      <td>0.001</td>\n",
              "      <td>100</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>2.292742</td>\n",
              "      <td>0.199367</td>\n",
              "      <td>14.246111</td>\n",
              "      <td>22.076545</td>\n",
              "      <td>0.001</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>2.236071</td>\n",
              "      <td>0.447967</td>\n",
              "      <td>14.219268</td>\n",
              "      <td>36.425139</td>\n",
              "      <td>0.001</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>2.091525</td>\n",
              "      <td>0.507483</td>\n",
              "      <td>14.311486</td>\n",
              "      <td>50.884022</td>\n",
              "      <td>0.001</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>1.793418</td>\n",
              "      <td>0.541983</td>\n",
              "      <td>14.301328</td>\n",
              "      <td>65.334593</td>\n",
              "      <td>0.001</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>1.389341</td>\n",
              "      <td>0.591767</td>\n",
              "      <td>14.315494</td>\n",
              "      <td>79.785230</td>\n",
              "      <td>0.001</td>\n",
              "      <td>10000</td>\n",
              "      <td>False</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    run  epoch      loss  accuracy  ...  run duration     lr  batch_size  shuffle\n",
              "0     1      1  0.563334  0.788967  ...     15.819127  0.010         100    False\n",
              "1     1      2  0.390641  0.856783  ...     31.714363  0.010         100    False\n",
              "2     1      3  0.355015  0.869167  ...     47.457055  0.010         100    False\n",
              "3     1      4  0.336299  0.876067  ...     63.260104  0.010         100    False\n",
              "4     1      5  0.327058  0.879283  ...     78.971243  0.010         100    False\n",
              "5     1      6  0.320862  0.882500  ...     94.723215  0.010         100    False\n",
              "6     1      7  0.315740  0.884567  ...    110.610630  0.010         100    False\n",
              "7     1      8  0.312779  0.886300  ...    126.688384  0.010         100    False\n",
              "8     1      9  0.307433  0.885633  ...    142.636922  0.010         100    False\n",
              "9     1     10  0.305354  0.887000  ...    158.647893  0.010         100    False\n",
              "10    2      1  2.116418  0.236300  ...     21.908264  0.010       10000    False\n",
              "11    2      2  1.416323  0.471967  ...     36.230755  0.010       10000    False\n",
              "12    2      3  1.030913  0.600100  ...     50.585569  0.010       10000    False\n",
              "13    2      4  0.872122  0.668333  ...     64.882834  0.010       10000    False\n",
              "14    2      5  0.782402  0.697233  ...     79.077844  0.010       10000    False\n",
              "15    2      6  0.715307  0.720083  ...     93.353997  0.010       10000    False\n",
              "16    2      7  0.664138  0.740250  ...    107.609992  0.010       10000    False\n",
              "17    2      8  0.621640  0.754383  ...    121.861334  0.010       10000    False\n",
              "18    2      9  0.587274  0.768200  ...    136.191067  0.010       10000    False\n",
              "19    2     10  0.566268  0.777283  ...    150.510084  0.010       10000    False\n",
              "20    3      1  0.765995  0.710717  ...     16.076290  0.001         100    False\n",
              "21    3      2  0.489363  0.819233  ...     32.232774  0.001         100    False\n",
              "22    3      3  0.412751  0.851500  ...     48.379370  0.001         100    False\n",
              "23    3      4  0.371405  0.866183  ...     64.557679  0.001         100    False\n",
              "24    3      5  0.345033  0.874267  ...     80.772673  0.001         100    False\n",
              "25    3      6  0.324508  0.881150  ...     96.900486  0.001         100    False\n",
              "26    3      7  0.308175  0.886883  ...    113.040430  0.001         100    False\n",
              "27    3      8  0.294865  0.892717  ...    129.213759  0.001         100    False\n",
              "28    3      9  0.283298  0.896400  ...    145.385956  0.001         100    False\n",
              "29    3     10  0.273086  0.899700  ...    161.522504  0.001         100    False\n",
              "30    4      1  2.292742  0.199367  ...     22.076545  0.001       10000    False\n",
              "31    4      2  2.236071  0.447967  ...     36.425139  0.001       10000    False\n",
              "32    4      3  2.091525  0.507483  ...     50.884022  0.001       10000    False\n",
              "33    4      4  1.793418  0.541983  ...     65.334593  0.001       10000    False\n",
              "34    4      5  1.389341  0.591767  ...     79.785230  0.001       10000    False\n",
              "\n",
              "[35 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}