Lecture 4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "\n",
    "N, D = 3, 4\n",
    "\n",
    "x = np.random.randn(N, D)\n",
    "y = np.random.randn(N, D)\n",
    "z = np.random.randn(N, D)\n",
    "\n",
    "a = x * y\n",
    "b = a + z\n",
    "c = np.sum(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_c = 1.0\n",
    "grad_b = grad_c * np.ones((N, D))\n",
    "grad_a = grad_b.copy()\n",
    "grad_z = grad_b.copy()\n",
    "grad_x = grad_a * y\n",
    "grad_y = grad_a * x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.76103773,  0.12167502,  0.44386323,  0.33367433],\n",
       "       [ 1.49407907, -0.20515826,  0.3130677 , -0.85409574],\n",
       "       [-2.55298982,  0.6536186 ,  0.8644362 , -0.74216502]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.76405235,  0.40015721,  0.97873798,  2.2408932 ],\n",
       "       [ 1.86755799, -0.97727788,  0.95008842, -0.15135721],\n",
       "       [-0.10321885,  0.4105985 ,  0.14404357,  1.45427351]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grad_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "x_torch = Variable(torch.from_numpy(x), requires_grad=True)\n",
    "y_torch = Variable(torch.from_numpy(y), requires_grad=True)\n",
    "z_torch = Variable(torch.from_numpy(z))\n",
    "\n",
    "# or create new random number by:\n",
    "# x_torch = torch.randn(N, D, requires_grad_True)\n",
    "# y_torch = torch.randn(N, D)\n",
    "# z_torch = torch.randn(N, D)\n",
    "\n",
    "a_torch = x_torch * y_torch\n",
    "b_torch = a_torch + z_torch\n",
    "c_torch = torch.sum(b_torch)\n",
    "c_torch.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.7610,  0.1217,  0.4439,  0.3337],\n",
       "        [ 1.4941, -0.2052,  0.3131, -0.8541],\n",
       "        [-2.5530,  0.6536,  0.8644, -0.7422]], dtype=torch.float64)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_torch.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.7641,  0.4002,  0.9787,  2.2409],\n",
       "        [ 1.8676, -0.9773,  0.9501, -0.1514],\n",
       "        [-0.1032,  0.4106,  0.1440,  1.4543]], dtype=torch.float64)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_torch.grad"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "device = 'cuda:0'\n",
    "\n",
    "# or create new random number by:\n",
    "x_torch = torch.randn(N, D, requires_grad=True, device=device)\n",
    "y_torch = torch.randn(N, D, requires_grad=True, device=device)\n",
    "z_torch = torch.randn(N, D, device=device)\n",
    "\n",
    "a_torch = x_torch * y_torch\n",
    "b_torch = a_torch + z_torch\n",
    "c_torch = torch.sum(b_torch)\n",
    "c_torch.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manually Calculate Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "device = torch.device(\"cpu\")\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "w1 = torch.randn(D_in, H, device=device)\n",
    "w2 = torch.randn(H, D_out, device=device)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    h = x.mm(w1)\n",
    "    h_relu = h.clamp(min=0)\n",
    "    y_pred = h_relu.mm(w2)\n",
    "    loss = (y_pred - y).pow(2).sum().item()\n",
    "    \n",
    "    if t % 100 == 99:\n",
    "        print(t, loss)\n",
    "\n",
    "    grad_y_pred = 2.0 * (y_pred - y)\n",
    "    grad_w2 = h_relu.t().mm(grad_y_pred)\n",
    "    grad_h_relu = grad_y_pred.mm(w2.t())\n",
    "    grad_h = grad_h_relu.clone()\n",
    "    grad_h[h < 0] = 0\n",
    "    grad_w1 = x.t().mm(grad_h)\n",
    "\n",
    "    w1 -= learning_rate * grad_w1\n",
    "    w2 -= learning_rate * grad_w2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Automatically Calculate Gradients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "w1 = torch.randn(D_in, H, requires_grad=True)\n",
    "w2 = torch.randn(H, D_out, requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    y_pred = x.mm(w1).clamp(min = 0).mm(w2)\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    \n",
    "    if t % 100 == 99:\n",
    "        print(t, loss)\n",
    "    \n",
    "    loss.backward()\n",
    "    \n",
    "    with torch.no_grad():   \n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## New Autograd Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "class MyReLU(torch.autograd.Function):\n",
    "    @staticmethod\n",
    "    def forward(ctx, x):\n",
    "        ctx.save_for_backward(x)\n",
    "        return x.clamp(min=0)\n",
    "\n",
    "    @staticmethod\n",
    "    def backward(ctx, grad_y):\n",
    "        x, = ctx.saved_tensors\n",
    "        grad_input = grad_y.clone()\n",
    "        grad_input[x < 0] = 0\n",
    "        return grad_input\n",
    "\n",
    "def my_relu(x):\n",
    "    return MyReLU.apply(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def my_relu(x):\n",
    "    return x.clamp(min=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "w1 = torch.randn(D_in, H, requires_grad = True)\n",
    "w2 = torch.randn(H, D_out, requires_grad = True)\n",
    "\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    y_pred = my_relu(x.mm(w1)).mm(w2)\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "   \n",
    "    loss.backward()\n",
    "\n",
    "    with torch.no_grad():\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out))\n",
    "\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "learning_rate = 1e-2\n",
    "for t in range(500):\n",
    "    y_pred = model(x)\n",
    "    loss = loss_fn(y_pred, y)\n",
    "\n",
    "    model.zero_grad()\n",
    "    \n",
    "    loss.backward()\n",
    "   \n",
    "    with torch.no_grad():\n",
    "        for param in model.parameters():\n",
    "            param -= learning_rate * param.grad"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out))\n",
    "\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "learning_rate = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for t in range(500):\n",
    "    y_pred = model(x)\n",
    "    loss = loss_fn(y_pred, y)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    loss.backward()\n",
    "    \n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: Custom nn Modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class TwoLayerNet(torch.nn.Module):\n",
    "    def __init__(self, D_in, H, D_out):\n",
    "        super(TwoLayerNet, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(D_in, H)\n",
    "        self.linear2 = torch.nn.Linear(H, D_out)\n",
    "\n",
    "    def forward(self, x):\n",
    "        h_relu = self.linear1(x).clamp(min=0)\n",
    "        y_pred = self.linear2(h_relu)\n",
    "        return y_pred\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "model = TwoLayerNet(D_in, H, D_out)\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=1e-4)\n",
    "for t in range(500):\n",
    "    y_pred = model(x)\n",
    "    loss = loss_fn(y_pred, y)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "class ParallelBlock(torch.nn.Module):\n",
    "    def __init__(self, D_in, D_out):\n",
    "        super(ParallelBlock, self).__init__()\n",
    "        self.linear1 = torch.nn.Linear(D_in, D_out)\n",
    "        self.linear2 = torch.nn.Linear(D_in, D_out)\n",
    "    def forward(self, x):\n",
    "        h1 = self.linear1(x)\n",
    "        h2 = self.linear2(x)\n",
    "        return(h1 * h2).clamp(min=0)\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    ParallelBlock(D_in, H),\n",
    "    ParallelBlock(H, H),\n",
    "    torch.nn.Linear(H, D_out))\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)\n",
    "for t in range(500):\n",
    "    y_pred = model(x)\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: Dataset & Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "transform = transforms.Compose(\n",
    "    [transforms.ToTensor(),\n",
    "     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n",
    "\n",
    "batch_size = 4\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=True,download=True, transform=transform)\n",
    "trainloader = torch.utils.data.DataLoader(\n",
    "    trainset, batch_size=batch_size,shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=False,download=True, transform=transform)\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "    testset, batch_size=batch_size,shuffle=False, num_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "class Net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = torch.flatten(x, 1) # flatten all dimensions except batch\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)\n",
    "net = Net()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(2):  # loop over the dataset multiple times\n",
    "\n",
    "    running_loss = 0.0\n",
    "    for i, data in enumerate(trainloader, 0):\n",
    "        # get the inputs; data is a list of [inputs, labels]\n",
    "        inputs, labels = data\n",
    "\n",
    "        # zero the parameter gradients\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # forward + backward + optimize\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        # print statistics\n",
    "        running_loss += loss.item()\n",
    "        if i % 2000 == 1999:    # print every 2000 mini-batches\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, i + 1, running_loss / 2000))\n",
    "            running_loss = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct = 0\n",
    "total = 0\n",
    "# since we're not training, we don't need to calculate the gradients for our outputs\n",
    "with torch.no_grad():\n",
    "    for data in testloader:\n",
    "        images, labels = data\n",
    "        # calculate outputs by running images through the network\n",
    "        outputs = net(images)\n",
    "        # the class with the highest energy is what we choose as prediction\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "\n",
    "print('Accuracy of the network on the 10000 test images: %d %%' % (\n",
    "    100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PyTorch: Saving and Loading Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model's state_dict:\n",
      "conv1.weight \t torch.Size([6, 3, 5, 5])\n",
      "conv1.bias \t torch.Size([6])\n",
      "conv2.weight \t torch.Size([16, 6, 5, 5])\n",
      "conv2.bias \t torch.Size([16])\n",
      "fc1.weight \t torch.Size([120, 400])\n",
      "fc1.bias \t torch.Size([120])\n",
      "fc2.weight \t torch.Size([84, 120])\n",
      "fc2.bias \t torch.Size([84])\n",
      "fc3.weight \t torch.Size([10, 84])\n",
      "fc3.bias \t torch.Size([10])\n"
     ]
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "\n",
    "# Define model\n",
    "class TheModelClass(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(TheModelClass, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(3, 6, 5)\n",
    "        self.pool = nn.MaxPool2d(2, 2)\n",
    "        self.conv2 = nn.Conv2d(6, 16, 5)\n",
    "        self.fc1 = nn.Linear(16 * 5 * 5, 120)\n",
    "        self.fc2 = nn.Linear(120, 84)\n",
    "        self.fc3 = nn.Linear(84, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.pool(F.relu(self.conv1(x)))\n",
    "        x = self.pool(F.relu(self.conv2(x)))\n",
    "        x = x.view(-1, 16 * 5 * 5)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = F.relu(self.fc2(x))\n",
    "        x = self.fc3(x)\n",
    "        return x\n",
    "\n",
    "# Initialize model\n",
    "model = TheModelClass()\n",
    "\n",
    "# Print model's state_dict\n",
    "print(\"Model's state_dict:\")\n",
    "for param_tensor in model.state_dict():\n",
    "    print(param_tensor, \"\\t\", model.state_dict()[param_tensor].size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save model parameters\n",
    "torch.save(model.state_dict(), PATH)\n",
    "\n",
    "# initialize a new model and load parameters into it\n",
    "model = TheModelClass()\n",
    "model.load_state_dict(torch.load(PATH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the whole model\n",
    "torch.save(model, PATH)\n",
    "\n",
    "# load the whole model\n",
    "model = torch.load(PATH)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensorboard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "import torch\n",
    "\n",
    "writer = SummaryWriter('runs/lecture_4')\n",
    "\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "x = torch.randn(N, D_in)\n",
    "y = torch.randn(N, D_out)\n",
    "\n",
    "model = torch.nn.Sequential(\n",
    "    ParallelBlock(D_in, H),\n",
    "    ParallelBlock(H, H),\n",
    "    torch.nn.Linear(H, D_out))\n",
    "loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "writer.add_graph(model, x)\n",
    "\n",
    "learning_rate = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "\n",
    "for t in range(500):\n",
    "    y_pred = model(x)\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    \n",
    "    writer.add_scalar('training loss', loss, t)\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TensorFlow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow: Neural Net (Pre-2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "# import tensorflow as tf\n",
    "import tensorflow.compat.v1 as tf\n",
    "tf.disable_v2_behavior()\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.placeholder(tf.float32, shape=(N, D))\n",
    "y = tf.placeholder(tf.float32, shape=(N, D))\n",
    "w1 = tf.placeholder(tf.float32, shape=(D, H))\n",
    "w2 = tf.placeholder(tf.float32, shape=(H, D))\n",
    "\n",
    "h = tf.maximum(tf.matmul(x, w1), 0)\n",
    "y_pred = tf.matmul(h, w2)\n",
    "diff = y_pred - y\n",
    "loss = tf.reduce_mean(tf.reduce_sum(diff ** 2, axis=1))\n",
    "grad_w1, grad_w2 = tf.gradients(loss, [w1, w2])\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    values = {\n",
    "        x: np.random.randn(N, D),\n",
    "        w1: np.random.randn(D, H),\n",
    "        w2: np.random.randn(H, D),\n",
    "        y: np.random.randn(N, D), }\n",
    "    out = sess.run([loss, grad_w1, grad_w2], feed_dict=values)\n",
    "    loss_val, grad_w1_val, grad_w2_val = out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you run the cell above, you should restart the notebook because you have called tf.disable_v2_behavior(), which turn off eager mode."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow: Neural Net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "y = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "w1 = tf.Variable(tf.random.uniform((D, H)))\n",
    "w2 = tf.Variable(tf.random.uniform((H, D)))\n",
    "\n",
    "with tf.GradientTape() as tape:\n",
    "    h = tf.maximum(tf.matmul(x, w1), 0)\n",
    "    y_pred = tf.matmul(h, w2)\n",
    "    diff = y_pred - y\n",
    "    loss = tf.reduce_mean(tf.reduce_sum(diff ** 2, axis=1))\n",
    "gradients = tape.gradient(loss, [w1, w2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gradients)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor: shape=(1000, 100), dtype=float32, numpy=\n",
       "array([[  69757.28 ,   47390.855,   77866.266, ...,   97641.016,\n",
       "          85308.18 ,   87784.734],\n",
       "       [  21925.076,   32033.234,   55169.79 , ...,   51803.402,\n",
       "          29309.635,   37013.484],\n",
       "       [ -58208.332,  -80454.07 ,  -83268.62 , ...,  -55570.953,\n",
       "         -45419.   ,  -57458.984],\n",
       "       ...,\n",
       "       [ -82133.46 ,  -70326.445,  -93047.16 , ..., -111118.875,\n",
       "         -78491.17 ,  -90642.45 ],\n",
       "       [ -32500.71 ,  -45708.32 ,  -46061.074, ...,  -56110.71 ,\n",
       "         -49317.277,  -36658.812],\n",
       "       [ -37277.84 ,  -19697.875,  -45709.242, ...,  -48515.668,\n",
       "         -54433.805,  -56148.664]], dtype=float32)>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gradients[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TensorFlow: Optimizer and Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "y = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "w1 = tf.Variable(tf.random.uniform((D, H)))\n",
    "w2 = tf.Variable(tf.random.uniform((H, D)))\n",
    "\n",
    "optimizer = tf.optimizers.SGD(1e-6)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(50):\n",
    "    with tf.GradientTape() as tape:\n",
    "        h = tf.maximum(tf.matmul(x, w1), 0)\n",
    "        y_pred = tf.matmul(h, w2)\n",
    "        diff = y_pred - y\n",
    "        loss = tf.losses.MeanSquaredError()(y_pred, y)\n",
    "    gradients = tape.gradient(loss, [w1, w2])\n",
    "    optimizer.apply_gradients(zip(gradients, [w1, w2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Keras: High-Level Wrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "y = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Dense(H, input_shape=(D,), activation=tf.nn.relu))\n",
    "model.add(tf.keras.layers.Dense(D))\n",
    "optimizer = tf.optimizers.SGD(1e-1)\n",
    "\n",
    "losses = []\n",
    "for t in range(50):\n",
    "    with tf.GradientTape() as tape:\n",
    "        y_pred = model(x)\n",
    "        loss = tf.losses.MeanSquaredError()(y_pred, y)\n",
    "    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "y = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Dense(H, input_shape=(D,), activation=tf.nn.relu))\n",
    "model.add(tf.keras.layers.Dense(D))\n",
    "optimizer = tf.optimizers.SGD(1e-1)\n",
    "\n",
    "model.compile(loss=tf.keras.losses.MeanSquaredError(), optimizer=optimizer)\n",
    "history = model.fit(x, y, steps_per_epoch=1, epochs=50, batch_size=N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "y = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Dense(H, input_shape=(D,), activation=tf.nn.relu))\n",
    "model.add(tf.keras.layers.Dense(D))\n",
    "optimizer = tf.optimizers.SGD(1e-1)\n",
    "\n",
    "\n",
    "@tf.function\n",
    "def model_func(x, y):\n",
    "    y_pred = model(x)\n",
    "    loss = tf.losses.MeanSquaredError()(y_pred, y)\n",
    "    return y_pred, loss\n",
    "\n",
    "\n",
    "for t in range(50):\n",
    "    with tf.GradientTape() as tape:\n",
    "        y_pred, loss = model_func(x, y)\n",
    "    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import timeit\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "y = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Dense(H, input_shape=(D,), activation=tf.nn.relu))\n",
    "model.add(tf.keras.layers.Dense(D))\n",
    "optimizer = tf.optimizers.SGD(1e-1)\n",
    "\n",
    "\n",
    "@tf.function\n",
    "def model_static(x, y):\n",
    "    y_pred = model(x)\n",
    "    loss = tf.losses.MeanSquaredError()(y_pred, y)\n",
    "    return y_pred, loss\n",
    "\n",
    "\n",
    "def model_dynamic(x, y):\n",
    "    y_pred = model(x)\n",
    "    loss = tf.losses.MeanSquaredError()(y_pred, y)\n",
    "\n",
    "\n",
    "print(\"dynamic graph:\", timeit.timeit(lambda: model_dynamic(x, y), number=10))\n",
    "print(\"static graph:\", timeit.timeit(lambda: model_static(x, y), number=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import timeit\n",
    "\n",
    "N, D, H = 64, 1000, 100\n",
    "x = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "y = tf.convert_to_tensor(np.random.randn(N, D), np.float32)\n",
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.layers.Dense(H, input_shape=(D,), activation=tf.nn.relu))\n",
    "model.add(tf.keras.layers.Dense(D))\n",
    "optimizer = tf.optimizers.SGD(1e-1)\n",
    "\n",
    "\n",
    "@tf.function\n",
    "def model_static(x, y):\n",
    "    y_pred = model(x)\n",
    "    loss = tf.losses.MeanSquaredError()(y_pred, y)\n",
    "    return y_pred, loss\n",
    "\n",
    "\n",
    "def model_dynamic(x, y):\n",
    "    y_pred = model(x)\n",
    "    loss = tf.losses.MeanSquaredError()(y_pred, y)\n",
    "\n",
    "\n",
    "print(\"dynamic graph:\", timeit.timeit(lambda: model_dynamic(x, y), number=1000))\n",
    "print(\"static graph:\", timeit.timeit(lambda: model_static(x, y), number=1000))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}