AllinOne.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Thesis_for_the_win-3-complete.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [
        "HpDzAdmUDPa3",
        "V1mOf-5jBZ2c",
        "0_RVPrbVBShK",
        "UFWz7BqCC8jx",
        "MqolTa3uBou9",
        "cqQSPk9mLjX4",
        "1YsCUgo8CFem",
        "axHPEiXjL030",
        "sbBE5e1igFhx",
        "KeXbUEsty-xg",
        "heGkEQ-uCb0o",
        "cwuZDuGCCx6D",
        "2ciwqcQZDEqy",
        "F8nVTx0BowPY",
        "fW0-SHrw54E_",
        "YVb3ZA_wjTw-",
        "eOHeGHo6jZOk",
        "UAdjDSygDNwq",
        "-LNPY8lhr80t",
        "CG5xH2NZ2BGu",
        "fRTUFMvzQV3i",
        "1WZi86qgysUL",
        "PB2DiufeJvyd",
        "i7_6QSK1JrTn",
        "kpE9ie499X4a",
        "v52wNTlqQct3",
        "WCQ9B9r5wf1p",
        "db7UY-yCYP_c",
        "SXQuiKKR0yTd"
      ]
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3.8.2 64-bit"
    },
    "accelerator": "GPU",
    "interpreter": {
      "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WB_y_9av0xM-"
      },
      "source": [
        "# **General**\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HpDzAdmUDPa3"
      },
      "source": [
        "## Mount\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4BTaL9IS3jTt"
      },
      "source": [
        "from google.colab import drive \n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V1mOf-5jBZ2c"
      },
      "source": [
        "## Import Packages\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0dgs6Yj3Rb3p"
      },
      "source": [
        "%tensorflow_version 1.x\n",
        "from keras.models import Sequential\n",
        "from keras.layers.core import Flatten, Dense, Dropout, Reshape, Lambda\n",
        "from keras.layers.convolutional import Conv2D, Convolution2D, MaxPooling2D, ZeroPadding2D\n",
        "from keras.optimizers import Adam\n",
        "import tensorflow as tf\n",
        "from keras import backend as K\n",
        "from IPython.display import SVG\n",
        "from keras.utils.vis_utils import model_to_dot\n",
        "from keras.layers.advanced_activations import LeakyReLU\n",
        "from keras.layers import Input, Dense\n",
        "from keras.models import Model\n",
        "import matplotlib.pyplot as plt\n",
        "from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint\n",
        "import copy\n",
        "import cv2, os\n",
        "import numpy as np\n",
        "from random import shuffle \n",
        "from keras.applications.vgg16 import VGG16 \n",
        "from keras.applications.mobilenet_v2 import MobileNetV2\n",
        "import pickle\n",
        "\n",
        "\n",
        "#From the source - ignore \n",
        "get_ipython().magic(u'matplotlib inline')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RqTyGz1IQc8c"
      },
      "source": [
        "You may have to run this for using the MobileNetV2 for 3D network's feature extractor on Google Colab. YOU NEED TO restart the session if running this for the first time."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FukB8g4St8m_"
      },
      "source": [
        "pip install 'h5py==2.10.0' --force-reinstall"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0_RVPrbVBShK"
      },
      "source": [
        "## Download the Dataset\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eoS1Nx_6QZiw"
      },
      "source": [
        "### Download images\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2b4RJ7GA5q3V"
      },
      "source": [
        "!wget https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_image_2.zip\n",
        "!unzip data_object_image_2.zip\n",
        "!ls -l training/image_2 | wc -l  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lT6pIOCuQa7o"
      },
      "source": [
        "### Download labels\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "73XNYjvJvY2y"
      },
      "source": [
        "!wget https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_label_2.zip\n",
        "! unzip data_object_label_2.zip\n",
        "!ls -l training/label_2 | wc -l"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EzgNLXKuQUUO"
      },
      "source": [
        "Delete the zip files to save space on Colab\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "68x3KbS4yj_h"
      },
      "source": [
        "rm -rf data_object_image_2.zip data_object_label_2.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UFWz7BqCC8jx"
      },
      "source": [
        "## Setup Tensorboard"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "doSZ0Y10QuhI"
      },
      "source": [
        "You may want to change the logdir depending on your training setup\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fw3EjqcgxGB-"
      },
      "source": [
        "%load_ext tensorboard\n",
        "%tensorboard --logdir /content/output_eval\n",
        "\n",
        "# If you want to kill tensorboard session:\n",
        "# !kill [PID]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MqolTa3uBou9"
      },
      "source": [
        "## Global Parameters"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cd1ipWwLQ-Hj"
      },
      "source": [
        "### The Global parameters used for the 3D part\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wOBCAQFRBtFA"
      },
      "source": [
        "# This is for the orientation estimation.\n",
        "BIN, OVERLAP = 6, 0.1\n",
        "\n",
        "W = 1.\n",
        "# The crop shapes for the input 3D network.\n",
        "NORM_H, NORM_W = 224, 224\n",
        "\n",
        "# Maximum bounding box jitter for data augmentation while training 3D network\n",
        "MAX_JIT=5\n",
        "\n",
        "# Classes to be considered and extracted from kitti dataset, while training the 3D network\n",
        "VEHICLES = ['Car', 'Truck', 'Van']\n",
        "\n",
        "# These are used to filter out useless and bad objects from dataset (for training)\n",
        "MAX_OCCLUSION =1 # anything <= will be included.\n",
        "MAX_TRUNCATION = 0.8 # anything < will be included.\n",
        "\n",
        "# Path to the images and labels dir.\n",
        "image_dir = 'training/image_2/'\n",
        "label_dir = 'training/label_2/'\n",
        "\n",
        "FEATURE_EXTRACTOR_3D = 'mobilenetv2' #OR vgg16"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oGQr8wVWCK-i"
      },
      "source": [
        "### Utils (compute_anchors, prepare_input_and_output, data_gen l2_normalize, orientation loss)\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LDq-xdUaB7yr"
      },
      "source": [
        "def compute_anchors(angle):\n",
        "    anchors = []\n",
        "    \n",
        "    wedge = 2.*np.pi/BIN  #Length of each bin in Radian.\n",
        "\n",
        "    # Each angle will lie somewhere in a bin. But we need to find the closest. That is, given a bin, we want to know which bound is closer to our angle.\n",
        "    # For instance if our bin is (pi/2, pi), we want to know whether the angle is easier to reference from pi/2 or pi.\n",
        "    # So we keep both lower and upper bound using l_index and r_index, respectively.\n",
        "    l_index = int(angle/wedge)\n",
        "    r_index = l_index + 1\n",
        "    \n",
        "    # Now we check if our angle is more closer to current bin's lower bound, or its upper bound.\n",
        "\n",
        "    # If close enough to the lower bound, consider current index..\n",
        "    if (angle - l_index*wedge) < wedge/2 * (1+OVERLAP/2):\n",
        "        anchors.append([l_index, angle - l_index*wedge])\n",
        "    # If close enough to upper bound,\n",
        "    if (r_index*wedge - angle) < wedge/2 * (1+OVERLAP/2):\n",
        "        anchors.append([r_index%BIN, angle - r_index*wedge])\n",
        "        \n",
        "    return anchors\n",
        "\n",
        "def prepare_input_and_output(train_inst):\n",
        "\n",
        "    # Read the image.\n",
        "    img = cv2.imread(image_dir + train_inst['image'])\n",
        "\n",
        "    # Read AND jitter each bounding box\n",
        "    xmin = train_inst['xmin'] + np.random.randint(-MAX_JIT, MAX_JIT+1)\n",
        "    ymin = train_inst['ymin'] + np.random.randint(-MAX_JIT, MAX_JIT+1)\n",
        "    xmax = train_inst['xmax'] + np.random.randint(-MAX_JIT, MAX_JIT+1)\n",
        "    ymax = train_inst['ymax'] + np.random.randint(-MAX_JIT, MAX_JIT+1)\n",
        "\n",
        "    # Ensure that your coordinates are withing ranges after jittering.\n",
        "    shape = img.shape\n",
        "    xmin = max(min(xmin, shape[1]-1),0)\n",
        "    xmax = max(min(xmax, shape[1]-1),0)\n",
        "    ymin = max(min(ymin, shape[0]-1),0)\n",
        "    ymax = max(min(ymax, shape[0]-1),0)\n",
        "    \n",
        "    # Crop the frame.\n",
        "    img = copy.deepcopy(img[ymin:ymax+1,xmin:xmax+1]).astype(np.float32)\n",
        "\n",
        "    # Randomly decide whether to flip or not.\n",
        "    flip = np.random.binomial(1, .5)\n",
        "    if flip > 0.5: img = cv2.flip(img, 1)\n",
        "        \n",
        "    # resize the patch to standard size\n",
        "    img = cv2.resize(img, (NORM_H, NORM_W))\n",
        "\n",
        "    # Make pixel values [-0.5 to 0.5].\n",
        "    img = img / 255.0 - 0.5\n",
        "    \n",
        "    #Return data but take care of returining proper orientation (will change if image is flipped).\n",
        "    if flip > 0.5:\n",
        "        return img, train_inst['dims'], train_inst['orient_flipped'], train_inst['conf_flipped']\n",
        "    else:\n",
        "        return img, train_inst['dims'], train_inst['orient'], train_inst['conf']\n",
        "\n",
        "def data_gen(all_objs, batch_size):\n",
        "\n",
        "    # Total objects (patches) that we have\n",
        "    num_obj = len(all_objs)\n",
        "    \n",
        "    keys = range(num_obj)\n",
        "    np.random.shuffle(list(keys))\n",
        "    \n",
        "    # For each batch, we will have indices of [ l_bound ...... r_bound ) (not including r_bound itself).\n",
        "    # Usually r_bound - l_bound  should be equal to batch size.\n",
        "    l_bound = 0\n",
        "    r_bound = batch_size if batch_size < num_obj else num_obj\n",
        "    \n",
        "    while True:\n",
        "        if l_bound == r_bound:\n",
        "            l_bound  = 0\n",
        "            r_bound = batch_size if batch_size < num_obj else num_obj\n",
        "            np.random.shuffle(list(keys))\n",
        "        \n",
        "        currt_inst = 0\n",
        "        x_batch = np.zeros((r_bound - l_bound, 224, 224, 3)) # Image batch\n",
        "        d_batch = np.zeros((r_bound - l_bound, 3)) # Dimension batch\n",
        "        o_batch = np.zeros((r_bound - l_bound, BIN, 2)) # orientation batch\n",
        "        c_batch = np.zeros((r_bound - l_bound, BIN)) # confidences batch\n",
        "        \n",
        "        # Iterate the batch\n",
        "        for key in keys[l_bound:r_bound]:\n",
        "            # Prepare the data for the current frame.\n",
        "            image, dimension, orientation, confidence = prepare_input_and_output(all_objs[key])\n",
        "            \n",
        "            x_batch[currt_inst, :] = image\n",
        "            d_batch[currt_inst, :] = dimension\n",
        "            o_batch[currt_inst, :] = orientation\n",
        "            c_batch[currt_inst, :] = confidence\n",
        "            \n",
        "            currt_inst += 1\n",
        "                \n",
        "        # Yield the prepared batch. \n",
        "        yield x_batch, [d_batch, o_batch, c_batch]\n",
        "\n",
        "        # Go for the next batch\n",
        "        l_bound  = r_bound\n",
        "        r_bound = r_bound + batch_size\n",
        "\n",
        "        # Limit the r_bount to max valid index.\n",
        "        if r_bound > num_obj: r_bound = num_obj \n",
        "\n",
        "def l2_normalize(x):\n",
        "    # Compute the second norm for each (sin, cos) pare and normalize the values.\n",
        "    # So (sin, cos) will be normalized into ( sin/sqrt(sin^2+cos^2) , cos/sqrt(sin^2+cos^2)).\n",
        "    # Thus if the network gives (a,b), we are always sure that a^2 + b^2 = 1 and we can use arctan with no worries.\n",
        "    return tf.nn.l2_normalize(x, axis=2)\n",
        "\n",
        "def orientation_loss(y_true, y_pred):\n",
        "    # Here we have two 3D arrays with shape (batch_size, bin count, 2). the 2 is for the (sin, cos) vector.\n",
        "    # The loss value, however, should be a scalar.\n",
        "    \n",
        "    # Make (sin, cos) into (sin^2, cos^2) and then sum them up into 1 scalar.\n",
        "    # So the shape is now (batch_size, bin_count)\n",
        "    anchors = tf.reduce_sum(tf.square(y_true), axis=2)\n",
        "\n",
        "    # Not sure about this line. I believe it assigns true for every bin that\n",
        "    # has enough overlap with the true angle. Because, in the \"Process data\" cell,\n",
        "    # we inserted (0,0) for other bins and only the one or two bins with enough overlap received the (sin, cos)\n",
        "    anchors = tf.greater(anchors, tf.constant(0.5))\n",
        "\n",
        "    # Now for each row, sum the values. So the shape is now (batch size).\n",
        "    anchors = tf.reduce_sum(tf.cast(anchors, tf.float32), 1)\n",
        "    \n",
        "\n",
        "    # We use cosine similarity for the loss. we compute  cos(alpha)= a.b / |a||b|. \n",
        "    # alpha is the angle between ground truth vector and the estimation.\n",
        "    # The ideal value would be cos(0) = 1. But gradient decent tries to MINIMIZE the loss.\n",
        "    # So we add a - behind it. Now the ideal loss value is -1 and the network gets trained in the right direction.\n",
        "    loss = -(y_true[:,:,0]*y_pred[:,:,0] + y_true[:,:,1]*y_pred[:,:,1]) \n",
        "\n",
        "    # For each batch, sum all loss values for all bins. So the shape becomes (batch size)\n",
        "    loss = tf.reduce_sum(loss, axis=1)\n",
        "\n",
        "    # Now normalize the loss. If I've got it right, the anchors are an array of (bin count, bin count, ...)\n",
        "    # So now the loss becomes (loss0/relevant_bin_count, loss1/relevant_bin_count, ...) with shape (batch size).\n",
        "    loss = loss / anchors\n",
        "    \n",
        "    # Use mean to turn the vector loss into scalar.\n",
        "    return tf.reduce_mean(loss)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cqQSPk9mLjX4"
      },
      "source": [
        "## Parse annotation (Function)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LzXgekEMLgoE"
      },
      "source": [
        "def parse_annotation(label_dir, image_dir):\n",
        "    # Here we prepare all objects (patches), their attributes, and the\n",
        "    # average dimension across the dataset.\n",
        "    all_objs = []\n",
        "    \n",
        "    # The average vector and the object count used for averaging.\n",
        "    dims_avg = {key:np.array([0, 0, 0]) for key in VEHICLES}\n",
        "    dims_cnt = {key:0 for key in VEHICLES}\n",
        "    \n",
        "    # Iterate through label files.\n",
        "    for label_file in os.listdir(label_dir):\n",
        "\n",
        "        # If you are using another dataset, take care of this.\n",
        "        image_file = label_file.replace('txt', 'png')\n",
        "\n",
        "        # Iterate through lines in each label file\n",
        "        for line in open(label_dir + label_file).readlines():\n",
        "            line = line.strip().split(' ')\n",
        "            # Each row will have this structure:\n",
        "            # Class Truncated Occluded Theta(local) Xmin Ymin Xmax Ymax Dim Dim Dim T T T \n",
        "            truncated = np.abs(float(line[1]))\n",
        "            occluded  = np.abs(float(line[2]))\n",
        "\n",
        "            # Make sure it is a relevant class and with enough visibility (based on global parameters)\n",
        "            if line[0] in VEHICLES and truncated < MAX_TRUNCATION and occluded <= MAX_OCCLUSION:\n",
        "\n",
        "                # This is pretty confusing here. I will explain it somewhere in the documentation. Sorry :(\n",
        "                new_alpha = -float(line[3]) + 3*np.pi/2\n",
        "                new_alpha = new_alpha - np.floor(new_alpha / (2. * np.pi)) * (2. * np.pi)\n",
        "\n",
        "                obj = {'name':line[0],\n",
        "                       'image':image_file,\n",
        "                       'xmin':int(float(line[4])),\n",
        "                       'ymin':int(float(line[5])),\n",
        "                       'xmax':int(float(line[6])),\n",
        "                       'ymax':int(float(line[7])),\n",
        "                       'dims':np.array([float(number) for number in line[8:11]]),\n",
        "                       'new_alpha': new_alpha, \n",
        "                      # The next 3 are not used for training the network, but for benchmarking translation vector accuracy. \n",
        "                       'translation':np.array([float(number) for number in line[11:14]]),\n",
        "                       'truncated': truncated,\n",
        "                       'occluded': occluded\n",
        "                      }\n",
        "                \n",
        "                # Update the average while reading the new object's dimensions\n",
        "                dims_avg[obj['name']]  = dims_cnt[obj['name']]*dims_avg[obj['name']] + obj['dims']\n",
        "                dims_cnt[obj['name']] += 1\n",
        "                dims_avg[obj['name']] /= dims_cnt[obj['name']]\n",
        "\n",
        "                # Add the object to the list\n",
        "                all_objs.append(obj)\n",
        "                \n",
        "    # return the object list and the average dimensions.\n",
        "    return all_objs, dims_avg"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1YsCUgo8CFem"
      },
      "source": [
        "## Process data\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZZQ2tOm9U5Uk"
      },
      "source": [
        "all_objs, dims_avg = parse_annotation(label_dir, image_dir)\n",
        "\n",
        "print(\"number of objects: {}\".format(len(all_objs)))\n",
        "print(\" ----- \\n A sample object(patch):\\n {} \\n -----\".format(all_objs[0]))\n",
        "print(\"The average dimensions: {}\".format(dims_avg))\n",
        "\n",
        "for obj in all_objs:\n",
        "\n",
        "    # Fix dimensions, compute the residual\n",
        "    obj['dims'] = obj['dims'] - dims_avg[obj['name']]\n",
        "\n",
        "    # Make all residuals 0\n",
        "    orientation = np.zeros((BIN,2))\n",
        "    # Make all bin confidences 0\n",
        "    confidence = np.zeros(BIN)\n",
        "    \n",
        "    #turn angles into -> [bin# , residual]\n",
        "    anchors = compute_anchors(obj['new_alpha'])\n",
        "    \n",
        "    for anchor in anchors:\n",
        "        #compute the cosine and sine of the residual\n",
        "        orientation[anchor[0]] = np.array([np.cos(anchor[1]), np.sin(anchor[1])])\n",
        "\n",
        "        #make the confidence of relative bin# 1\n",
        "        confidence[anchor[0]] = 1.\n",
        "        \n",
        "    #normalize confidences\n",
        "    confidence = confidence / np.sum(confidence)\n",
        "        \n",
        "    #add the computed orientation and confidence to the obj\n",
        "    obj['orient'] = orientation\n",
        "    obj['conf'] = confidence\n",
        "        \n",
        "    # Fix orientation and confidence for flip\n",
        "    orientation = np.zeros((BIN,2))\n",
        "    confidence = np.zeros(BIN)\n",
        "    \n",
        "    anchors = compute_anchors(2.*np.pi - obj['new_alpha'])\n",
        "    \n",
        "    for anchor in anchors:\n",
        "        orientation[anchor[0]] = np.array([np.cos(anchor[1]), np.sin(anchor[1])])\n",
        "        confidence[anchor[0]] = 1\n",
        "        \n",
        "    confidence = confidence / np.sum(confidence)\n",
        "        \n",
        "    obj['orient_flipped'] = orientation\n",
        "    obj['conf_flipped'] = confidence"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "crdtSN0Fu6eH"
      },
      "source": [
        "# **Set up and Train 3D Model**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "heGkEQ-uCb0o"
      },
      "source": [
        "## Build 3D Model(Function and usage)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hu4_ckJAVn4f"
      },
      "source": [
        "# Construct the network\n",
        "def build_model(input_shape=(224, 224, 3), weights=None, freeze=False, feature_extractor='vgg16'):\n",
        "\n",
        "    if feature_extractor == 'mobilenetv2':\n",
        "        feature_extractor_model = MobileNetV2(include_top=False, weights=weights, input_shape=input_shape)\n",
        "    elif feature_extractor == 'vgg16':\n",
        "        feature_extractor_model = VGG16(include_top=False, weights=weights, input_shape=input_shape)\n",
        "    else:\n",
        "        print(\"Requested a non-existing feature extractor model. Either choose from mobilenetv2 and vgg16 or add your own to the code\")\n",
        "        exit(-1)\n",
        "        \n",
        "    if freeze:\n",
        "        for layer in feature_extractor_model.layers:\n",
        "            layer.trainable = False\n",
        "\n",
        "    x = Flatten()(feature_extractor_model.output)\n",
        "\n",
        "    dimension   = Dense(512)(x)\n",
        "    dimension   = LeakyReLU(alpha=0.1)(dimension)\n",
        "    dimension   = Dropout(0.5)(dimension)\n",
        "    dimension   = Dense(3)(dimension)\n",
        "    dimension   = LeakyReLU(alpha=0.1, name='dimension')(dimension)\n",
        "\n",
        "    orientation = Dense(256)(x)\n",
        "    orientation = LeakyReLU(alpha=0.1)(orientation)\n",
        "    orientation = Dropout(0.5)(orientation)\n",
        "    orientation = Dense(BIN*2)(orientation)\n",
        "    orientation = LeakyReLU(alpha=0.1)(orientation)\n",
        "    orientation = Reshape((BIN,-1))(orientation)\n",
        "    orientation = Lambda(l2_normalize, name='orientation')(orientation)\n",
        "\n",
        "    confidence  = Dense(256)(x)\n",
        "    confidence  = LeakyReLU(alpha=0.1)(confidence)\n",
        "    confidence  = Dropout(0.5)(confidence)\n",
        "    confidence  = Dense(BIN, activation='softmax', name='confidence')(confidence)\n",
        "\n",
        "    model = Model(feature_extractor_model.input, outputs=[dimension, orientation, confidence])\n",
        "    model.summary() \n",
        "\n",
        "\n",
        "    #load weights\n",
        "    #model.load_weights('/content/gdrive/MyDrive/weights_new2.hdf5')\n",
        "    return model "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cwuZDuGCCx6D"
      },
      "source": [
        "## Pre-training Setup\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Sls_srHxbFAT"
      },
      "source": [
        "# You can replace 'mobilenet' with 'vgg16' to change the feature extractor implementation in build_model().\n",
        "model = build_model(input_shape = (224, 224, 3), weights = 'imagenet', freeze=False, feature_extractor=FEATURE_EXTRACTOR_3D)\n",
        "\n",
        "# Make sure to change the checkpoint path based on your setup.\n",
        "checkpoint  = ModelCheckpoint('/content/saved_weights_mobilenet_transition.hdf5', monitor='val_loss', verbose=1, save_best_only=True, mode='min', period=1)\n",
        "early_stop  = EarlyStopping(monitor='val_loss', min_delta=0.001, patience=10, mode='min', verbose=1)\n",
        "tensorboard = TensorBoard(log_dir='../logs/', histogram_freq=0, write_graph=True, write_images=False)\n",
        "\n",
        "all_exams  = len(all_objs)\n",
        "trv_split  = int(0.85*all_exams)\n",
        "batch_size = 32\n",
        "\n",
        "# As I understood, Kitti's 2D Object dataset is already shuffeled. So there is no need to shuffle the dataset.\n",
        "# Nevertheless, you can do so with next line: \n",
        "# np.random.shuffle(all_objs)\n",
        "\n",
        "train_gen = data_gen(all_objs[:trv_split], batch_size)\n",
        "valid_gen = data_gen(all_objs[trv_split:all_exams], batch_size)\n",
        "\n",
        "train_num = int(np.ceil(trv_split/batch_size))\n",
        "valid_num = int(np.ceil((all_exams - trv_split)/batch_size))\n",
        "\n",
        "print(\"training configurations:\")\n",
        "print(\"all data:\", all_exams)\n",
        "print(\"training data:\", trv_split)\n",
        "print(\"batch size:\", batch_size)\n",
        "print(\"train_num\", train_num)\n",
        "print(\"valid_num\", valid_num)\n",
        "\n",
        "\n",
        "minimizer  = Adam(lr=1e-5)\n",
        "model.compile(optimizer=minimizer, #minimizer,\n",
        "              loss={'dimension': 'mean_squared_error', 'orientation': orientation_loss, 'confidence': 'categorical_crossentropy'},\n",
        "                  loss_weights={'dimension': 2., 'orientation': 1., 'confidence': 4.}, metrics=['mse','mae'])\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2ciwqcQZDEqy"
      },
      "source": [
        "## Train 3D Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6RbwQVwibH3M"
      },
      "source": [
        "model.fit_generator(generator = train_gen, \n",
        "                    steps_per_epoch = train_num, \n",
        "                    epochs = 150, \n",
        "                    verbose = 1, \n",
        "                    callbacks = [early_stop, checkpoint, tensorboard],\n",
        "                    validation_data = valid_gen, \n",
        "                    validation_steps = valid_num,\n",
        "                    ) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PVB5lHvq2grb"
      },
      "source": [
        "# Test 3D Model's Performance"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fW0-SHrw54E_"
      },
      "source": [
        "### Open a sample image and patch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PdrjuctU67lk"
      },
      "source": [
        "# The testing/ dir, is a completely separate dir that is neither used nor annotated. Just a series of image files.\n",
        "# So for using these samples, you must have a 2D detector at hand. Or you can use the training/ dir, but this dir  \n",
        "# includes your training samples as well. So you can use training dir just for testing the code not the accuracy.\n",
        "test_number = '001037'\n",
        "test_image_path = 'testing/image_2/{test_number}.png'.format(test_number = test_number)\n",
        "calib_path = 'testing/calib/{test_number}.txt'.format(test_number = test_number)\n",
        "\n",
        "# You can also read from train dir (this includes both train and test instances)\n",
        "# test_image_path = 'training/image_2/{test_number}.png'.format(test_number = test_number)\n",
        "# calib_path = 'training/calib/{test_number}.txt'.format(test_number = test_number)\n",
        "# test_label_path = 'training/label_2/{test_number}.txt'.format(test_number = test_number)\n",
        "\n",
        "# Plot the sample\n",
        "test_image = cv2.imread(test_image_path)\n",
        "image_plot = test_image[:, :, ::-1]\n",
        "plt.figure(figsize=(60,60))\n",
        "plt.imshow(image_plot/255.)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "## Get 2D boxes (3 options)"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "BqKtsMQKJEIE"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YVb3ZA_wjTw-"
      },
      "source": [
        "### Option A: Manually select a bounding box\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVRzAf1FjR1q"
      },
      "source": [
        "# If you are want to do a quick test, you can insert bounding box annotations manually.\n",
        "xmin,ymin,xmax,ymax = (175, 551, 209, 610)\n",
        "final_boxes = [[xmin,ymin,xmax,ymax]] #Only one box of course.\n",
        "\n",
        "# Plot the patch\n",
        "patch = image_plot[ymin:ymax, xmin:xmax]\n",
        "plt.figure()\n",
        "plt.imshow(patch/255.)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eOHeGHo6jZOk"
      },
      "source": [
        "### Option B: Get bounding boxes from label file (if using training dir)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3EnVLS9679L"
      },
      "source": [
        "patches = []\n",
        "final_boxes = []\n",
        "for line in open(test_label_path).readlines():\n",
        "    line = line.strip().split(' ')\n",
        "\n",
        "    xmin=int(float(line[4]))\n",
        "    ymin=int(float(line[5]))\n",
        "    xmax=int(float(line[6]))\n",
        "    ymax=int(float(line[7]))\n",
        "\n",
        "    patch = plot_image[ymin:ymax, xmin:xmax]\n",
        "    patch = cv2.resize(patch, (224, 224))\n",
        "    patches.append(patch)\n",
        "    final_boxes.append((xmin, ymin, xmax, ymax))\n",
        "\n",
        "num_patches = len(patches)\n",
        "fig=plt.figure(figsize=(15, 15))\n",
        "\n",
        "for index, patch in enumerate(patches):\n",
        "    fig.add_subplot(1, num_patches, index+1)\n",
        "    plt.imshow(patch)\n",
        "\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "Choose only one of the above if you want."
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "Zgkw3_aeJEIG"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dRCbArcQY7iu"
      },
      "source": [
        "# If you don't want all of the above boxes, choose only one.s\n",
        "index = 0\n",
        "patch = patches[index]\n",
        "final_boxes = [final_boxes[index]]\n",
        "plt.figure()\n",
        "plt.imshow(patch)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "### Option C: Use Tensorflow Object Detection API (Recommended)"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "qOKZ2p0LJEIG"
      }
    },
    {
      "source": [
        "Step 1: Get the pretrained Faster RCNN model"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "i521Zzp3JEIH"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G_2w_idjJEIH"
      },
      "outputs": [],
      "source": [
        "!wget http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_kitti_2018_01_28.tar.gz\n",
        "!tar -xvf faster_rcnn_resnet101_kitti_2018_01_28.tar.gz\n",
        "# Clean up\n",
        "!rm -rf faster_rcnn_resnet101_kitti_2018_01_28.tar.gz"
      ]
    },
    {
      "source": [
        "Step 2: Load the model and detect cars in 2D\n",
        "\n",
        "Keep in mind that this model is trained to classify cars and pedestrians. But we want Cars, Vans, and Trucks.\n",
        "At the end of this notebook, there is a tutorial on re-training the model (or any other model) on these classes.\n",
        "But for now, we assume that detecting cars is sufficient."
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "BI9iV86mJEIH"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IKCN0kZIJEIH"
      },
      "outputs": [],
      "source": [
        "def load_graph(frozen_graph_filename):\n",
        "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
        "        graph_def = tf.GraphDef()\n",
        "        graph_def.ParseFromString(f.read())\n",
        "\n",
        "    with tf.Graph().as_default() as graph:\n",
        "        tf.import_graph_def(graph_def, name=\"2D\")\n",
        "\n",
        "    return graph\n",
        "\n",
        "# Make sure to change the path based on your setup\n",
        "graph = load_graph('/content/faster_rcnn_resnet101_kitti_2018_01_28/frozen_inference_graph.pb')\n",
        "\n",
        "# Input tensor\n",
        "image_tensor = graph.get_tensor_by_name('2D/image_tensor:0')\n",
        "\n",
        "# Output tensors\n",
        "scores_tensor = graph.get_tensor_by_name('2D/detection_scores:0')\n",
        "boxes_tensor = graph.get_tensor_by_name('2D/detection_boxes:0')\n",
        "classes_tensor = graph.get_tensor_by_name('2D/detection_classes:0')\n",
        "num_tensor = graph.get_tensor_by_name('2D/num_detections:0')\n",
        "\n",
        "# We launch a Session\n",
        "image_width = test_image.shape[1]\n",
        "image_height = test_image.shape[0]\n",
        "with tf.Session(graph=graph) as sess:\n",
        "    (boxes, scores, classes, num) = sess.run(\n",
        "        [boxes_tensor, scores_tensor, classes_tensor, num_tensor],\n",
        "        feed_dict={image_tensor: [test_image]})\n",
        "\n",
        "show= test_image.copy()\n",
        "final_boxes=[]\n",
        "for index in range(int(num[0])):\n",
        "  # Apply confidence threshold\n",
        "  if (scores[0][index] < 0.7): \n",
        "      continue\n",
        "  box = boxes[0][index]\n",
        "  real_box = (int(box[0]*image_height), int(box[1]*image_width), int(box[2]*image_height), int(box[3]*image_width))\n",
        "  print(real_box, scores[0][index])\n",
        "  show = cv2.rectangle(show, (real_box[1], real_box[0]), (real_box[3], real_box[2]), color=(0,0,255), thickness=2)\n",
        "  final_boxes.append(real_box)\n",
        "\n",
        "show_plot = show[:, :, ::-1]\n",
        "plt.figure(figsize=(60,60))\n",
        "plt.imshow(show_plot/255.)\n",
        "plt.show()"
      ]
    },
    {
      "source": [
        "# Make sure to change the checkpoint path based on your setup.\n",
        "model.load_weights('/content/gdrive/MyDrive/saves/saved_weights_mobilenet_new.hdf5')"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "30j5t4fWJEIJ"
      }
    },
    {
      "source": [
        "### Download Calibration Files"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "CXA5TVlVJEIJ"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xi696teIJEIJ"
      },
      "outputs": [],
      "source": [
        "!wget https://s3.eu-central-1.amazonaws.com/avg-kitti/data_object_calib.zip\n",
        "!unzip data_object_calib.zip"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "txLuyGG280hd"
      },
      "source": [
        "### Read Projection Matrix"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q50qUL35BGT4"
      },
      "source": [
        "with open(calib_path, \"r\") as f:\n",
        "    c2c_file = f.readlines()\n",
        "\n",
        "    for line in c2c_file:\n",
        "          (key, val) = line.split(':', 1)\n",
        "          if key == ('P2'):\n",
        "                P_ = np.fromstring(val, sep=' ', dtype = 'float')\n",
        "                P_ = P_.reshape(3, 4)\n",
        "                \n",
        "                calib_mat = P_\n",
        "                break\n",
        "print(\"Projection mat:\")\n",
        "print(calib_mat)\n",
        "\n",
        "# #Just for quick tests\n",
        "# calib_mat= [[7.215377e+02, 0.000000e+00, 6.095593e+02, 4.485728e+01],\n",
        "#             [0.000000e+00, 7.215377e+02, 1.728540e+02, 2.163791e-01],\n",
        "#             [0.000000e+00, 0.000000e+00, 1.000000e+00, 2.745884e-03]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UAdjDSygDNwq"
      },
      "source": [
        "### Define and Prepare Projection utils"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FypLHQZCDNQa"
      },
      "source": [
        "def init_points3D(dims):\n",
        "    points3D = np.zeros((8, 3))\n",
        "    cnt = 0\n",
        "    for i in [1, -1]:\n",
        "        for j in [1, -1]:\n",
        "            for k in [1, -1]:\n",
        "                points3D[cnt] = dims[[1, 0, 2]].T / 2.0 * [i, k, j * i]\n",
        "                cnt += 1\n",
        "    return points3D\n",
        "\n",
        "def gen_3D_box(yaw,dims,cam_to_img,box_2D):\n",
        "    \n",
        "    dims = dims.reshape((-1,1))\n",
        "    box_2D = box_2D.reshape((-1,1))\n",
        "    points3D = init_points3D(dims)\n",
        "\n",
        "    # Here the rotation is done around the Y axis. Just a convention in the code.\n",
        "    rot_M = np.asarray([[np.cos(yaw), 0, np.sin(yaw)], [0, 1, 0], [-np.sin(yaw), 0, np.cos(yaw)]]) \n",
        "    center = compute_center(points3D, rot_M, cam_to_img, box_2D, inds)\n",
        "\n",
        "    points2D = points3D_to_2D(points3D, center, rot_M, cam_to_img)\n",
        "\n",
        "    return points2D\n",
        "\n",
        "def compute_center(points3D,rot_M,cam_to_img,box_2D,inds):\n",
        "    \n",
        "    fx = cam_to_img[0][0]\n",
        "    fy = cam_to_img[1][1]\n",
        "    u0 = cam_to_img[0][2]\n",
        "    v0 = cam_to_img[1][2]\n",
        "    \n",
        "    W = np.array([[fx, 0, u0 - box_2D[0]],\n",
        "                  [fx, 0, u0 - box_2D[2]],\n",
        "                  [0, fy, v0 - box_2D[1]],\n",
        "                  [0, fy, v0 - box_2D[3]]], dtype = 'float')\n",
        "    center =None\n",
        "    error_min = 1e10\n",
        "\n",
        "    for ind in inds:\n",
        "        y = np.zeros((4, 1))\n",
        "        for i in range(len(ind)):\n",
        "            \n",
        "            RP = np.dot(rot_M, (points3D[ind[i]]).reshape((-1, 1)))\n",
        "            y[i] = box_2D[i] * cam_to_img[2, 3] - np.dot(W[i], RP) - cam_to_img[i // 2, 3]\n",
        "            \n",
        "        result = solve_least_squre(W, y)\n",
        "        error = compute_error(points3D, result, rot_M, cam_to_img, box_2D)\n",
        "        \n",
        "        if error < error_min and result[2,0]>0:\n",
        "            center = result\n",
        "            error_min = error\n",
        "            \n",
        "    return center\n",
        "\n",
        "def draw_3D_box(image,points):\n",
        "    points = points.astype(np.int)\n",
        "\n",
        "    for i in range(4):\n",
        "        point_1_ = points[2 * i]\n",
        "        point_2_ = points[2 * i + 1]\n",
        "        cv2.line(image, (point_1_[0], point_1_[1]), (point_2_[0], point_2_[1]), (0, 255, 0), 1)\n",
        "\n",
        "    # The red X at the front.\n",
        "    cv2.line(image,tuple(points[0]),tuple(points[7]),(0, 0, 255), 2)\n",
        "    cv2.line(image, tuple(points[1]), tuple(points[6]), (0, 0, 255), 2)\n",
        "\n",
        "    for i in range(8):\n",
        "        point_1_ = points[i]\n",
        "        point_2_ = points[(i + 2) % 8]\n",
        "        cv2.line(image, (point_1_[0], point_1_[1]), (point_2_[0], point_2_[1]), (0, 255, 0), 1)\n",
        "\n",
        "    return image;\n",
        "\n",
        "def solve_least_squre(W,y):\n",
        "    U, Sigma, VT = np.linalg.svd(W)\n",
        "    result = np.dot(np.dot(np.dot(VT.T, np.linalg.pinv(np.eye(4, 3) * Sigma)), U.T), y)\n",
        "    return result\n",
        "\n",
        "def points3D_to_2D(points3D,center,rot_M,cam_to_img):\n",
        "    \n",
        "    # General formula is: [2D] = K[R T][3D]\n",
        "    # So for each 3D point, apply rotation, add translation (3D center)\n",
        "    # and multiply K. At last, you will have a 3x1 vector, which you should normalize\n",
        "    # by the third element (Homogenous coordinates).\n",
        "    points2D = []\n",
        "    for point3D in points3D:\n",
        "        point3D = point3D.reshape((-1,1))\n",
        "        point = center + np.dot(rot_M, point3D)\n",
        "        point = np.append(point, 1)\n",
        "        point = np.dot(cam_to_img, point)\n",
        "        point2D = point[:2] / point[2]\n",
        "        points2D.append(point2D)\n",
        "    points2D = np.asarray(points2D)\n",
        "\n",
        "    return points2D\n",
        "\n",
        "def compute_error(points3D,center,rot_M, cam_to_img,box_2D):\n",
        "    # Get all of 8 corners from 3D box projected on image.\n",
        "    points2D = points3D_to_2D(points3D, center, rot_M, cam_to_img)\n",
        "\n",
        "    # Get a new bounding box from the 8 projected coreners.\n",
        "    new_box_2D = np.asarray([np.min(points2D[:,0]),\n",
        "                             np.max(points2D[:,0]),\n",
        "                             np.min(points2D[:,1]),\n",
        "                             np.max(points2D[:,1])]).reshape((-1,1))\n",
        "\n",
        "    # Sum the absolute difference of xmin, xmax, ymin, ymax for the 2D bbox,\n",
        "    # and the new bbox from 8 projections.\n",
        "    error = np.sum(np.abs(new_box_2D - box_2D))\n",
        "    return error\n",
        "\n",
        "# These will be used in the next cell\n",
        "inds = []\n",
        "indx = [1, 3, 5, 7]\n",
        "indy = [0, 1, 2, 3]\n",
        "for i in indx:\n",
        "    for j in indx:\n",
        "        for m in indy:\n",
        "            for n in indy:\n",
        "                inds.append([i, j, m, n])\n",
        "\n",
        "# Dimension averages collected from the dataset\n",
        "dims_avg = {'Car' : [1.52130159, 1.64441129, 3.85729945],\n",
        "            'Truck': [ 3.07044968,  2.62877944, 11.17126338],\n",
        "            'Van': [2.18560847, 1.91077601, 5.08042328],\n",
        "            'Tram': [3.56005102,  2.4002551,  18.52173469] }"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fRTUFMvzQV3i"
      },
      "source": [
        "## 2D to 3D\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fqqkqna6QXwk"
      },
      "source": [
        "final_image = test_image.copy()\n",
        "for index, box in enumerate(final_boxes):\n",
        "    ymin, xmin, ymax, xmax = box\n",
        "    # Crop\n",
        "    patch = test_image[ymin:ymax, xmin:xmax]\n",
        "    # Resize\n",
        "    patch = cv2.resize(patch, (224, 224))\n",
        "    # Set the input pixels to be within (-0.5 , 0.5).\n",
        "    patch = patch / 255.0 - 0.5\n",
        "\n",
        "    patch = np.expand_dims(patch, axis = 0)\n",
        "    prediction = model.predict(patch)\n",
        "\n",
        "    # Get the (cos, sin) of the bin with highest probabilitys\n",
        "    max_anc = np.argmax(prediction[2][0])\n",
        "    anchors = prediction[1][0][max_anc]\n",
        "\n",
        "    # anchors=(cos, sin)\n",
        "    if anchors[1] > 0:\n",
        "        angle_offset = np.arccos(anchors[0])\n",
        "    else:\n",
        "        angle_offset = -np.arccos(anchors[0])\n",
        "\n",
        "    wedge = 2.*np.pi/BIN\n",
        "    theta_loc = angle_offset + max_anc*wedge\n",
        "\n",
        "    fx = calib_mat[0][0]\n",
        "    u0 = calib_mat[0][2]\n",
        "    v0 = calib_mat[1][2]\n",
        "\n",
        "    # As suggested in the original paper, we estimate the raye to 2D box center instead of\n",
        "    # object's 3D center. Of course, it is not entirely accurate, but it is sufficient.\n",
        "    box2d_center_x= (xmin + xmax) / 2.0\n",
        "    theta_ray = np.arctan(fx /(box2d_center_x - u0))\n",
        "    \n",
        "    # Arctan's output is (-pi/2, pi/2). But theta_ray should be (0, pi).\n",
        "    # From 0 to pi/2 the values are ok but if the object's center is on the left half,\n",
        "    # the arctan will result in a negative theta_ray. This can be fixed by adding pi.\n",
        "    if theta_ray<0:\n",
        "          theta_ray = theta_ray+np.pi\n",
        "\n",
        "    # Final theta\n",
        "    theta = theta_loc + theta_ray\n",
        "    yaw = np.pi/2 - theta\n",
        "                \n",
        "    # Here we use average dims of cars. If you use a trained model for 2D detection with \n",
        "    # correct classes, you can simply replace this line. The training steps of 2D network is \n",
        "    # provided at the end of the notebook.\n",
        "    dims = dims_avg['Car'] + prediction[0][0]\n",
        "\n",
        "    box_2D = np.asarray([box[1], box[0], box[3], box[2]], dtype = np.float)\n",
        "    points2D = gen_3D_box(yaw, dims, calib_mat, box_2D) #switched yaw -> theta    \n",
        "    final_image = draw_3D_box(final_image, points2D)\n",
        "\n",
        "final_image_show = final_image[:, :, ::-1]\n",
        "plt.figure(figsize=(20,20))\n",
        "plt.imshow(final_image_show/255.)\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "# Test Translation Vector Accuracy\n",
        "\n",
        "Since this code is independent from testing 3D model's accuracy, I have redifned many functions so it remains independent."
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "4nJAwem8tg0y"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WFOFxI2Mafn"
      },
      "source": [
        "inds = []\n",
        "indx = [1, 3, 5, 7]\n",
        "indy = [0, 1, 2, 3]\n",
        "for i in indx:\n",
        "      for j in indx:\n",
        "      for m in indy:\n",
        "            for n in indy:\n",
        "                  inds.append([i, j, m, n])\n",
        "\n",
        "def get_t(yaw,dims,cam_to_img,box_2D, ind):\n",
        "    dims = dims.reshape((-1,1))\n",
        "    box_2D = box_2D.reshape((-1,1))\n",
        "    points3D = init_points3D(dims)\n",
        "    rot_M = np.asarray([[np.cos(yaw), 0, np.sin(yaw)], [0, 1, 0], [-np.sin(yaw), 0, np.cos(yaw)]]) \n",
        "    center = compute_center(points3D, rot_M, cam_to_img, box_2D, inds)\n",
        "    return center\n",
        "\n",
        "def compare_t(estimated_t, true_t):\n",
        "  estimated = np.asarray([float(estimated_t[0]), float(estimated_t[1]), float(estimated_t[2])])\n",
        "  dist = np.linalg.norm(true_t - estimated)\n",
        "  normalized_dist = np.linalg.norm(true_t - estimated) / np.linalg.norm(true_t)\n",
        "  return dist, normalized_dist\n",
        "\n",
        "avg_t_loss=0\n",
        "avg_nt_loss=0\n",
        "cnt = 0\n",
        "for obj in all_objs[trv_split:all_exams]:\n",
        "    truncated = obj['truncated']\n",
        "    occluded = obj['occluded']\n",
        "    height = int(obj['ymax'] - obj['ymin'])\n",
        "    width = int(obj['xmax'] - obj['xmin'])\n",
        "    # Filter based on occlusion and trunction.\n",
        "    # Many objects' centers are not even in front of the camera\n",
        "    if (truncated > 0.5 or occluded> 1 or height <=25 or width <=25):\n",
        "      continue\n",
        "\n",
        "    with open('/content/training/calib/'+obj['image'].split('.')[0]+'.txt', \"r\") as f:\n",
        "      c2c_file = f.readlines()\n",
        "\n",
        "    for line in c2c_file:\n",
        "          (key, val) = line.split(':', 1)\n",
        "          if key == ('P2'):\n",
        "                P_ = np.fromstring(val, sep=' ', dtype = 'float')\n",
        "                P_ = P_.reshape(3, 4)\n",
        "                calib_mat = P_\n",
        "                break\n",
        "\n",
        "    xmin = int(obj['xmin'])\n",
        "    ymin = int(obj['ymin'])\n",
        "    xmax = int(obj['xmax'])\n",
        "    ymax = int(obj['ymax'])\n",
        "\n",
        "    test_image = cv2.imread('/content/training/image_2/'+obj['image'])\n",
        "    patch = test_image[ymin:ymax, xmin:xmax]\n",
        "    patch = cv2.resize(patch, (224, 224))\n",
        "    patch = patch / 255.0 - 0.5\n",
        "\n",
        "    patch = np.expand_dims(patch, axis = 0)\n",
        "\n",
        "    prediction = model.predict(patch)\n",
        "    max_anc = np.argmax(prediction[2][0])\n",
        "    anchors = prediction[1][0][max_anc]\n",
        "\n",
        "    if anchors[1] > 0:\n",
        "        angle_offset = np.arccos(anchors[0])\n",
        "    else:\n",
        "        angle_offset = -np.arccos(anchors[0])\n",
        "        \n",
        "    wedge = 2.*np.pi/BIN\n",
        "    theta_loc = angle_offset + max_anc*wedge\n",
        "\n",
        "    fx = calib_mat[0][0]\n",
        "    u0 = calib_mat[0][2]\n",
        "    v0 = calib_mat[1][2]\n",
        "\n",
        "    box2d_center_x= (xmin + xmax) / 2.0\n",
        "    \n",
        "    theta_ray = np.arctan(fx /(box2d_center_x - u0))\n",
        "    if theta_ray<0:\n",
        "          theta_ray = theta_ray+np.pi\n",
        "\n",
        "    # Theta Yaw\n",
        "    theta = theta_loc + theta_ray\n",
        "    yaw = np.pi/2 - theta\n",
        "\n",
        "    #DIMENSION\n",
        "    dims_avg = {'Car' : [1.52130159, 1.64441129, 3.85729945],\n",
        "                'Truck': [ 3.07044968,  2.62877944, 11.17126338],\n",
        "                'Van': [2.18560847, 1.91077601, 5.08042328],\n",
        "                'Tram': [3.56005102,  2.4002551,  18.52173469] }\n",
        "    dims = dims_avg[obj['name']] + prediction[0][0]\n",
        "\n",
        "\n",
        "\n",
        "    box_2D = np.asarray([xmin, ymin, xmax, ymax], dtype = np.float)\n",
        "    points2D = gen_3D_box(yaw, dims, calib_mat, box_2D) #switched yaw -> theta    \n",
        "    final_image = draw_3D_box(test_image, points2D)\n",
        "\n",
        "  \n",
        "    t = get_t(yaw, dims, calib_mat, box_2D, inds) #switched yaw -> theta    \n",
        "    t_loss, nt_loss = compare_t(t, obj['translation'])\n",
        "    cnt += 1\n",
        "    avg_t_loss = ( (avg_t_loss * (cnt-1)) + t_loss )/cnt\n",
        "    avg_nt_loss = ( (avg_nt_loss * (cnt-1)) + nt_loss )/cnt\n",
        "    \n",
        "print(\"Average loss is: \", avg_t_loss)\n",
        "print(\"Average normalized loss is: \", avg_nt_loss)\n",
        "print(\"Count: \", cnt)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1WZi86qgysUL"
      },
      "source": [
        "# Train a 2D Network Yourself"
      ]
    },
    {
      "source": [
        "### Setup the environment"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "y6zfylJeJEIO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aZSSBB08JEIP"
      },
      "outputs": [],
      "source": [
        "!pip install protobuf protobuf-compiler\n",
        "\n",
        "# Clone the repo\n",
        "!git clone https://github.com/tensorflow/models.git\n",
        "\n",
        "# Compile protos.\n",
        "%cd models/research\n",
        "!protoc object_detection/protos/*.proto --python_out=.\n",
        "\n",
        "# Install TensorFlow Object Detection API.\n",
        "%cp object_detection/packages/tf1/setup.py .\n",
        "!python -m pip install .\n",
        "\n",
        "# Test the installation\n",
        "!python object_detection/builders/model_builder_tf1_test.py\n",
        "%cd ../.."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PB2DiufeJvyd"
      },
      "source": [
        "### Get FRCNN_Resnet_101"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zyRzOBr4JvF7"
      },
      "source": [
        "%cd /content\n",
        "!wget http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_kitti_2018_01_28.tar.gz\n",
        "!tar -xvf faster_rcnn_resnet101_kitti_2018_01_28.tar.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i7_6QSK1JrTn"
      },
      "source": [
        "### Get Mobilenet_V2+SSD\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oerbiRMOJq8w"
      },
      "source": [
        "%cd /content\n",
        "!wget http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v2_coco_2018_03_29.tar.gz\n",
        "!tar -xvf ssd_mobilenet_v2_coco_2018_03_29.tar.gz"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kpE9ie499X4a"
      },
      "source": [
        "### Generate TF record"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tA2uzl7D-vLm"
      },
      "source": [
        "# In case you are running this cell again:\n",
        "%cd /content\n",
        "!rm -rf data\n",
        "!rm -rf ouptut\n",
        "!rm -rf data_object_image_2\n",
        "\n",
        "\n",
        "%mkdir data_object_image_2\n",
        "%cd data_object_image_2\n",
        "%mkdir training\n",
        "%cd ..\n",
        "%cp -r training/image_2 data_object_image_2/training/\n",
        "%mkdir data\n",
        "!mkdir output\n",
        "\n",
        "# Create the labelmap\n",
        "!touch data/kitti_label_map.pbtxt\n",
        "!printf 'item {\\n id: 1\\n name: \"car\"\\n}\\nitem {\\n id: 2\\n name: \"pedestrian\"\\n}\\nitem {\\n id: 3\\n name: \"van\"\\n}\\nitem {\\n id: 4\\n name: \"truck\"\\n}' >> data/kitti_label_map.pbtxt\n",
        "print(\"MAKE SURE TO DOUBLE CHECK THE LABELMAP FILE' \\n S\" )\n",
        "\n",
        "# Do not bother with this line\n",
        "!rm -rf /content/data_object_image_2/training/image_2/.ipynb_checkpoints"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uAuRhA6I9aPg"
      },
      "source": [
        "# Generate TFRecord\n",
        "!python /content/models/research/object_detection/dataset_tools/create_kitti_tf_record.py \\\n",
        "--data_dir=/content \\\n",
        "--output_path=/content/output/kitti.record \\\n",
        "--classes_to_use=car,pedestrian,van,truck \\\n",
        "--validation_set_side=500"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "As you may suspect, we do not really need the Pedestrian class. However, in my experience, removing the pedestrain class from TFRecords and LabelMap will result in a model that detects nothing! I am not sure why, but I guess this has something to do with the transfer learning and weight re-initialization part. Again, not sure! If you have any idea on this, do send me a PR or open an issue."
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "px-GPOI3JEIR"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v52wNTlqQct3"
      },
      "source": [
        "## Train the 2D model"
      ]
    },
    {
      "source": [
        "### Copy the **train.py** and **eval.py** to the main object_detection dir"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "7va7dE4zJEIR"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J2eQ2Yx9JEIR"
      },
      "outputs": [],
      "source": [
        "%cp models/research/object_detection/legacy/train.py models/research/object_detection\n",
        "%cp /content/models/research/object_detection/legacy/eval.py /content/models/research/object_detection"
      ]
    },
    {
      "source": [
        "### Train the model\n",
        "\n",
        "A default **pipeline.config** is provided with the checkpoints you download from modelzoo and they can work with small modifications. But, due to the lack of documentation and out-dated configs, you may come across errors. Therefore, I suggest using the pipelines that I have provided in the repo. They are not the best config but at least they work. Make sure to do your best on optimizing the configuration. I would appreciate it if you share your config changes with PR or and issue.\n",
        "\n",
        "The next cell checks the output_train and starts with its latest checkpoint, and creates the first if there is none. So if you can stop the cell and run it again."
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "WRlJtK8gJEIS"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1sSJcU_FGS6T"
      },
      "source": [
        "%cd /content\n",
        "\n",
        "# To clean the entire training story and re-train the model:\n",
        "# !rm -rf /content/output_train\n",
        "\n",
        "# Make sure to fix the paths according to your setup.\n",
        "!python models/research/object_detection/train.py \\\n",
        "--logdir=/content/log \\\n",
        "--train_dir=/content/output_train \\\n",
        "--pipeline_config_path=/content/ssd_mobilenet_v2_coco_2018_03_29/pipeline.config"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "### Evaluate the model"
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "27np6U6gJEIS"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gd2rv2AE9Ia3"
      },
      "source": [
        "%rm -rf /content/output_eval\n",
        "%cd /content/models/research\n",
        "\n",
        "# Make sure to fix the paths according to your setup.\n",
        "!python object_detection/eval.py \\\n",
        "        --logtostderr \\\n",
        "        --checkpoint_dir=/content/output_train \\\n",
        "        --eval_dir=/content/output_eval \\\n",
        "        --pipeline_config_path=/content/output_train/pipeline.config"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "source": [
        "### Export the frozen graph\n",
        "\n",
        "With the frozen graph you can easily load the model to get the 2D boxes. The graph can be loaded in the same way I loaded the pre-trained frcnn graph. Only the class numbers can change based on your labelmap."
      ],
      "cell_type": "markdown",
      "metadata": {
        "id": "LtpR-RzqJEIT"
      }
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xBPrq6qsAlh6"
      },
      "source": [
        "%cd /content/models/research/\n",
        "# Make sure to fix the paths according to your setup.\n",
        "!python object_detection/export_inference_graph.py \\\n",
        "--input_type=image_tensor \\\n",
        "--pipeline_config_path=/content/gdrive/MyDrive/colab/new/25_May/output_train/pipeline.config \\\n",
        "--trained_checkpoint_prefix=/content/gdrive/MyDrive/colab/new/25_May/output_train/model.ckpt-25000 \\\n",
        "--output_directory=/content/gdrive/MyDrive/colab/new/25_May/Frozen\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}