CarSim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing Jupyter notebook from Classifier.ipynb\n"
     ]
    }
   ],
   "source": [
    "#CLOCKSPEED = 0.75\n",
    "\n",
    "import airsim\n",
    "import numpy as np\n",
    "import cv2\n",
    "import os\n",
    "import datetime\n",
    "from PIL import Image\n",
    "\n",
    "import nbimporter\n",
    "import Classifier\n",
    "import matplotlib.pyplot as plt\n",
    "import timeit\n",
    "import time\n",
    "\n",
    "depth_det = 10\n",
    "depth_cf = 40\n",
    "stacksize = 5\n",
    "\n",
    "detectable = ['No_gesture', 'Swiping_Left', 'Swiping_Right', 'Stop_Sign', 'Pulling_Hand_In'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetCurrentFrame():\n",
    "    responses = client.simGetImages([\n",
    "        airsim.ImageRequest(\"1\", airsim.ImageType.Scene, False, False)])  #scene vision image in uncompressed RGB array\n",
    "\n",
    "    response = responses[0]\n",
    "    img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) # get numpy array\n",
    "    img_rgb = img1d.reshape(response.height, response.width, 3) # reshape array to 3 channel image array H X W X 3\n",
    "    #cv2.imwrite('fr.png',img_rgb)\n",
    "    \n",
    "    return img_rgb #as nparray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_checkpoint(filepath):\n",
    "    checkpoint = torch.load(filepath, map_location='cpu')\n",
    "\n",
    "    my_dict = checkpoint['state_dict']\n",
    "    my_dict_copy = my_dict.copy()\n",
    "\n",
    "    for key in my_dict_copy.keys():\n",
    "        newkey = key[7:]\n",
    "        my_dict[newkey] = my_dict.pop(key)\n",
    "\n",
    "    checkpoint['state_dict'] = my_dict \n",
    "    \n",
    "    model = resnet.resnet10(\n",
    "        num_classes=2,\n",
    "        shortcut_type='B',\n",
    "        cardinality=32,\n",
    "        sample_size=112,\n",
    "        sample_duration=depth_det)\n",
    "\n",
    "    model.load_state_dict(checkpoint['state_dict'])\n",
    "\n",
    "    model.eval()\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getTransforms():\n",
    "    scales = [1]\n",
    "    for i in range(1, 5):\n",
    "        scales.append(scales[-1] * 0.84089641525)\n",
    "        \n",
    "    norm_method = Normalize(get_mean(norm_value), [1, 1, 1])\n",
    "            \n",
    "    spatial_transform = spatial_transforms.Compose([\n",
    "        Scale(112),\n",
    "        CenterCrop(112),\n",
    "        ToTensor(norm_value), norm_method \n",
    "    ])\n",
    "\n",
    "    return spatial_transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TURN THE 37 FRAMES FROM ./TESTVID INTO TENSOR INPUT <1,3,32,112,112>\n",
    "def TransformData(spatial_transform, frameST):\n",
    "    a = []\n",
    "    for frame in frameST:\n",
    "        image = torchvision.transforms.ToPILImage()(frame)\n",
    "        a.append(image)\n",
    "\n",
    "    spatial_transform.randomize_parameters()\n",
    "    input_spat = [spatial_transform(image) for image in a]\n",
    "    \n",
    "    im_dim = input_spat[0].size()[-2:]\n",
    "    clip = torch.cat(input_spat, 0).view((duration, -1) + im_dim).permute(1, 0, 2, 3)\n",
    "    input = clip[None, :, :, :, :] #Add 5th Dimension; batchSize=1\n",
    "    print(input.shape)\n",
    "    return input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CropHuman(frame, show=False):\n",
    "    hog = cv2.HOGDescriptor()\n",
    "    hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())\n",
    "    frame = cv2.resize(frame, (768, 288))\n",
    "    gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY)\n",
    "    boxes, weights = hog.detectMultiScale(frame, winStride=(8,8) )\n",
    "    boxes = np.array([[x, y, x + w, y + h] for (x, y, w, h) in boxes])\n",
    "    for (xA, yA, xB, yB) in boxes:\n",
    "        cv2.rectangle(frame, (xA, yA), (xB, yB),\n",
    "                        (0, 255, 0), 2)\n",
    "\n",
    "    if show:\n",
    "        plt.imshow(frame, cmap = 'gray', interpolation = 'bicubic')\n",
    "        plt.xticks([]), plt.yticks([])  # to hide tick values on X and Y axis\n",
    "        plt.show()\n",
    "    \n",
    "    #return(boxes[0][0],boxes[0][1],boxes[0][2],boxes[0][3])\n",
    "    return(boxes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def human_detector(fr, verbose=False):\n",
    "    cv2.imwrite('hum_det.png',fr)\n",
    "    boxes = CropHuman(fr, verbose)\n",
    "    if boxes.size == 0:\n",
    "        return 0,0,0,0\n",
    "    else:\n",
    "        x = boxes[0][0] * 2\n",
    "        y = boxes[0][1] * 2\n",
    "        xF = boxes[0][2] * 2\n",
    "        yF = boxes[0][3] * 2\n",
    "        return x,y,xF,yF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def CropLastX(vstreamshort,x,y,xF,yF):\n",
    "    newstream = []\n",
    "    for fr in vstreamshort:\n",
    "        cv2.imwrite('OL1.png',fr) ######\n",
    "        fr = fr[y:yF,x:xF]\n",
    "        cv2.imwrite('OL2.png',fr) ##########\n",
    "        height = fr.shape[0]\n",
    "        width = fr.shape[1]\n",
    "        #fr = fr[int(round(height/14)):int(round(height/2.5)),int(round(width/14)):int(round(width*9/10))]\n",
    "        fr = fr[int(round(height/7)):int(round(height/3)),int(round(width/9)):int(round(width*4/5))]\n",
    "        cv2.imwrite('OL3.png',fr) ########\n",
    "        newstream.append(fr)\n",
    "    return newstream"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Detector(frameST):\n",
    "    #model = load_checkpoint('NameOfPTH')\n",
    "    #input_det = TransformData(getTransforms(), frameST)\n",
    "    #model.eval()\n",
    "    #out_det = model(input_det)\n",
    "    \n",
    "    #classes = [0,1] #No Gest, Gest\n",
    "    #_, index = torch.max(out_det, 1)\n",
    "    #percentage = torch.nn.functional.softmax(out_det, dim=1)[0]\n",
    "    #print(classes[index[0]], percentage[index[0]].item())\n",
    "    \n",
    "    #return(classes[index[0]])\n",
    "    return True\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def actuator(gesture):\n",
    "    if gesture == 'No_gesture': #break\n",
    "        #car_straightsmall()\n",
    "        car_break()\n",
    "    elif gesture == 'Swiping_Left': #go right\n",
    "        car_straightsmall()\n",
    "        car_right()\n",
    "    elif gesture == 'Swiping_Right': #go left\n",
    "        car_straightsmall()\n",
    "        car_left()\n",
    "    elif gesture == 'Stop_Sign': #break\n",
    "        car_break()\n",
    "    elif gesture == 'Pulling_Hand_In': #go straight\n",
    "        import random\n",
    "        num = random.randint(0,3)\n",
    "        if num == 3:\n",
    "            car_straightsmall()\n",
    "            car_right()\n",
    "        else:\n",
    "            car_straight()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def car_break():\n",
    "    car_controls.brake = 1\n",
    "    client.setCarControls(car_controls)\n",
    "    print(\"Apply brakes\")\n",
    "    time.sleep(1)   # let car drive a bit\n",
    "    car_controls.brake\n",
    "    client.reset()\n",
    "    client.enableApiControl(False)\n",
    "    print(\"RESET BISH\")\n",
    "    trigger = True\n",
    "    print(\"DONE RESETTING\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def car_right():\n",
    "    car_controls.throttle = 1\n",
    "    car_controls.steering = 0.98\n",
    "    client.setCarControls(car_controls)\n",
    "    print(\"Go Forward, steer right\")\n",
    "    time.sleep(3.12)   # let car drive a bit\n",
    "    car_straight()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def car_left():\n",
    "    car_controls.throttle = 1.0\n",
    "    car_controls.steering = -0.7\n",
    "    client.setCarControls(car_controls)\n",
    "    print(\"Go Forward, steer left\")\n",
    "    time.sleep(3.82)   # let car drive a bit\n",
    "    car_straight()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def car_straight():\n",
    "    car_controls.throttle = 1.0\n",
    "    car_controls.steering = 0\n",
    "    client.setCarControls(car_controls)\n",
    "    print(\"Go Forward\")\n",
    "    time.sleep(6)   # let car drive a bit\n",
    "    car_break()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def car_straightsmall():\n",
    "    car_controls.throttle = 1.0\n",
    "    car_controls.steering = 0\n",
    "    client.setCarControls(car_controls)\n",
    "    print(\"Go Forward\")\n",
    "    time.sleep(3.2)   # let car drive a bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def DetectAndDrive():\n",
    "    global client, car_controls\n",
    "    client = airsim.CarClient()\n",
    "    client.confirmConnection()\n",
    "    client.enableApiControl(True)\n",
    "    car_controls = airsim.CarControls()\n",
    "    \n",
    "    videostream = []\n",
    "    \n",
    "    while len(videostream) < depth_cf:\n",
    "        try:\n",
    "            videostream.append(GetCurrentFrame())\n",
    "        except ValueError:\n",
    "            continue #skip to next frame\n",
    "        if len(videostream) >= depth_cf:\n",
    "            if len(videostream) % stacksize == 0: #every X frames run human_detector\n",
    "                print(\"Got enough frames. Analyzing\")\n",
    "                x,y,xF,yF = human_detector(videostream[len(videostream)-1], False)\n",
    "                if x or y or xF or yF: #Only continue if human detected\n",
    "                    croppedstream_recent = CropLastX(videostream[-depth_cf:],x,y,xF,yF) #Crop last X frames of VS\n",
    "                    exists = Detector(croppedstream_recent[-depth_det:]) #See if gesture detected\n",
    "                    if exists:\n",
    "                        print('Classification Time')\n",
    "                        classes, indices, percentages = Classifier.Classify(croppedstream_recent, True)\n",
    "                        for i in range(0,26):\n",
    "                            gesture = classes[indices[0][i]]\n",
    "                            if gesture not in detectable:\n",
    "                                continue\n",
    "                            else:\n",
    "                                print(gesture)\n",
    "                                actuator(gesture)\n",
    "                                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\jcruz\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Got enough frames. Analyzing\n",
      "Classification Time\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\jcruz\\Downloads\\GestureRecognition\\airsim\\resnext.py:121: UserWarning: nn.init.kaiming_normal is now deprecated in favor of nn.init.kaiming_normal_.\n",
      "  m.weight = nn.init.kaiming_normal(m.weight, mode='fan_out')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pulling_Hand_In\n",
      "Go Forward\n",
      "Apply brakes\n",
      "RESET BISH\n",
      "DONE RESETTING\n",
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\jcruz\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\ipykernel_launcher.py:6: DeprecationWarning: The binary mode of fromstring is deprecated, as it behaves surprisingly on unicode inputs. Use frombuffer instead\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Got enough frames. Analyzing\n",
      "Classification Time\n",
      "Pulling_Hand_In\n",
      "Go Forward\n",
      "Apply brakes\n",
      "RESET BISH\n",
      "DONE RESETTING\n",
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n",
      "Got enough frames. Analyzing\n",
      "Classification Time\n",
      "Pulling_Hand_In\n",
      "Go Forward\n",
      "Apply brakes\n",
      "RESET BISH\n",
      "DONE RESETTING\n",
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n",
      "Got enough frames. Analyzing\n",
      "Classification Time\n",
      "Swiping_Right\n",
      "Go Forward\n",
      "Go Forward, steer left\n",
      "Go Forward\n",
      "Apply brakes\n",
      "RESET BISH\n",
      "DONE RESETTING\n",
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n",
      "Got enough frames. Analyzing\n",
      "Classification Time\n",
      "Pulling_Hand_In\n",
      "Go Forward\n",
      "Go Forward, steer right\n",
      "Go Forward\n",
      "Apply brakes\n",
      "RESET BISH\n",
      "DONE RESETTING\n",
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n",
      "Got enough frames. Analyzing\n",
      "Classification Time\n",
      "Pulling_Hand_In\n",
      "Go Forward\n",
      "Apply brakes\n",
      "RESET BISH\n",
      "DONE RESETTING\n",
      "Connected!\n",
      "Client Ver:1 (Min Req: 1), Server Ver:1 (Min Req: 1)\n",
      "\n",
      "Got enough frames. Analyzing\n",
      "Classification Time\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':  \n",
    "    global trigger\n",
    "    trigger = True\n",
    "    \n",
    "    while 1:\n",
    "        DetectAndDrive()\n",
    "        #print(trigger)\n",
    "        #if trigger:s\n",
    "        #    print(\"GOING\")\n",
    "        #    DetectAndDrive()\n",
    "        #    trigger = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}