IV. Testing on real-time data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4ae6c3b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import cv2\n",
    "import mediapipe as mp\n",
    "from model import SignDetection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "de29e4f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from collections import deque"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "09687c3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_PATH=os.path.join('model','model')\n",
    "RENDER_PATH=os.path.join('renders')\n",
    "gesture_model=torch.load(os.path.join(MODEL_PATH,'model_2000.pt')).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3fcb0861",
   "metadata": {},
   "outputs": [],
   "source": [
    "actions=['I am OK','STOP !','Descend','I am not OK','Ascend']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "46bd9ab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "mp_draw=mp.solutions.drawing_utils\n",
    "mp_hands=mp.solutions.hands\n",
    "mp_holistic=mp.solutions.holistic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "79d59016",
   "metadata": {},
   "outputs": [],
   "source": [
    "cap =cv2.VideoCapture(0)\n",
    "frame_deque=deque(maxlen=20)\n",
    "conclusion_deque=deque(maxlen=10)\n",
    "draw_frames=False #This flag decides if you want to draw the wireframe on the video stream or not for aesthetics. \n",
    "#draw_frames if True will cover the hands, body and the hands with the wireframes. False will keep the video clean.\n",
    "#draw_frames does not impact the quality of the detection\n",
    "\n",
    "vid_filename=os.path.join(RENDER_PATH,'render'+'.mp4')\n",
    "fourcc_codec=cv2.VideoWriter_fourcc(*'MPEG')\n",
    "fps=cap.get(cv2.CAP_PROP_FPS)*0.8\n",
    "dimensions=(cap.get(cv2.CAP_PROP_FRAME_WIDTH),cap.get(cv2.CAP_PROP_FRAME_HEIGHT))\n",
    "output = cv2.VideoWriter(vid_filename, cv2.VideoWriter_fourcc(*'MPEG'),fps, (640,480))\n",
    "\n",
    "with mp_hands.Hands(min_detection_confidence=0.5, min_tracking_confidence=0.5) as hands:\n",
    "    with mp_holistic.Holistic(min_detection_confidence=0.5, min_tracking_confidence=0.5) as holistic:\n",
    "        while cap.isOpened():\n",
    "            isframe, frame=cap.read()\n",
    "            if not isframe:\n",
    "                print('No frame')\n",
    "            frame=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)\n",
    "            frame.flags.writeable = False \n",
    "            grid=hands.process(frame)\n",
    "            whole=holistic.process(frame)\n",
    "            frame.flags.writeable = True\n",
    "            grids=grid.multi_hand_landmarks\n",
    "            if draw_frames:\n",
    "                mp_draw.draw_landmarks(frame, whole.face_landmarks, mp_holistic.FACEMESH_CONTOURS)\n",
    "                mp_draw.draw_landmarks(frame, whole.pose_landmarks, mp_holistic.POSE_CONNECTIONS) \n",
    "            if grids:\n",
    "                if draw_frames:\n",
    "                    for landmark in grids:\n",
    "                        mp_draw.draw_landmarks(frame,landmark,mp_hands.HAND_CONNECTIONS)\n",
    "                hand1=np.array([[lm.x,lm.y,lm.z] for lm in grids[0].landmark]).flatten()\n",
    "                if len(grids)>1:\n",
    "                    hand2=np.array([[lm.x,lm.y,lm.z] for lm in grids[1].landmark]).flatten()\n",
    "                else:\n",
    "                    hand2=np.zeros((21,3)).flatten() #21 landmarks each having x,y,z coordinates\n",
    "                f_grid=np.hstack((hand1,hand2))\n",
    "                frame_deque.append(f_grid)\n",
    "                if len(frame_deque)==20:\n",
    "                    input_gest=torch.from_numpy(np.expand_dims(np.array(frame_deque),axis=0)).type(torch.FloatTensor).cuda() #LSTM model needs the input as a 3-D tensor - batch X frames X token_no. per frame\n",
    "                    predicted_label=gesture_model(input_gest)\n",
    "                    label=actions[torch.argmax(predicted_label, dim=1).item()]\n",
    "                    conclusion_deque.append(label)\n",
    "                    prob=np.round(torch.max(predicted_label).cpu().detach().numpy(),3)\n",
    "                    if label == np.unique(conclusion_deque)[0]:\n",
    "                        text=label\n",
    "                    else:\n",
    "                        text=''\n",
    "                    cv2.putText(frame,text,(60,65),cv2.FONT_HERSHEY_SIMPLEX, 1.2, (0,0, 0), 2, cv2.LINE_AA)\n",
    "                else:\n",
    "                    pass\n",
    "            frame=cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)\n",
    "            output.write(frame)\n",
    "            cv2.imshow('Output',frame)\n",
    "\n",
    "            if cv2.waitKey(10) & 0xFF==ord('q'):\n",
    "                break\n",
    "        cap.release()\n",
    "        cv2.destroyAllWindows()\n",
    "        output.release()   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b01f23dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}