doc/Getting Started.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "219c0a05",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import json\n",
    "from datetime import datetime\n",
    "import io\n",
    "import os\n",
    "import sys\n",
    "\n",
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "from IPython.display import Video as JupyterVideo\n",
    "from IPython.display import HTML\n",
    "from IPython.display import display\n",
    "from os import system, name\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '1'\n",
    "os.environ['PYOPENGL_PLATFORM'] = 'osmesa'\n",
    "\n",
    "import pose_pipeline\n",
    "from pose_pipeline import *\n",
    "from pose_pipeline.utils.jupyter import play, play_grid\n",
    "pose_pipeline.set_environmental_variables()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "496b51da",
   "metadata": {},
   "source": [
    "[Data organization](erd.png)\n",
    "\n",
    "## Import test videos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc572807",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pose_pipeline.utils.video_format import insert_local_video\n",
    "\n",
    "test_videos_path = '/videos'\n",
    "files = os.listdir(test_videos_path)\n",
    "\n",
    "for f in files:\n",
    "    insert_local_video(f, datetime.now(), os.path.join(test_videos_path, f), video_project='DEMO_TEST', skip_duplicates=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6fa956fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "filt = {'video_project': \"DEMO_TEST\"}\n",
    "\n",
    "# List videos in database\n",
    "VideoInfo.populate(filt)\n",
    "Video & 'video_project=\"DEMO_TEST\"'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pose_pipeline.pipeline import BottomUpBridging,BottomUpBridgingPerson\n",
    "\n",
    "BottomUpBridging.populate(filt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BottomUpBridging()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "video_keys = (Video & 'video_project=\"DEMO_TEST\"').fetch('KEY')\n",
    "\n",
    "for v in video_keys:\n",
    "    print(v)\n",
    "    v[\"bottom_up_method_name\"] = \"Bridging_OpenPose\"\n",
    "\n",
    "    BottomUpMethod.insert1(v, skip_duplicates=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BottomUpPeople.populate(filt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BottomUpPeople()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BottomUpPeople & filt & 'bottom_up_method_name=\"Bridging_OpenPose\"'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BlurredVideo.populate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BlurredVideo()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64d6fce1",
   "metadata": {},
   "source": [
    "## Run tracking algorithm\n",
    "\n",
    "After running this, use the GUI to annotate the video to indicate the subject of interest."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b94e962",
   "metadata": {},
   "outputs": [],
   "source": [
    "# select tracking method\n",
    "\n",
    "filt = 'video_project=\"DEMO_TEST\" AND filename LIKE \"%test%\"'\n",
    "\n",
    "keys = ((Video & filt)).fetch('KEY')\n",
    "\n",
    "print(f'Missing tracking methods: {len(keys)}')\n",
    "tracking_method = (TrackingBboxMethodLookup & 'tracking_method_name=\"MMDet_deepsort\"').fetch1('tracking_method')\n",
    "for key in keys:\n",
    "    key['tracking_method'] = tracking_method\n",
    "    TrackingBboxMethod.insert1(key, skip_duplicates=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TrackingBboxMethod()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c6c96a9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "TrackingBbox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df3ebc1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# OpenPose.populate(filt)\n",
    "filt = 'video_project=\"DEMO_TEST\" AND filename LIKE \"%test%\"'\n",
    "TrackingBbox.populate(filt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TrackingBbox()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BottomUpPeople & video_keys & 'bottom_up_method_name=\"Bridging_OpenPose\"'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70b426df",
   "metadata": {},
   "outputs": [],
   "source": [
    "# produce visualizations of these outputs used for annotation\n",
    "\n",
    "BlurredVideo.populate(filt)\n",
    "TrackingBboxVideo.populate(filt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pose_pipeline.utils.annotation import annotate\n",
    "\n",
    "filt = {'video_project': \"DEMO_TEST\"}\n",
    "\n",
    "# Run annotation GUI\n",
    "to_label = ((TrackingBboxVideo & filt) - PersonBboxValid).fetch('KEY')\n",
    "\n",
    "print(f'{len(to_label)} found to annotate')\n",
    "\n",
    "if len(to_label) > 0:\n",
    "\n",
    "    key = to_label[0]\n",
    "\n",
    "    annotate(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e1388d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "PersonBboxValid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "PersonBbox.populate(filt)\n",
    "\n",
    "top_down_method = (TopDownMethodLookup & 'top_down_method_name=\"MMPose_RTMPose_Cocktail14\"').fetch1('top_down_method')\n",
    "\n",
    "keys = ((PersonBbox & filt) - (TopDownMethod & {'top_down_method': top_down_method})).fetch('KEY')\n",
    "\n",
    "print(f'Missing top down methods: {len(keys)}')\n",
    "for key in keys:\n",
    "    key['top_down_method'] = top_down_method\n",
    "    TopDownMethod.insert1(key)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ce650d96",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = (TopDownMethod() & \"top_down_method=16\").fetch('KEY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate the selected analyses\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = '1'\n",
    "TopDownPerson.populate(keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "75660786",
   "metadata": {},
   "outputs": [],
   "source": [
    "kp = (TopDownPerson() & filt).fetch('keypoints')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d93ba9bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "kp[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TopDownPersonVideo.populate(keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35ad41f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "TopDownPersonVideo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# keys = (TopDownPersonVideo & filt).fetch('KEY')\n",
    "# print(keys)\n",
    "play(TopDownPersonVideo & {'filename':'koa_212_test.mp4'} & {'top_down_method': 16})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "((PersonBbox & filt) - (TopDownMethod & {'top_down_method': top_down_method})).fetch('KEY')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(PersonBbox())\n",
    "display(PersonBboxValid())\n",
    "display(TopDownMethod())\n",
    "display(TopDownPerson())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4221a864",
   "metadata": {},
   "source": [
    "## Perform additional analysis on the selected subject"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2703b77",
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the final bounding box for subjects of interest\n",
    "pose_pipeline.PersonBbox.populate(filt)\n",
    "\n",
    "# select the top down approach to run\n",
    "top_down_method = (pose_pipeline.TopDownMethodLookup & 'top_down_method_name=\"MMPose_RTMPose_Cocktail14\"').fetch1('top_down_method')\n",
    "keys = ((pose_pipeline.PersonBbox & filt) - (pose_pipeline.TopDownMethod & {'top_down_method': top_down_method})).fetch('KEY')\n",
    "print(f'Missing top down methods: {len(keys)}')\n",
    "for key in keys:\n",
    "    key['top_down_method'] = top_down_method\n",
    "    pose_pipeline.TopDownMethod.insert1(key)\n",
    "\n",
    "# select an SMPL method to run\n",
    "smpl_method = (pose_pipeline.SMPLMethodLookup & 'smpl_method_name=\"HybrIK\"').fetch1('smpl_method')\n",
    "keys = ((pose_pipeline.PersonBbox & filt) - (pose_pipeline.SMPLMethod & {'smpl_method': smpl_method})).fetch('KEY')\n",
    "print(f'Missing SMPL methods: {len(keys)}')\n",
    "for key in keys:\n",
    "    key['smpl_method'] = smpl_method\n",
    "    pose_pipeline.SMPLMethod.insert1(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6942c9e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "(SMPLMethod() & \"smpl_method=4\").delete()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96b0d3b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# populate the selected analyses\n",
    "\n",
    "# TopDownPerson.populate(keys)\n",
    "SMPLPerson.populate(keys)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0eee8339",
   "metadata": {},
   "source": [
    "## Produce and review visualizations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9319194a",
   "metadata": {},
   "outputs": [],
   "source": [
    "TopDownPersonVideo.populate(filt)\n",
    "SMPLPersonVideo.populate(filt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e797e258",
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = (TopDownPersonVideo & filt).fetch('KEY')\n",
    "play(TopDownPersonVideo & keys[0])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}