From c2071c840850f3b1152df2be4c213e7530c206d0 Mon Sep 17 00:00:00 2001 From: lingjiekong Date: Sun, 7 May 2023 15:11:32 -0700 Subject: [PATCH 1/2] fix base base Dockerfile issue --- docker/base/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 6b39c3ea..179cc2f1 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -30,7 +30,7 @@ RUN apt-get update && \ RUN pip3 install setuptools Cython wheel RUN pip3 install h5py==2.10.0 --verbose -RUN pip3 install future==0.17.1 mock==3.0.5 keras_preprocessing==1.0.5 keras_applications==1.0.8 gast==0.2.2 futures protobuf pybind11 --verbose +RUN pip3 install future==0.17.1 mock==3.0.5 keras_preprocessing==1.0.5 keras_applications==1.0.8 gast==0.2.2 futures protobuf==3.19.4 pybind11 --verbose RUN pip3 install numpy --verbose ARG TENSORFLOW_URL=https://nvidia.box.com/shared/static/rummpy6q1km1wivomalpkwt2jy28mndf.whl @@ -79,7 +79,8 @@ RUN echo -e "\e[48;5;172m Install Jupyter Lab \e[0m" RUN apt-get update RUN apt-get install -y curl RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - -RUN apt-get install -y nodejs libffi-dev +RUN apt-get install -y nodejs libffi-dev +RUN pip3 install packaging RUN pip3 install jupyter jupyterlab==2.2.6 RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager From ee45f6756892fe1e2e01733c54f55753b2f3d553 Mon Sep 17 00:00:00 2001 From: Lingjie Kong Date: Sat, 17 Jun 2023 10:33:56 -0700 Subject: [PATCH 2/2] combine jetbot with jetson voice --- docker/base/Dockerfile | 2 + docker/configure.sh | 6 +- notebooks/voice_motion/voice_motion.ipynb | 785 ++++++++++++++++++++++ 3 files changed, 791 insertions(+), 2 deletions(-) create mode 100644 notebooks/voice_motion/voice_motion.ipynb diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 179cc2f1..1ffe014a 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -86,6 +86,8 @@ RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager # Install jupyter_clickable_image_widget +RUN pip3 install nbconvert pygments==2.4.1 + RUN echo "\e[42m Install jupyter_clickable_image_widget \e[0m" RUN cd && \ apt-get install -y libssl1.0-dev && \ diff --git a/docker/configure.sh b/docker/configure.sh index 127e2ab1..620ff543 100755 --- a/docker/configure.sh +++ b/docker/configure.sh @@ -14,10 +14,12 @@ then JETBOT_BASE_IMAGE=nvcr.io/nvidia/l4t-pytorch:r32.4.3-pth1.6-py3 elif [[ "$L4T_VERSION" == "32.4.4" ]] then - JETBOT_BASE_IMAGE=nvcr.io/nvidia/l4t-pytorch:r32.4.4-pth1.6-py3 + #JETBOT_BASE_IMAGE=nvcr.io/nvidia/l4t-pytorch:r32.4.4-pth1.6-py3 + JETBOT_BASE_IMAGE=dustynv/jetson-voice:r32.4.4 elif [[ "$L4T_VERSION" == "32.5.0" ]] || [[ "$L4T_VERSION" == "32.5.1" ]] then - JETBOT_BASE_IMAGE=nvcr.io/nvidia/l4t-pytorch:r32.5.0-pth1.6-py3 + #JETBOT_BASE_IMAGE=nvcr.io/nvidia/l4t-pytorch:r32.5.0-pth1.6-py3 + JETBOT_BASE_IMAGE=dustynv/jetson-voice:r32.5.0 else echo "JETBOT_BASE_IMAGE not found for ${L4T_VERSION}. Please manually set the JETBOT_BASE_IMAGE environment variable. (ie: export JETBOT_BASE_IMAGE=...)" fi diff --git a/notebooks/voice_motion/voice_motion.ipynb b/notebooks/voice_motion/voice_motion.ipynb new file mode 100644 index 00000000..9bc78a82 --- /dev/null +++ b/notebooks/voice_motion/voice_motion.ipynb @@ -0,0 +1,785 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic Motion\n", + "\n", + "Welcome to JetBot's browser based programming interface! This document is\n", + "called a *Jupyter Notebook*, which combines text, code, and graphic\n", + "display all in one! Pretty neat, huh? If you're unfamiliar with *Jupyter* we suggest clicking the \n", + "``Help`` drop down menu in the top toolbar. This has useful references for\n", + "programming with *Jupyter*. \n", + "\n", + "In this notebook, we'll cover the basics of controlling JetBot. \n", + "\n", + "### Importing the Robot class\n", + "\n", + "To get started programming JetBot, we'll need to import the ``Robot`` class. This class\n", + "allows us to easily control the robot's motors! This is contained in the ``jetbot`` package.\n", + "\n", + "> If you're new to Python, a *package* is essentially a folder containing \n", + "> code files. These code files are called *modules*.\n", + "\n", + "To import the ``Robot`` class, highlight the cell below and press ``ctrl + enter`` or the ``play`` icon above.\n", + "This will execute the code contained in the cell" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": {}, + "outputs": [], + "source": [ + "from jetbot import Robot\n", + "from jetson_voice import ASR, AudioInput, ConfigArgParser, list_audio_devices\n", + "import time\n", + "\n", + "robot = Robot()\n", + "robot.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [], + "source": [ + "# # robot.set_motors(0.8, 0.1)\n", + "# # robot.forward(0.3)\n", + "# robot.left_motor.value = 0.3\n", + "# robot.right_motor.value = 0.292" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "robot.stop()" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[NeMo W 2023-05-14 23:11:45 nemo_logging:349] /usr/local/lib/python3.6/dist-packages/pydub/utils.py:170: RuntimeWarning: Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\n", + " warn(\"Couldn't find ffmpeg or avconv - defaulting to ffmpeg, but may not work\", RuntimeWarning)\n", + " \n", + "[NeMo W 2023-05-14 23:11:49 experimental:28] Module is experimental, not ready for production and is not fully supported. Use at your own risk.\n", + "################################################################################\n", + "### WARNING, path does not exist: KALDI_ROOT=/mnt/matylda5/iveselyk/Tools/kaldi-trunk\n", + "### (please add 'export KALDI_ROOT=' in your $HOME/.profile)\n", + "### (or run as: KALDI_ROOT= python .py)\n", + "################################################################################\n", + "\n", + "[2023-05-14 23:11:50] resource.py:114 - loading model '/jetson-voice/data/networks/asr/matchboxnet-3x1x64_subset/matchboxnet-3x1x64_subset.onnx' with jetson_voice.backends.tensorrt.TRTModel\n", + "[2023-05-14 23:11:50] trt_model.py:41 - loading cached TensorRT engine from /jetson-voice/data/networks/asr/matchboxnet-3x1x64_subset/matchboxnet-3x1x64_subset.engine\n", + "[2023-05-14 23:12:53] trt_model.py:59 - loaded TensorRT engine from /jetson-voice/data/networks/asr/matchboxnet-3x1x64_subset/matchboxnet-3x1x64_subset.engine\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "binding 0 - 'audio_signal'\n", + " input: True\n", + " shape: (1, 64, -1)\n", + " dtype: DataType.FLOAT\n", + " size: -256\n", + " dynamic: True\n", + " profiles: [{'min': (1, 64, 10), 'opt': (1, 64, 150), 'max': (1, 64, 300)}]\n", + "\n", + "\n", + "binding 1 - 'logits'\n", + " input: False\n", + " shape: (1, 12)\n", + " dtype: DataType.FLOAT\n", + " size: 48\n", + " dynamic: False\n", + " profiles: []\n", + "\n", + "Audio Input Device:\n", + "{'defaultHighInputLatency': 0.034829931972789115,\n", + " 'defaultHighOutputLatency': 0.034829931972789115,\n", + " 'defaultLowInputLatency': 0.008684807256235827,\n", + " 'defaultLowOutputLatency': 0.008684807256235827,\n", + " 'defaultSampleRate': 44100.0,\n", + " 'hostApi': 0,\n", + " 'index': 11,\n", + " 'maxInputChannels': 2,\n", + " 'maxOutputChannels': 2,\n", + " 'name': 'USB PnP Audio Device: Audio (hw:2,0)',\n", + " 'structVersion': 2}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "[2023-05-14 23:13:04] audio.py:156 - trying to open audio input 11 with sample_rate=16000 chunk_size=16000\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "audio stream opened on device 11 (USB PnP Audio Device: Audio (hw:2,0))\n", + "you can begin speaking now... (press Ctrl+C to exit)\n", + "\n", + "class 'up' (0.47758257389068604)\n", + "class 'up' (0.17972323298454285)\n", + "class 'unknown' (0.5959203243255615)\n", + "class 'down' (0.39055681228637695)\n", + "class 'up' (0.18999901413917542)\n", + "class 'unknown' (0.36330747604370117)\n", + "class 'down' (0.39250972867012024)\n", + "class 'unknown' (0.8131647109985352)\n", + "class 'down' (0.559116780757904)\n", + "class 'on' (0.3552650213241577)\n", + "class 'on' (0.5195392966270447)\n", + "class 'down' (0.24975326657295227)\n", + "class 'stop' (0.4126604497432709)\n", + "class 'down' (0.5239905118942261)\n", + "class 'up' (0.19063876569271088)\n", + "class 'down' (0.8860142230987549)\n", + "class 'down' (0.4649629592895508)\n", + "class 'go' (0.18217088282108307)\n", + "class 'on' (0.22337010502815247)\n", + "class 'off' (0.5042861700057983)\n", + "class 'stop' (0.2374659925699234)\n", + "class 'stop' (0.900909423828125)\n", + "class 'stop' (0.9965978264808655)\n", + "class 'up' (0.5936501026153564)\n", + "class 'up' (0.3471977710723877)\n", + "class 'off' (0.7540872693061829)\n", + "class 'up' (0.3489254117012024)\n", + "class 'up' (0.9776104092597961)\n", + "class 'up' (0.1744718700647354)\n", + "class 'up' (0.48238179087638855)\n", + "class 'on' (0.19342714548110962)\n", + "class 'on' (0.21242938935756683)\n", + "class 'left' (0.3764282464981079)\n", + "class 'left' (0.9536210894584656)\n", + "class 'up' (0.38639336824417114)\n", + "class 'right' (0.8279017806053162)\n", + "class 'right' (0.908586323261261)\n", + "class 'left' (0.3611232042312622)\n", + "class 'up' (0.2703484296798706)\n", + "class 'silence' (0.9673715829849243)\n", + "class 'off' (0.5617191195487976)\n", + "class 'up' (0.30249470472335815)\n", + "class 'stop' (0.24597758054733276)\n", + "class 'off' (0.4977096915245056)\n", + "class 'off' (0.9413579106330872)\n", + "class 'off' (0.5274643301963806)\n", + "class 'up' (0.31993696093559265)\n", + "class 'up' (0.38446396589279175)\n", + "class 'up' (0.5273299813270569)\n", + "class 'stop' (0.44564932584762573)\n", + "class 'up' (0.21743406355381012)\n", + "class 'silence' (0.6260421872138977)\n", + "class 'silence' (0.4524206817150116)\n", + "class 'up' (0.1738329827785492)\n", + "class 'off' (0.26901260018348694)\n", + "class 'off' (0.3267706632614136)\n", + "class 'off' (0.37149500846862793)\n", + "class 'stop' (0.6259336471557617)\n", + "class 'stop' (0.9348324537277222)\n", + "class 'unknown' (0.48082834482192993)\n", + "class 'unknown' (0.5176094174385071)\n", + "class 'up' (0.3379831314086914)\n", + "class 'unknown' (0.38183698058128357)\n", + "class 'unknown' (0.1721474826335907)\n", + "class 'stop' (0.48369938135147095)\n", + "class 'unknown' (0.15475134551525116)\n", + "class 'unknown' (0.27649638056755066)\n", + "class 'up' (0.21431605517864227)\n", + "class 'yes' (0.2700769305229187)\n", + "class 'yes' (0.7635130286216736)\n", + "class 'unknown' (0.6434178352355957)\n", + "class 'down' (0.9041785001754761)\n", + "class 'down' (0.973041832447052)\n", + "class 'up' (0.15651315450668335)\n", + "class 'on' (0.5197495222091675)\n", + "class 'stop' (0.3398621678352356)\n", + "class 'off' (0.6326382160186768)\n", + "class 'go' (0.34792226552963257)\n", + "class 'silence' (0.27743372321128845)\n", + "class 'go' (0.11794213950634003)\n", + "class 'off' (0.1754484921693802)\n", + "class 'off' (0.16860008239746094)\n", + "class 'go' (0.1937856823205948)\n", + "class 'on' (0.18360832333564758)\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m\u001b[0m", + "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 19\u001b[0m \u001b[0;31m# run transcription\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 21\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 22\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/jetson-voice/jetson_voice/utils/audio.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 208\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m__next__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 209\u001b[0;31m \u001b[0msamples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 210\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 211\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/jetson-voice/jetson_voice/utils/audio.py\u001b[0m in \u001b[0;36mnext\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 194\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 195\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 196\u001b[0;31m \u001b[0msamples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstream\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice_chunk_size\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexception_on_overflow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 197\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfrombuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint16\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/pyaudio.py\u001b[0m in \u001b[0;36mread\u001b[0;34m(self, num_frames, exception_on_overflow)\u001b[0m\n\u001b[1;32m 606\u001b[0m paCanNotReadFromAnOutputOnlyStream)\n\u001b[1;32m 607\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 608\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mpa\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_stream\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stream\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_frames\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mexception_on_overflow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 609\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 610\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_read_available\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "model = \"matchboxnet\"\n", + "wav = None\n", + "# wav = \"data/audio/commands.wav\"\n", + "mic = \"11\"\n", + "# mic = None\n", + "\n", + "\n", + "# load the model\n", + "asr = ASR(model)\n", + "\n", + "# create the audio input stream\n", + "stream = AudioInput(wav=wav, mic=mic, \n", + " sample_rate=asr.sample_rate, \n", + " chunk_size=asr.chunk_size)\n", + "\n", + "# robot\n", + "robot = Robot()\n", + "\n", + "# run transcription\n", + "for samples in stream:\n", + " results = asr(samples)\n", + " \n", + " if asr.classification:\n", + "# print(f\"class '{results[0]}' ({results[1]:.3f})\")\n", + " print(f\"class '{results[0]}' ({results[1]})\")\n", + " motion = results[0]\n", + " accuracy = results[1]\n", + " if motion == 'left' and accuracy > 0.5:\n", + " robot.left(speed=0.3)\n", + "# time.sleep(2)\n", + "# robot.stop()\n", + " elif motion == 'right' and accuracy > 0.5:\n", + " robot.right(speed=0.3)\n", + "# time.sleep(2)\n", + "# robot.stop()\n", + " elif motion == 'up' and accuracy > 0.5:\n", + "# robot.forward(speed=0.3)\n", + " robot.left_motor.value = 0.3\n", + " robot.right_motor.value = 0.292\n", + "# time.sleep(2)\n", + "# robot.stop()\n", + " elif motion == 'down' and accuracy > 0.5:\n", + "# robot.backward(speed=0.3)\n", + " robot.left_motor.value = -0.3\n", + " robot.right_motor.value = -0.292\n", + "# time.sleep(2)\n", + "# robot.stop()\n", + " elif motion == 'stop' and accuracy > 0.5:\n", + " robot.stop()\n", + "# time.sleep(2)\n", + "# else:\n", + "# # do nothing\n", + " \n", + " else:\n", + " for transcript in results:\n", + " print(transcript['text'])\n", + " \n", + " if transcript['end']:\n", + " print('')\n", + " \n", + "print('\\naudio stream closed.')\n", + " " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we've imported the ``Robot`` class we can initialize the class *instance* as follows. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Commanding the robot" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we've created our ``Robot`` instance we named \"robot\", we can use this instance\n", + "to control the robot. To make the robot spin counterclockwise at 30% of it's max speed\n", + "we can call the following\n", + "\n", + "> WARNING: This next command will make the robot move! Please make sure the robot has clearance." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "robot.left(speed=0.3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Cool, you should see the robot spin counterclockwise!\n", + "\n", + "> If your robot didn't turn left, that means one of the motors is wired backwards! Try powering down your\n", + "> robot and swapping the terminals that the ``red`` and ``black`` cables of the incorrect motor.\n", + "> \n", + "> REMINDER: Always be careful to check your wiring, and don't change the wiring on a running system!\n", + "\n", + "Now, to stop the robot you can call the ``stop`` method." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "robot.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Maybe we only want to run the robot for a set period of time. For that, we can use the Python ``time`` package. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import time" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This package defines the ``sleep`` function, which causes the code execution to block for the specified number of seconds\n", + "before running the next command. Try the following to make the robot turn left only for half a second." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "robot.left(0.3)\n", + "time.sleep(0.5)\n", + "robot.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great. You should see the robot turn left for a bit and then stop.\n", + "\n", + "> Wondering what happened to the ``speed=`` inside the ``left`` method? Python allows \n", + "> us to set function parameters by either their name, or the order that they are defined\n", + "> (without specifying the name).\n", + "\n", + "The ``BasicJetbot`` class also has the methods ``right``, ``forward``, and ``backward``. Try creating your own cell to make\n", + "the robot move forward at 50% speed for one second.\n", + "\n", + "Create a new cell by highlighting an existing cell and pressing ``b`` or the ``+`` icon above. Once you've done that, type in the code that you think will make the robot move forward at 50% speed for one second." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Controlling motors individually\n", + "\n", + "Above we saw how we can control the robot using commands like ``left``, ``right``, etc. But what if we want to set each motor speed \n", + "individually? Well, there are two ways you can do this\n", + "\n", + "The first way is to call the ``set_motors`` method. For example, to turn along a left arch for a second we could set the left motor to 30% and the right motor to 60% like follows." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "robot.set_motors(0.3, 0.6)\n", + "time.sleep(1.0)\n", + "robot.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great! You should see the robot move along a left arch. But actually, there's another way that we could accomplish the same thing.\n", + "\n", + "The ``Robot`` class has two attributes named ``left_motor`` and ``right_motor`` that represent each motor individually.\n", + "These attributes are ``Motor`` class instances, each which contains a ``value`` attribute. This ``value`` attribute\n", + "is a [traitlet](https://github.com/ipython/traitlets) which generates ``events`` when assigned a new value. In the motor\n", + "class, we attach a function that updates the motor commands whenever the value changes.\n", + "\n", + "So, to accomplish the exact same thing we did above, we could execute the following." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "robot.left_motor.value = 0.3\n", + "robot.right_motor.value = 0.6\n", + "time.sleep(1.0)\n", + "robot.left_motor.value = 0.0\n", + "robot.right_motor.value = 0.0" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should see the robot move in the same exact way!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Link motors to traitlets" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A really cool feature about these [traitlets](https://github.com/ipython/traitlets) is that we can \n", + "also link them to other traitlets! This is super handy because Jupyter Notebooks allow us\n", + "to make graphical ``widgets`` that use traitlets under the hood. This means we can attach\n", + "our motors to ``widgets`` to control them from the browser, or just visualize the value.\n", + "\n", + "To show how to do this, let's create and display two sliders that we'll use to control our motors." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import ipywidgets.widgets as widgets\n", + "from IPython.display import display\n", + "\n", + "# create two sliders with range [-1.0, 1.0]\n", + "left_slider = widgets.FloatSlider(description='left', min=-1.0, max=1.0, step=0.01, orientation='vertical')\n", + "right_slider = widgets.FloatSlider(description='right', min=-1.0, max=1.0, step=0.01, orientation='vertical')\n", + "\n", + "# create a horizontal box container to place the sliders next to each other\n", + "slider_container = widgets.HBox([left_slider, right_slider])\n", + "\n", + "# display the container in this cell's output\n", + "display(slider_container)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should see two ``vertical`` sliders displayed above. \n", + "\n", + "> HELPFUL TIP: In Jupyter Lab, you can actually \"pop\" the output of cells into entirely separate window! It will still be \n", + "> connected to the notebook, but displayed separately. This is helpful if we want to pin the output of code we executed elsewhere.\n", + "> To do this, right click the output of the cell and select ``Create New View for Output``. You can then drag the new window\n", + "> to a location you find pleasing.\n", + "\n", + "Try clicking and dragging the sliders up and down. Notice nothing happens when we move the sliders currently. That's because we haven't connected them to motors yet! We'll do that by using the ``link`` function from the traitlets package." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import traitlets\n", + "\n", + "left_link = traitlets.link((left_slider, 'value'), (robot.left_motor, 'value'))\n", + "right_link = traitlets.link((right_slider, 'value'), (robot.right_motor, 'value'))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now try dragging the sliders (slowly at first). You should see the respective motor turn!\n", + "\n", + "The ``link`` function that we created above actually creates a bi-directional link! That means,\n", + "if we set the motor values elsewhere, the sliders will update! Try executing the code block below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "robot.forward(0.3)\n", + "time.sleep(1.0)\n", + "robot.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should see the sliders respond to the motor commands! If we want to remove this connection we can call the\n", + "``unlink`` method of each link." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "left_link.unlink()\n", + "right_link.unlink()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "But what if we don't want a *bi-directional* link, let's say we only want to use the sliders to display the motor values,\n", + "but not control them. For that we can use the ``dlink`` function. The left input is the ``source`` and the right input is the ``target``" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "left_link = traitlets.dlink((robot.left_motor, 'value'), (left_slider, 'value'))\n", + "right_link = traitlets.dlink((robot.right_motor, 'value'), (right_slider, 'value'))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now try moving the sliders. You should see that the robot doesn't respond. But when set the motors using a different method,\n", + "the sliders will update and display the value!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Attach functions to events" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Another way to use traitlets, is by attaching functions (like ``forward``) to events. These\n", + "functions will get called whenever a change to the object occurs, and will be passed some information about that change\n", + "like the ``old`` value and the ``new`` value. \n", + "\n", + "Let's create and display some buttons that we'll use to control the robot." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create buttons\n", + "button_layout = widgets.Layout(width='100px', height='80px', align_self='center')\n", + "stop_button = widgets.Button(description='stop', button_style='danger', layout=button_layout)\n", + "forward_button = widgets.Button(description='forward', layout=button_layout)\n", + "backward_button = widgets.Button(description='backward', layout=button_layout)\n", + "left_button = widgets.Button(description='left', layout=button_layout)\n", + "right_button = widgets.Button(description='right', layout=button_layout)\n", + "\n", + "# display buttons\n", + "middle_box = widgets.HBox([left_button, stop_button, right_button], layout=widgets.Layout(align_self='center'))\n", + "controls_box = widgets.VBox([forward_button, middle_box, backward_button])\n", + "display(controls_box)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You should see a set of robot controls displayed above! But right now they wont do anything. To do that\n", + "we'll need to create some functions that we'll attach to the button's ``on_click`` event. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def stop(change):\n", + " robot.stop()\n", + " \n", + "def step_forward(change):\n", + " robot.forward(0.4)\n", + " time.sleep(0.5)\n", + " robot.stop()\n", + "\n", + "def step_backward(change):\n", + " robot.backward(0.4)\n", + " time.sleep(0.5)\n", + " robot.stop()\n", + "\n", + "def step_left(change):\n", + " robot.left(0.3)\n", + " time.sleep(0.5)\n", + " robot.stop()\n", + "\n", + "def step_right(change):\n", + " robot.right(0.3)\n", + " time.sleep(0.5)\n", + " robot.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now that we've defined the functions, let's attach them to the on-click events of each button" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# link buttons to actions\n", + "stop_button.on_click(stop)\n", + "forward_button.on_click(step_forward)\n", + "backward_button.on_click(step_backward)\n", + "left_button.on_click(step_left)\n", + "right_button.on_click(step_right)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now when you click each button, you should see the robot move!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Heartbeat Killswitch\n", + "\n", + "Here we show how to connect a 'heartbeat' to stop the robot from moving. This is a simple way to detect if the robot connection is alive. You can lower the slider below to reduce the period (in seconds) of the heartbeat. If a round-trip communication between browser cannot be made within two heartbeats, the '`status`' attribute of the heartbeat will be set ``dead``. As soon as the connection is restored, the ``status`` attribute will return to ``alive``." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from jetbot import Heartbeat\n", + "\n", + "heartbeat = Heartbeat()\n", + "\n", + "# this function will be called when heartbeat 'alive' status changes\n", + "def handle_heartbeat_status(change):\n", + " if change['new'] == Heartbeat.Status.dead:\n", + " robot.stop()\n", + " \n", + "heartbeat.observe(handle_heartbeat_status, names='status')\n", + "\n", + "period_slider = widgets.FloatSlider(description='period', min=0.001, max=0.5, step=0.01, value=0.5)\n", + "traitlets.dlink((period_slider, 'value'), (heartbeat, 'period'))\n", + "\n", + "display(period_slider, heartbeat.pulseout)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Try executing the code below to start the motors, and then lower the slider to see what happens. You can also try disconnecting your robot or PC." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "robot.left(0.2) \n", + "\n", + "# now lower the `period` slider above until the network heartbeat can't be satisfied" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Conclusion\n", + "\n", + "That's it for this example notebook! Hopefully you feel confident that you can program your robot to move around now :)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +}