Jetson

README.md

@@ -1,24 +1,5 @@
-# CV-piano
-Project source code from https://github.com/Mayuresh1611/Paper-Piano
+# Deployed on JETSON NANO  
 
-## Setting up project
-Python version 3.11 and above
-1. Clone the repository ```git clone https://github.com/Mayuresh1611/Paper-Piano.git```
-2. run command ```pip install -r requirements.txt``` in the command line.
-3. Execute ```run.py``` file
-
-## HOW TO USE   
-This is a little trickier part as the project requires you to set up a webcam in a specific angle at a specific height and distance. Also  stronger the light, the better the performance. 
-#### STUFF YOU WILL REQUIRE 
-1. webcam or you can use third-party tools for webcam. 
-2. Two A4-sized white paper, horizontally joined together. 2 rectangles need to be drawn at both ends of the paper with a black marker, thicker lines yield better results. 
-3. The recommended position for the webcam will be such that it can capture the finger and shadow beneath the finger and should have both boxes we drew on joined paper in the FOV of the camera.
-Just like shown in the demo video.
-4. A light source in front, ie. behind the camera would be preferred. Casting sharp shadows.
-4. Hand with all fingers.
-
-
-# Deployed on JETSON NANO
 It is recommended to burn a new SD card and to configure the environment directly in the system environment.  
 **Configuration environment:** `JetPack4.6.1, Cuda10.2, Python3.6.9`  
 ## MediaPipe installation
@@ -59,9 +40,27 @@ After successfully installing `h5py3.1.0`, download the `Jetpack4.6.1` version o
 sudo pip install xxx/xxx/xxx.whl
 ```
 
-## We need to install opencv-python：
+## We need to install opencv-python
 Reference [Howto-Install-Mediapipe-in-Jetson-Nano](https://github.com/Melvinsajith/How-to-Install-Mediapipe-in-Jetson-Nano). Since the OpenCV that comes with the Jetson system will cause some problems, opencv needs to be reinstalled here. The `remove` operation is completed after the `install`.
 ```
  sudo apt-get install python3-opencv 
  sudo apt-get remove python3-opencv
 ```
+
+After deploying to `Jetson Nano`, due to the lack of memory on the Nano development board, an error occurs when calling multi-threaded playback of mp3 audio, as well as when calling the `pygame` library, it will not run. So in this branch of the code, the `pygame` library is removed, and the `GPIO` port on the development board is used to implement the mp3 audio playback function with the `BY8002` voice module.  
+
+![ca8887bc235ab42e6f3b583cb2bd098.png](https://github.com/cccp421/CV-piano/blob/jetson/ca8887bc235ab42e6f3b583cb2bd098.png)
+# run  
+```
+python3 run.py
+```
+
+## Jetson.GPIO User's Guide  
+```
+sudo pip install Jetson.GPIO
+```
+### Problems with GPIOs not working  
+```
+sudo chmod a+rw /dev/gpiochip0  
+sudo chmod a+rw /dev/gpiochip1
+```

models/model.py

@@ -1,21 +1,17 @@
 import cv2
-from tensorflow.keras.models import load_model
+from tensorflow.keras import models
 import numpy as np
 import sys
 sys.path.append("src")
-from src import GLOBAL
-# import GLOBAL
 import os
 
+TRACKING_BOX_RESOLUTION = (40 , 40)
 
 model_list = os.listdir("models")
-if "touch_detection_model.h5" not in model_list:
-    print("We need to train model on your finger's data")
-else:
-    model = load_model("models/touch_detection_model.h5")
 
 def Predict(img):
-  resized_img = cv2.resize(img, GLOBAL.TRACKING_BOX_RESOLUTION)
+  model = models.load_model("models/touch_detection_model.keras")
+  resized_img = cv2.resize(img, TRACKING_BOX_RESOLUTION)
   # Add the batch dimension and normalize pixel values
   data = np.expand_dims(resized_img/255, axis=0)
   # Make the prediction

requirements.txt

@@ -1,7 +1,7 @@
 numpy
 matplotlib
 tensorflow
-opencv-python
+opencv-tensorpython
 mediapipe
 scipy
 pygame

run.py

@@ -1,112 +1,46 @@
-import cv2 
-import mediapipe as mp 
-import os 
+import os
 import time
-from src import fetch_data , train_model, GLOBAL , piano
-from models import model  
-
-FINGER = [GLOBAL.INDEX_FINGER]
+from src import fetch_data, train_model, piano
 
-def RUN(finger): 
-    mp_drawing = mp.solutions.drawing_utils 
-    mp_hands = mp.solutions.hands  #导入了使用 MediaPipe 库进行手部检测和跟踪的工具。
-
-    cap = cv2.VideoCapture(GLOBAL.WEB_CAM) 
-    iter = 0 
-    with mp_hands.Hands(
-        min_detection_confidence=0.5,   #设置了检测和跟踪的置信度阈值。
-        min_tracking_confidence=0.5) as hands:
-        while cap.isOpened():
-            success, image = cap.read()
-            if not success:
-                print("Ignoring empty camera frame.")
-                continue
-
-            image = cv2.cvtColor(image , cv2.COLOR_BGR2RGB)
-            copy_image = image.copy()
-
-            image.flags.writeable = False
-            results = hands.process(image)
-            image.flags.writeable = True
-            image = cv2.cvtColor(image, cv2.COLOR_RGB2BGR)
-
-            finger_tracking_frame = None # initializing region of interest
-
-            if results.multi_hand_landmarks:
-                for hand_landmarks in results.multi_hand_landmarks:
-                    mp_drawing.draw_landmarks(
-                        image, hand_landmarks, mp_hands.HAND_CONNECTIONS)
-                    for finger_tip_id in [finger]:  # Landmark IDs for all five fingers' tips
-                        finger_tip = hand_landmarks.landmark[finger_tip_id]
-                        height, width, _ = image.shape
-                        tip_x, tip_y, tip_z = int(finger_tip.x * width), int(finger_tip.y * height), finger_tip.z
-
-                        box_size = int(GLOBAL.BOX_SIZE // 2)  # Adjust the size of the box as needed
-                        box_color = (0, 255, 0)  # Green color
-
-                        # Coordinates of the rectangle
-                        x1, y1 = tip_x - box_size, tip_y - box_size
-                        x2, y2 = tip_x + box_size, tip_y + box_size
-
-                        # Draw a square box around the finger tip
-                        cv2.rectangle(image, (x1, y1), (x2, y2), box_color, 2)
-
-                        # Crop the region of interest (ROI)
-                        finger_tracking_frame = copy_image[y1:y2, x1:x2]
-                        color = (0, 0, 255)
-                        if finger_tracking_frame is not None and finger_tracking_frame.shape[0] > 0 and finger_tracking_frame.shape[1] > 0:
-                            finger_tracking_frame = cv2.cvtColor(finger_tracking_frame , cv2.COLOR_BGR2RGB)
-                            pred = model.Predict(finger_tracking_frame)
-                            if pred:
-                                color=(0, 255, 0)
-                            else:
-                                if color == (0, 255, 0):
-                                    pass
-                                else:
-                                    color = (0, 0, 255)
-                            image = cv2.circle(image, (250 , 300), 2, color, 20)
-            cv2.imshow('Tocuh tracking', image)            
-            key = cv2.waitKey(5)
-            if key == ord('q'):
-                cap.release()
-                cv2.destroyAllWindows()
-                break
+INDEX_FINGER = 8
+MIDDLE_FINGER = 12
+RING_FINGER = 16
+PINKY_FINGER = 20
+FINGER = [INDEX_FINGER]
+UNTOUCH_FOLDER = "src/training_data/untouched"
+TOUCH_FOLDER = "src/training_data/touched"
 
 
 def fetch_train_data():
-    print("The First window is try window so that you can adjust the finger position, adjust hand position so that box will cover  finger tip and finger tip\n1. Window after try will be touch train window\n\t do not lift any finger, move fingers slowly on the paper to get all angles\n2. After this window untouch train window will pop up\n\t lift fingers so that it can take pics of finger tips for training\n\t Then model will be trained and you should see the prediction window for Index finger")
+    print(
+        "The First window is try window so that you can adjust the finger position, adjust hand position so that box will cover  finger tip and finger tip\n1. Window after try will be touch train window\n\t do not lift any finger, move fingers slowly on the paper to get all angles\n2. After this window untouch train window will pop up\n\t lift fingers so that it can take pics of finger tips for training\n\t Then model will be trained and you should see the prediction window for Index finger")
     print("Press Y to move for training stage")
     while 1:
         key = input(">> ")
         if key.lower() == 'y':
             break
     time.sleep(2)
-    fetch_data.Try(FINGER) 
+    fetch_data.Try(FINGER)
     time.sleep(2)
-    fetch_data.Capture(GLOBAL.TOUCH_FOLDER , "touched" , FINGER) 
+    fetch_data.Capture(TOUCH_FOLDER, "touched", FINGER)
     time.sleep(2)
-    fetch_data.Capture(GLOBAL.UNTOUCH_FOLDER , "untouched" , FINGER) 
+    fetch_data.Capture(UNTOUCH_FOLDER, "untouched", FINGER)
 
-    train_model.start_training() 
+    train_model.start_training()
 
     print("Model Training Complete")
     time.sleep(3)
 
-    RUN(GLOBAL.INDEX_FINGER)
 
-print("welcome to Paper Piano")
-# fetch_data.delete_model()
 run = True
 
-fetch_data.clear_training_data()
-
 while run:
     model_list = os.listdir("models")
-    if "touch_detection_model.h5" not in model_list:
-        print("We need to train model on your finger's data")
+    if "touch_detection_model.keras" not in model_list:
+        # print("We need to train model on your finger's data")
         fetch_data.clear_training_data()
         fetch_train_data()
-        
+
     else:
 
         print("-------------*MENU*-------------\n[1] Retrain model\n[2] Start Paper Piano\n[3] Exit")
@@ -121,7 +55,8 @@ def fetch_train_data():
                 check = False
                 print("Adjust paper accordingly until you see mesh of keys and press 'q'")
                 time.sleep(3)
-                piano.start_piano(GLOBAL.INDEX_FINGER)
-            elif opt==3:
+                piano.start_piano(INDEX_FINGER)
+            elif opt == 3:
                 check = False
                 run = False
+