Final integration tweaks (#62)

* Updates the gif with the dataset generation.
* Explain how to change the number of frames to generate.
* Updates the learning ratio.
* Includes a bounding box maximum for the final calibration.
* Improves fruit scale and lighting conditions.
* Updates the README with the simulated pipeline gif.

---------

Signed-off-by: Agustin Alba Chicar <[email protected]>

README.md

@@ -112,6 +112,8 @@ The following .gif video shows pictures where the ground plane conditions color
 
 And once it finishes (note the scene does not evolve anymore) check the generated folder under `isaac_ws/datasets/YYYYMMDDHHMMSS_out_fruit_sdg` where `YYYYMMDDHHMMSS` is the stamp of the dataset creation.
 
+Typically 300 images are not enough. For a quick iteration it is recommended to go with 300 images to validate everything works as expected. When running the system for training a model more seriously you can go up to five thousands and spend between 15 and 20 minutes in doing so. You can quickly change the number of images to generate by editing `./isaac_ws/simulation_ws/conf/config.yml` and look for the `NUM_FRAMES` item.
+
 ## Training the model
 
 To train a model you need a NVidia Omniverse synthetic dataset built in the previous step. You first need to set up the following environment variable:
@@ -206,13 +208,18 @@ docker compose -f docker/docker-compose.yml --profile simulated_pipeline up
 docker compose -f docker/docker-compose.yml --profile simulated_pipeline down
 ```
 
+![Simulated pipeline](./doc/simulated_pipeline.gif)
+
 ### Parameter tuning
 
-The following detection node parameters are exposed and can be modified via rqt_gui when running any pipeline:
+The following detection node parameters are exposed and can be modified via rqt_gui when running any pipeline. Please note that values set outside of their range will simply be discarded.
+
 
-#### Minimum bounding box
+![Parameter tunning](./doc/parameter_tunning.png)
 
-The two parameters `bbox_min_x` and `bbox_min_y` are both measured in pixels. `bbox_min_x` can take values between 0 and 640, and `bbox_min_y` can take values between 0 and 480. They can be used to filter the inferences based on the size of the bounding boxes generated.
+#### Bounding box
+
+The parameters `bbox_min_x`, `bbox_min_y`, `bbox_max_x`, `bbox_max_y` are measured in pixels. `bbox_min_x` and `bbox_max_x` can take values between 0 and 640. `bbox_min_y` and `bbox_max_y` can take values between 0 and 480. They can be used to filter the inferences based on the size of the bounding boxes generated.
 
 #### Score threshold
 
@@ -291,3 +298,7 @@ We faced some situations in which precedence of access to the GPU yields to race
 - Close all Google Chrome related processes.
 - Try to open the simulator using one of the provided instructions in the readme.
 - Verify that you can open the simulator, otherwise, perhaps you need to reboot your system :/ and try again.
+
+4. Detection calibration
+
+You may need to further calibrate the detection node post-training. This is usually done considering the ambient conditions. We offer two parameters via dynamic reconfigure to be used. Refer to the "Parameter tuning" section for further details.

detection_ws/src/fruit_detection/config/params.yml

@@ -6,4 +6,6 @@
     olive_camera_topic: "/olive/camera/id01/image/compressed"
     bbox_min_x: 60
     bbox_min_y: 60
+    bbox_max_x: 150
+    bbox_max_y: 150
     score_threshold: 0.90

detection_ws/src/fruit_detection/fruit_detection/fruit_detection_node.py

@@ -100,8 +100,9 @@ def __init__(self) -> None:
         )
         self.declare_parameter("bbox_min_x", 60)
         self.declare_parameter("bbox_min_y", 60)
+        self.declare_parameter("bbox_max_x", 200)
+        self.declare_parameter("bbox_max_y", 200)
         self.declare_parameter("score_threshold", 0.9)
-
         self.add_on_set_parameters_callback(self.validate_parameters)
 
         self.__model_path = (
@@ -154,37 +155,93 @@ def validate_parameters(self, params):
         """
         Validate parameter changes.
 
-        :param params: list of parameters.
+        Args:
+        -----
+            params (list[Parameter]): list of parameters to analyze.
 
-        :return: SetParametersResult.
+        Returns:
+        --------
+            SetParametersResult with the result of the analysis
         """
-        parameters_are_valid = True
+        is_valid = True
+        reason = ""
         for param in params:
             if param.name == "bbox_min_x":
                 if param.type_ != Parameter.Type.INTEGER or not (
                     FruitDetectionNode.MINIMUM_BBOX_SIZE_X
                     <= param.value
                     <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_X
                 ):
-                    parameters_are_valid = False
+                    is_valid = False
+                    reason = (
+                        "bbox_min_x: is not in range ["
+                        f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_X}; "
+                        f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_X}]"
+                    )
                     break
+                else:
+                    reason = "bbox_min_x successfully set."
             elif param.name == "bbox_min_y":
                 if param.type_ != Parameter.Type.INTEGER or not (
                     FruitDetectionNode.MINIMUM_BBOX_SIZE_Y
                     <= param.value
                     <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y
                 ):
-                    parameters_are_valid = False
+                    is_valid = False
+                    reason = (
+                        "bbox_min_y: is not in range ["
+                        f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_Y}; "
+                        f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y}]"
+                    )
+                    break
+                else:
+                    reason = "bbox_min_y successfully set."
+            if param.name == "bbox_max_x":
+                if param.type_ != Parameter.Type.INTEGER or not (
+                    FruitDetectionNode.MINIMUM_BBOX_SIZE_X
+                    <= param.value
+                    <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_X
+                ):
+                    is_valid = False
+                    reason = (
+                        "bbox_max_x: is not in range ["
+                        f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_X}; "
+                        f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_X}]"
+                    )
+                    break
+                else:
+                    reason = "bbox_max_x successfully set."
+            elif param.name == "bbox_max_y":
+                if param.type_ != Parameter.Type.INTEGER or not (
+                    FruitDetectionNode.MINIMUM_BBOX_SIZE_Y
+                    <= param.value
+                    <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y
+                ):
+                    is_valid = False
+                    reason = (
+                        "bbox_max_y: is not in range ["
+                        f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_Y}; "
+                        f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y}]"
+                    )
                     break
+                else:
+                    reason = "bbox_max_y successfully set."
             elif param.name == "score_threshold":
                 if param.type_ != Parameter.Type.DOUBLE or not (
                     FruitDetectionNode.MINIMUM_SCORE_THRESHOLD
                     <= param.value
                     <= FruitDetectionNode.MAXIMUM_SCORE_THRESHOLD
                 ):
-                    parameters_are_valid = False
+                    is_valid = False
+                    reason = (
+                        "score_threshold: is not in range ["
+                        f"{FruitDetectionNode.MINIMUM_SCORE_THRESHOLD}; "
+                        f"{FruitDetectionNode.MAXIMUM_SCORE_THRESHOLD}]"
+                    )
                     break
-        return SetParametersResult(successful=parameters_are_valid)
+        if not is_valid:
+            self.get_logger().warn(f"Skipping to set parameter: {reason}")
+        return SetParametersResult(successful=is_valid, reason=reason)
 
     def load_model(self):
         """Load the torch model."""
@@ -212,18 +269,26 @@ def cv2_to_torch_frame(self, img):
         """Prepare cv2 image for inference."""
         return self.image_to_tensor(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
 
-    def bbox_has_minimum_size(self, box, min_x, min_y):
+    def bbox_is_in_range(self, box, min_x, min_y, max_x, max_y):
         """
-        Check if a box is bigger than a minimum size.
+        Check if a box is within size.
 
-        :param box: bounding box from inference.
-        :param min_x: minimum horizontal length of the bounding box.
-        :param min_y: minimum vertical length of the bounding box.
+        Args:
+        -----
+            box (tuple[int, int, int, int]): bounding box from inference.
+            min_x (int): minimum horizontal length of the bounding box.
+            min_y (int): minimum vertical length of the bounding box.
+            max_x (int): maximum horizontal length of the bounding box.
+            max_y (int): maximum vertical length of the bounding box.
 
-        :return: True if the box has the minimum size.
+        Returns:
+        --------
+            bool True if the box size is in range.
         """
         x1, y1, x2, y2 = int(box[0]), int(box[1]), int(box[2]), int(box[3])
-        return (min_x <= (x2 - x1)) and (min_y <= (y2 - y1))
+        dx = x2 - x1
+        dy = y2 - y1
+        return (min_x <= dx <= max_x) and (min_y <= dy <= max_y)
 
     def score_frame(self, frame):
         """
@@ -255,13 +320,23 @@ def score_frame(self, frame):
                     .get_parameter_value()
                     .integer_value  # Minimum bbox y size
                 )
+                bbox_max_x = (
+                    self.get_parameter("bbox_max_x")
+                    .get_parameter_value()
+                    .integer_value  # Maximum bbox x size
+                )
+                bbox_max_y = (
+                    self.get_parameter("bbox_max_y")
+                    .get_parameter_value()
+                    .integer_value  # Maximum bbox y size
+                )
                 score_threshold = (
                     self.get_parameter("score_threshold")
                     .get_parameter_value()
                     .double_value  # Score threshold
                 )
-                if score >= score_threshold and self.bbox_has_minimum_size(
-                    box, bbox_min_x, bbox_min_y
+                if score >= score_threshold and self.bbox_is_in_range(
+                    box, bbox_min_x, bbox_min_y, bbox_max_x, bbox_max_y
                 ):
                     results.append(
                         {

isaac_ws/simulation_ws/scene/scene.usda

@@ -145,7 +145,7 @@ def Xform "World"
     {
         quatf xformOp:orient = (1, 0, 0, 0)
         float3 xformOp:scale = (1, 1, 1)
-        double3 xformOp:scale:unitsResolve = (0.02, 0.02, 0.02)
+        double3 xformOp:scale:unitsResolve = (0.015, 0.015, 0.015)
         double3 xformOp:translate = (-0.1, -0.05, 0)
         uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale", "xformOp:scale:unitsResolve"]
     }
@@ -156,7 +156,7 @@ def Xform "World"
     {
         quatf xformOp:orient = (1, 0, 0, 0)
         float3 xformOp:scale = (1, 1, 1)
-        double3 xformOp:scale:unitsResolve = (0.02, 0.02, 0.02)
+        double3 xformOp:scale:unitsResolve = (0.015, 0.015, 0.015)
         double3 xformOp:translate = (0, 0.1, 0)
         uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale", "xformOp:scale:unitsResolve"]
     }
@@ -167,7 +167,7 @@ def Xform "World"
     {
         quatf xformOp:orient = (1, 0, 0, 0)
         float3 xformOp:scale = (1, 1, 1)
-        double3 xformOp:scale:unitsResolve = (0.02, 0.02, 0.02)
+        double3 xformOp:scale:unitsResolve = (0.015, 0.015, 0.015)
         double3 xformOp:translate = (0.1, -0.05, 0)
         uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale", "xformOp:scale:unitsResolve"]
     }
@@ -384,14 +384,14 @@ def Xform "Environment"
     )
     {
         float inputs:angle = 1
-        float inputs:intensity = 3000
+        float inputs:intensity = 2000
         float inputs:shaping:cone:angle = 180
         float inputs:shaping:cone:softness
         float inputs:shaping:focus
         color3f inputs:shaping:focusTint
         asset inputs:shaping:ies:file
         token visibility = "inherited"
-        quatd xformOp:orient = (0.6532814824381883, 0.2705980500730985, 0.27059805007309845, 0.6532814824381882)
+        quatd xformOp:orient = (0.7071068, 0.0, 0.0, 0.7071068)
         double3 xformOp:scale = (1, 1, 1)
         double3 xformOp:translate = (0, 0, 0)
         uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale"]

training_ws/train.py

@@ -221,7 +221,7 @@ def create_model(num_classes):
 # Constants
 NUM_EPOCHS = 5
 TRAINING_PARTITION_RATIO = 0.7
-OPTIMIZER_LR = 0.001
+OPTIMIZER_LR = 0.00001
 
 
 def main():