diff --git a/README.md b/README.md index a5c2402..85896ba 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,8 @@ The following .gif video shows pictures where the ground plane conditions color And once it finishes (note the scene does not evolve anymore) check the generated folder under `isaac_ws/datasets/YYYYMMDDHHMMSS_out_fruit_sdg` where `YYYYMMDDHHMMSS` is the stamp of the dataset creation. +Typically 300 images are not enough. For a quick iteration it is recommended to go with 300 images to validate everything works as expected. When running the system for training a model more seriously you can go up to five thousands and spend between 15 and 20 minutes in doing so. You can quickly change the number of images to generate by editing `./isaac_ws/simulation_ws/conf/config.yml` and look for the `NUM_FRAMES` item. + ## Training the model To train a model you need a NVidia Omniverse synthetic dataset built in the previous step. You first need to set up the following environment variable: @@ -206,13 +208,18 @@ docker compose -f docker/docker-compose.yml --profile simulated_pipeline up docker compose -f docker/docker-compose.yml --profile simulated_pipeline down ``` +![Simulated pipeline](./doc/simulated_pipeline.gif) + ### Parameter tuning -The following detection node parameters are exposed and can be modified via rqt_gui when running any pipeline: +The following detection node parameters are exposed and can be modified via rqt_gui when running any pipeline. Please note that values set outside of their range will simply be discarded. + -#### Minimum bounding box +![Parameter tunning](./doc/parameter_tunning.png) -The two parameters `bbox_min_x` and `bbox_min_y` are both measured in pixels. `bbox_min_x` can take values between 0 and 640, and `bbox_min_y` can take values between 0 and 480. They can be used to filter the inferences based on the size of the bounding boxes generated. +#### Bounding box + +The parameters `bbox_min_x`, `bbox_min_y`, `bbox_max_x`, `bbox_max_y` are measured in pixels. `bbox_min_x` and `bbox_max_x` can take values between 0 and 640. `bbox_min_y` and `bbox_max_y` can take values between 0 and 480. They can be used to filter the inferences based on the size of the bounding boxes generated. #### Score threshold @@ -291,3 +298,7 @@ We faced some situations in which precedence of access to the GPU yields to race - Close all Google Chrome related processes. - Try to open the simulator using one of the provided instructions in the readme. - Verify that you can open the simulator, otherwise, perhaps you need to reboot your system :/ and try again. + +4. Detection calibration + +You may need to further calibrate the detection node post-training. This is usually done considering the ambient conditions. We offer two parameters via dynamic reconfigure to be used. Refer to the "Parameter tuning" section for further details. diff --git a/detection_ws/src/fruit_detection/config/params.yml b/detection_ws/src/fruit_detection/config/params.yml index 7c75060..a6b8182 100644 --- a/detection_ws/src/fruit_detection/config/params.yml +++ b/detection_ws/src/fruit_detection/config/params.yml @@ -6,4 +6,6 @@ olive_camera_topic: "/olive/camera/id01/image/compressed" bbox_min_x: 60 bbox_min_y: 60 + bbox_max_x: 150 + bbox_max_y: 150 score_threshold: 0.90 diff --git a/detection_ws/src/fruit_detection/fruit_detection/fruit_detection_node.py b/detection_ws/src/fruit_detection/fruit_detection/fruit_detection_node.py index b1f35d3..2ddad8d 100644 --- a/detection_ws/src/fruit_detection/fruit_detection/fruit_detection_node.py +++ b/detection_ws/src/fruit_detection/fruit_detection/fruit_detection_node.py @@ -100,8 +100,9 @@ def __init__(self) -> None: ) self.declare_parameter("bbox_min_x", 60) self.declare_parameter("bbox_min_y", 60) + self.declare_parameter("bbox_max_x", 200) + self.declare_parameter("bbox_max_y", 200) self.declare_parameter("score_threshold", 0.9) - self.add_on_set_parameters_callback(self.validate_parameters) self.__model_path = ( @@ -154,11 +155,16 @@ def validate_parameters(self, params): """ Validate parameter changes. - :param params: list of parameters. + Args: + ----- + params (list[Parameter]): list of parameters to analyze. - :return: SetParametersResult. + Returns: + -------- + SetParametersResult with the result of the analysis """ - parameters_are_valid = True + is_valid = True + reason = "" for param in params: if param.name == "bbox_min_x": if param.type_ != Parameter.Type.INTEGER or not ( @@ -166,25 +172,76 @@ def validate_parameters(self, params): <= param.value <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_X ): - parameters_are_valid = False + is_valid = False + reason = ( + "bbox_min_x: is not in range [" + f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_X}; " + f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_X}]" + ) break + else: + reason = "bbox_min_x successfully set." elif param.name == "bbox_min_y": if param.type_ != Parameter.Type.INTEGER or not ( FruitDetectionNode.MINIMUM_BBOX_SIZE_Y <= param.value <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y ): - parameters_are_valid = False + is_valid = False + reason = ( + "bbox_min_y: is not in range [" + f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_Y}; " + f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y}]" + ) + break + else: + reason = "bbox_min_y successfully set." + if param.name == "bbox_max_x": + if param.type_ != Parameter.Type.INTEGER or not ( + FruitDetectionNode.MINIMUM_BBOX_SIZE_X + <= param.value + <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_X + ): + is_valid = False + reason = ( + "bbox_max_x: is not in range [" + f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_X}; " + f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_X}]" + ) + break + else: + reason = "bbox_max_x successfully set." + elif param.name == "bbox_max_y": + if param.type_ != Parameter.Type.INTEGER or not ( + FruitDetectionNode.MINIMUM_BBOX_SIZE_Y + <= param.value + <= FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y + ): + is_valid = False + reason = ( + "bbox_max_y: is not in range [" + f"{FruitDetectionNode.MINIMUM_BBOX_SIZE_Y}; " + f"{FruitDetectionNode.MAXIMUM_BBOX_SIZE_Y}]" + ) break + else: + reason = "bbox_max_y successfully set." elif param.name == "score_threshold": if param.type_ != Parameter.Type.DOUBLE or not ( FruitDetectionNode.MINIMUM_SCORE_THRESHOLD <= param.value <= FruitDetectionNode.MAXIMUM_SCORE_THRESHOLD ): - parameters_are_valid = False + is_valid = False + reason = ( + "score_threshold: is not in range [" + f"{FruitDetectionNode.MINIMUM_SCORE_THRESHOLD}; " + f"{FruitDetectionNode.MAXIMUM_SCORE_THRESHOLD}]" + ) break - return SetParametersResult(successful=parameters_are_valid) + if not is_valid: + self.get_logger().warn(f"Skipping to set parameter: {reason}") + return SetParametersResult(successful=is_valid, reason=reason) def load_model(self): """Load the torch model.""" @@ -212,18 +269,26 @@ def cv2_to_torch_frame(self, img): """Prepare cv2 image for inference.""" return self.image_to_tensor(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) - def bbox_has_minimum_size(self, box, min_x, min_y): + def bbox_is_in_range(self, box, min_x, min_y, max_x, max_y): """ - Check if a box is bigger than a minimum size. + Check if a box is within size. - :param box: bounding box from inference. - :param min_x: minimum horizontal length of the bounding box. - :param min_y: minimum vertical length of the bounding box. + Args: + ----- + box (tuple[int, int, int, int]): bounding box from inference. + min_x (int): minimum horizontal length of the bounding box. + min_y (int): minimum vertical length of the bounding box. + max_x (int): maximum horizontal length of the bounding box. + max_y (int): maximum vertical length of the bounding box. - :return: True if the box has the minimum size. + Returns: + -------- + bool True if the box size is in range. """ x1, y1, x2, y2 = int(box[0]), int(box[1]), int(box[2]), int(box[3]) - return (min_x <= (x2 - x1)) and (min_y <= (y2 - y1)) + dx = x2 - x1 + dy = y2 - y1 + return (min_x <= dx <= max_x) and (min_y <= dy <= max_y) def score_frame(self, frame): """ @@ -255,13 +320,23 @@ def score_frame(self, frame): .get_parameter_value() .integer_value # Minimum bbox y size ) + bbox_max_x = ( + self.get_parameter("bbox_max_x") + .get_parameter_value() + .integer_value # Maximum bbox x size + ) + bbox_max_y = ( + self.get_parameter("bbox_max_y") + .get_parameter_value() + .integer_value # Maximum bbox y size + ) score_threshold = ( self.get_parameter("score_threshold") .get_parameter_value() .double_value # Score threshold ) - if score >= score_threshold and self.bbox_has_minimum_size( - box, bbox_min_x, bbox_min_y + if score >= score_threshold and self.bbox_is_in_range( + box, bbox_min_x, bbox_min_y, bbox_max_x, bbox_max_y ): results.append( { diff --git a/doc/dataset_gen.gif b/doc/dataset_gen.gif index f96e1ed..6f14313 100644 Binary files a/doc/dataset_gen.gif and b/doc/dataset_gen.gif differ diff --git a/doc/parameter_tunning.png b/doc/parameter_tunning.png new file mode 100644 index 0000000..5a49dee Binary files /dev/null and b/doc/parameter_tunning.png differ diff --git a/doc/simulated_pipeline.gif b/doc/simulated_pipeline.gif new file mode 100644 index 0000000..50026b3 Binary files /dev/null and b/doc/simulated_pipeline.gif differ diff --git a/isaac_ws/simulation_ws/scene/scene.usda b/isaac_ws/simulation_ws/scene/scene.usda index 55a7ced..7b34980 100644 --- a/isaac_ws/simulation_ws/scene/scene.usda +++ b/isaac_ws/simulation_ws/scene/scene.usda @@ -145,7 +145,7 @@ def Xform "World" { quatf xformOp:orient = (1, 0, 0, 0) float3 xformOp:scale = (1, 1, 1) - double3 xformOp:scale:unitsResolve = (0.02, 0.02, 0.02) + double3 xformOp:scale:unitsResolve = (0.015, 0.015, 0.015) double3 xformOp:translate = (-0.1, -0.05, 0) uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale", "xformOp:scale:unitsResolve"] } @@ -156,7 +156,7 @@ def Xform "World" { quatf xformOp:orient = (1, 0, 0, 0) float3 xformOp:scale = (1, 1, 1) - double3 xformOp:scale:unitsResolve = (0.02, 0.02, 0.02) + double3 xformOp:scale:unitsResolve = (0.015, 0.015, 0.015) double3 xformOp:translate = (0, 0.1, 0) uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale", "xformOp:scale:unitsResolve"] } @@ -167,7 +167,7 @@ def Xform "World" { quatf xformOp:orient = (1, 0, 0, 0) float3 xformOp:scale = (1, 1, 1) - double3 xformOp:scale:unitsResolve = (0.02, 0.02, 0.02) + double3 xformOp:scale:unitsResolve = (0.015, 0.015, 0.015) double3 xformOp:translate = (0.1, -0.05, 0) uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale", "xformOp:scale:unitsResolve"] } @@ -384,14 +384,14 @@ def Xform "Environment" ) { float inputs:angle = 1 - float inputs:intensity = 3000 + float inputs:intensity = 2000 float inputs:shaping:cone:angle = 180 float inputs:shaping:cone:softness float inputs:shaping:focus color3f inputs:shaping:focusTint asset inputs:shaping:ies:file token visibility = "inherited" - quatd xformOp:orient = (0.6532814824381883, 0.2705980500730985, 0.27059805007309845, 0.6532814824381882) + quatd xformOp:orient = (0.7071068, 0.0, 0.0, 0.7071068) double3 xformOp:scale = (1, 1, 1) double3 xformOp:translate = (0, 0, 0) uniform token[] xformOpOrder = ["xformOp:translate", "xformOp:orient", "xformOp:scale"] diff --git a/training_ws/train.py b/training_ws/train.py index f3a9f6d..51fcb29 100644 --- a/training_ws/train.py +++ b/training_ws/train.py @@ -221,7 +221,7 @@ def create_model(num_classes): # Constants NUM_EPOCHS = 5 TRAINING_PARTITION_RATIO = 0.7 -OPTIMIZER_LR = 0.001 +OPTIMIZER_LR = 0.00001 def main():