From ffa9cd9ba5b1a841a9f338ea8e3fa3a16f568f2f Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 19:58:28 +0200 Subject: [PATCH 1/9] [perception] added mir_perception_models --- mir_perception/mir_object_recognition/package.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/mir_perception/mir_object_recognition/package.xml b/mir_perception/mir_object_recognition/package.xml index 6ae751bf9..f209b3c81 100644 --- a/mir_perception/mir_object_recognition/package.xml +++ b/mir_perception/mir_object_recognition/package.xml @@ -23,6 +23,7 @@ roslint mir_object_segmentation mir_perception_utils + mir_perception_models cv_bridge geometry_msgs From 7f7afb72eb681770d9774532fea506f76463fbe1 Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 20:11:00 +0200 Subject: [PATCH 2/9] [perception/recognition] added pointcloud source frame id as arg --- .../ros/launch/multimodal_object_recognition.launch | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mir_perception/mir_object_recognition/ros/launch/multimodal_object_recognition.launch b/mir_perception/mir_object_recognition/ros/launch/multimodal_object_recognition.launch index e9260ff66..07c0a7604 100644 --- a/mir_perception/mir_object_recognition/ros/launch/multimodal_object_recognition.launch +++ b/mir_perception/mir_object_recognition/ros/launch/multimodal_object_recognition.launch @@ -4,6 +4,8 @@ + + @@ -18,6 +20,7 @@ + From 8bda8350dd00c18a22999cd11df5d2d5e9b4cb2c Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 20:13:23 +0200 Subject: [PATCH 3/9] [perception/cloud_recognition] use model dir from mir_perception_models --- .../ros/launch/pc_object_recognition.launch | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mir_perception/mir_object_recognition/ros/launch/pc_object_recognition.launch b/mir_perception/mir_object_recognition/ros/launch/pc_object_recognition.launch index 785cb4a48..125d5966f 100644 --- a/mir_perception/mir_object_recognition/ros/launch/pc_object_recognition.launch +++ b/mir_perception/mir_object_recognition/ros/launch/pc_object_recognition.launch @@ -5,13 +5,15 @@ - + + + From 2e6cb8846990b4102151890cd874ca7b01f0101f Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 20:15:07 +0200 Subject: [PATCH 4/9] [perception/2d_recognition] use model from mir_perception_models --- .../ros/launch/rgb_object_recognition.launch | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/mir_perception/mir_object_recognition/ros/launch/rgb_object_recognition.launch b/mir_perception/mir_object_recognition/ros/launch/rgb_object_recognition.launch index c1320c839..f7ccb2f61 100644 --- a/mir_perception/mir_object_recognition/ros/launch/rgb_object_recognition.launch +++ b/mir_perception/mir_object_recognition/ros/launch/rgb_object_recognition.launch @@ -2,14 +2,16 @@ - + + - + + From ccfd05a79636e07d53f2ef8ae5530e1a9bf050a4 Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 20:17:51 +0200 Subject: [PATCH 5/9] [perception/2d_recognition] updated rosparam using new model dir from mir_perception_models --- .../ros/script/rgb_object_recognizer_node | 78 ++----------------- 1 file changed, 5 insertions(+), 73 deletions(-) diff --git a/mir_perception/mir_object_recognition/ros/script/rgb_object_recognizer_node b/mir_perception/mir_object_recognition/ros/script/rgb_object_recognizer_node index 977fc07bc..b5ce821a4 100755 --- a/mir_perception/mir_object_recognition/ros/script/rgb_object_recognizer_node +++ b/mir_perception/mir_object_recognition/ros/script/rgb_object_recognizer_node @@ -1,7 +1,5 @@ #!/usr/bin/env python -PACKAGE = "mir_object_recognition" - import colorsys import os import pickle @@ -31,7 +29,7 @@ class RGBObjectRecognizer(): self.net = net self.model_name = model_name - config_file = os.path.join(roslib.packages.get_pkg_dir(PACKAGE), + config_file = os.path.join(roslib.packages.get_pkg_dir("mir_object_recognition"), 'ros', 'config', "rgb_classifier_config.yaml") if os.path.isfile(config_file): @@ -76,12 +74,6 @@ class RGBObjectRecognizer(): result.roi = roi objects.append(result) - # remove overlapping bboxes - idxs_to_remove, objects = self.remove_overlapping_bbox(objects) - bboxes = [bbox for i,bbox in enumerate(bboxes) if i not in idxs_to_remove] - probs = [prob for i,prob in enumerate(probs) if i not in idxs_to_remove] - labels = [label for i,label in enumerate(labels) if i not in idxs_to_remove] - # Publish result_list result_list.objects = objects self.pub_result.publish(result_list) @@ -104,66 +96,6 @@ class RGBObjectRecognizer(): elif self.net == 'classification': print "TODO: MobileNet" - - def remove_overlapping_bbox(self, objects): - """ - Remove object which has bigger intersection areas - Args: - objects: object_list - Returns: - indices: object indices to remove - filtered_objects: objects filtered - """ - object_to_remove_idxs = [] - for i,object_1 in enumerate(objects): - for j,object_2 in enumerate(objects): - if i != j: - bbox1 = [object_1.roi.x_offset,object_1.roi.y_offset, - object_1.roi.width,object_1.roi.height] - bbox2 = [object_2.roi.x_offset,object_2.roi.y_offset, - object_2.roi.width,object_2.roi.height] - - intersection = float(self.compute_intersection(bbox1, bbox2)) - if intersection > 1e-2: - intersection_over_area1 = intersection / (object_1.roi.height * object_1.roi.width) - intersection_over_area2 = intersection / (object_2.roi.height * object_2.roi.width) - - if intersection_over_area1 >= intersection_over_area2: - if i not in object_to_remove_idxs: - object_to_remove_idxs.append(i) - else: - if i not in object_to_remove_idxs: - object_to_remove_idxs.append(j) - - filtered_objects = [] - for i,obj in enumerate(objects): - if i not in object_to_remove_idxs: - filtered_objects.append(obj) - - return object_to_remove_idxs, filtered_objects - - def compute_intersection(self, box1, box2): - """ - Compute iou - Args: - box1: array of 4 elements [cx, cy, width, height]. - box2: same as above - Returns: - iou: a float number in range [0, 1]. iou of the two boxes. - """ - - lr = min(box1[0]+box1[2], box2[0]+box2[2]) - \ - max(box1[0], box2[0]) - if lr > 0: - tb = min(box1[1]+box1[3], box2[1]+box2[3]) - \ - max(box1[1], box2[1]) - if tb > 0: - intersection = tb*lr - union = box1[2]*box1[3]+box2[2]*box2[3]-intersection - - return float(intersection) - - return 0 def publish_debug_img(self, debug_img): debug_img = np.array(debug_img, dtype=np.uint8) @@ -175,9 +107,9 @@ if __name__ == '__main__': rospy.loginfo('Started object recognition node.') net = rospy.get_param("~net") classifier_name = rospy.get_param("~classifier") - dataset_type = rospy.get_param("~dataset_type") - dataset_dir = os.path.join(roslib.packages.get_pkg_dir(PACKAGE), 'common', 'config', classifier_name, dataset_type) + dataset = rospy.get_param("~dataset") + model_dir = rospy.get_param("~model_dir") - object_recognizer = RGBObjectRecognizer(model_dir=dataset_dir, net=net, model_name=classifier_name, debug_mode=True) - rospy.loginfo('\033[92m'+"RGB Recognizer is ready using %s : %s , dataset: %s ", net, classifier_name, dataset_type) + object_recognizer = RGBObjectRecognizer(model_dir=model_dir, net=net, model_name=classifier_name, debug_mode=True) + rospy.loginfo('\033[92m'+"RGB Recognizer is ready using %s : %s , dataset: %s ", net, classifier_name, dataset) rospy.spin() From aa7e76ab0764340aefb96527bdb3ce8304bb7296 Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 20:18:46 +0200 Subject: [PATCH 6/9] [perception/3d_recognition] updated model dir using mir_perception_models --- .../ros/script/pc_object_recognizer_node | 47 ++++++------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/mir_perception/mir_object_recognition/ros/script/pc_object_recognizer_node b/mir_perception/mir_object_recognition/ros/script/pc_object_recognizer_node index d33d2ed23..e045e6fba 100755 --- a/mir_perception/mir_object_recognition/ros/script/pc_object_recognizer_node +++ b/mir_perception/mir_object_recognition/ros/script/pc_object_recognizer_node @@ -1,8 +1,5 @@ #!/usr/bin/env python -PACKAGE = 'mir_object_recognition' -NODE = 'pc_object_recognizer' - import colorsys import importlib import os @@ -33,17 +30,13 @@ class PointcloudObjectRecognizer(): :param variance: Variance of gmm model for FV """ - def __init__(self, model, model_id, dataset, ngaussians=2, variance=0.05): + def __init__(self, model, model_id, dataset, model_dir, ngaussians=2, variance=0.05): self.model = model dataset_dir = model_id + "_" + dataset - cfg_folder = os.path.join(roslib.packages.get_pkg_dir(PACKAGE), 'common', 'config') + #cfg_folder = os.path.join(roslib.packages.get_pkg_dir("mir_object_recognition"), 'common', 'config') - model_config_file = os.path.join(roslib.packages.get_pkg_dir(PACKAGE), + model_config_file = os.path.join(roslib.packages.get_pkg_dir("mir_object_recognition"), 'ros', 'config', "pc_classifier_config.yaml") - - #threshold for alu profile color - #black < 0.3, gray >= 03 - self.alu_profile_color_th = 0.3 if os.path.isfile(model_config_file): configs = {} @@ -66,12 +59,12 @@ class PointcloudObjectRecognizer(): elif model_id == "rdd": feature_extraction.set_rdd_params(color=True) - classifier_file = os.path.join(cfg_folder, dataset_dir, 'classifier.joblib') - label_encoder = os.path.join(cfg_folder, dataset_dir, 'label_encoder.joblib') + classifier_file = os.path.join(model_dir, 'classifier.pkl') + label_encoder = os.path.join(model_dir, 'label_encoder.pkl') self.classifier = FeatureBasedClassifiers(classifier_file, label_encoder) elif model == "cnn_based": - checkpoint = os.path.join(cfg_folder, dataset_dir , 'model.ckpt') + checkpoint = os.path.join(model_dir , 'model.ckpt') module_name = "pc_object_recognition" + "." + model_id.lower()+"_"+"classifier" class_name = model_id.upper()+"Classifier" ClassifierClass = getattr(importlib.import_module(module_name), class_name) @@ -95,33 +88,19 @@ class PointcloudObjectRecognizer(): recognized_object_list = object_list.objects for object in recognized_object_list: if self.model == "feature_based": - cloud = self.extract_pointcloud(object.views[0].point_cloud, color="rgb") + cloud = self.extract_pointcloud(object.views[0].point_cloud, color="hsv") features = self.fe_method(cloud) features = np.reshape(features, (1,-1)) name, probability = self.classifier.classify(features) - name = self.label_to_name[name[0]] + name = name[0] elif self.model == "cnn_based": cloud = self.extract_pointcloud(object.views[0].point_cloud, color="rgb") label, probability = self.classifier.classify(cloud, center=True, rotate=True, pad=True) name = self.label_to_name[label] - - # check if it's profile, replace the recognition result if - # it's misclassified with color according to threshold - if "S40_40" in name or "F20_20" in name: - median_color = np.median(cloud[:,3:], axis=0) - mean_median_color = np.mean(median_color) - rospy.loginfo("Checking color for %s: mean_median_color= %s", name, mean_median_color) - - if mean_median_color < self.alu_profile_color_th and "G" in name: - new_label = name.replace("G", "B", 1) - name = new_label - elif mean_median_color > self.alu_profile_color_th and "B" in name: - new_label = name.replace("B", "G", 1) - name = new_label object.name = name object.probability = probability - + self.pub.publish(recognized_object_list) def extract_pointcloud(self, pc, color="hsv"): @@ -153,11 +132,13 @@ class PointcloudObjectRecognizer(): return pointcloud if __name__ == '__main__': - rospy.init_node(NODE) + rospy.init_node('pc_object_recognizer') + model = rospy.get_param("~model") model_id = rospy.get_param("~model_id") + model_dir = rospy.get_param("~model_dir") dataset = rospy.get_param("~dataset") - object_recognizer = PointcloudObjectRecognizer(model, model_id, dataset) - #rospy.loginfo('\033[92m'+"PCL Recognizer is ready using: %s , model: %s ", model, model_id) + + object_recognizer = PointcloudObjectRecognizer(model, model_id, dataset, model_dir) rospy.loginfo("PCL Recognizer is ready using: %s , model: %s ", model, model_id) rospy.spin() From 37d0a63d7882ccc3e60bf2cb7483048f1f65392e Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 20:20:20 +0200 Subject: [PATCH 7/9] [perception/recognition] moved script -> scripts --- .../ros/{script => scripts}/pc_object_recognizer_node | 0 .../ros/{script => scripts}/rgb_object_recognizer_node | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename mir_perception/mir_object_recognition/ros/{script => scripts}/pc_object_recognizer_node (100%) rename mir_perception/mir_object_recognition/ros/{script => scripts}/rgb_object_recognizer_node (100%) diff --git a/mir_perception/mir_object_recognition/ros/script/pc_object_recognizer_node b/mir_perception/mir_object_recognition/ros/scripts/pc_object_recognizer_node similarity index 100% rename from mir_perception/mir_object_recognition/ros/script/pc_object_recognizer_node rename to mir_perception/mir_object_recognition/ros/scripts/pc_object_recognizer_node diff --git a/mir_perception/mir_object_recognition/ros/script/rgb_object_recognizer_node b/mir_perception/mir_object_recognition/ros/scripts/rgb_object_recognizer_node similarity index 100% rename from mir_perception/mir_object_recognition/ros/script/rgb_object_recognizer_node rename to mir_perception/mir_object_recognition/ros/scripts/rgb_object_recognizer_node From cf3de6b1551bccb3fad40160d55021f86130a210 Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sat, 16 Oct 2021 20:36:40 +0200 Subject: [PATCH 8/9] Added source for mir_perception_models --- repository.rosinstall | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repository.rosinstall b/repository.rosinstall index 364e9202c..f6b49f989 100644 --- a/repository.rosinstall +++ b/repository.rosinstall @@ -102,3 +102,8 @@ local-name: atwork-commander uri: https://github.com/robocup-at-work/atwork-commander.git version: master + +- git: + local-name: mir_perception_models + uri: https://github.com/b-it-bots/mir_perception_models.git + version: main From aa0e4759fc2efd572ef166ea6d8f286373631eb6 Mon Sep 17 00:00:00 2001 From: Mohammad Wasil Date: Sun, 17 Oct 2021 11:12:28 +0200 Subject: [PATCH 9/9] [perception/recognition] added rgb and pointcloud object recognition models --- mir_perception/mir_object_recognition/package.xml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mir_perception/mir_object_recognition/package.xml b/mir_perception/mir_object_recognition/package.xml index f209b3c81..f78a0704a 100644 --- a/mir_perception/mir_object_recognition/package.xml +++ b/mir_perception/mir_object_recognition/package.xml @@ -23,7 +23,8 @@ roslint mir_object_segmentation mir_perception_utils - mir_perception_models + mir_rgb_object_recognition_models + mir_pointcloud_object_recognition_models cv_bridge geometry_msgs