diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..2d3a595 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,16 @@ +[paths] +source = + src + */site-packages + +[run] +branch = true +parallel = true +source = + src/scene_synthesizer + test + +[report] +show_missing = true +precision = 1 + diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index e8a4be3..dd601da 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -4,7 +4,7 @@ From the root directory of the repo, run the following: ``` -python -m pytest -n auto tests/ +python -m pytest ``` ## Generate Documentation diff --git a/README.md b/README.md index ae769c8..89f3f12 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,44 @@ pip install -e.[recommend] ``` See the [documentation](https://scene-synthesizer.github.io/getting_started/install.html) for detailed installation instructions. +## Quick Start + +```python +import scene_synthesizer as synth +import scene_synthesizer.procedural_assets as pa + +# create procedural assets +table = synth.procedural_assets.TableAsset(width=1.2, depth=0.8, height=0.75) +cabinet = synth.procedural_assets.CabinetAsset(width=0.5, height=0.5, depth=0.4, compartment_mask=[[0], [1]], compartment_types=['drawer','drawer']) + +# load asset from file +# Make sure to first download the file: +# wget https://raw.githubusercontent.com/clemense/kitchen-assets-cc-by/refs/heads/main/assets/chair/meshes/chair.{mtl,obj} +chair = synth.Asset('chair.obj', up=(0, 0, 1), front=(-1, 0, 0)) + +# create scene +scene = synth.Scene() + +# add table to scene +scene.add_object(table) +# put cabinet next to table +scene.add_object(cabinet, connect_parent_anchor=('right', 'front', 'bottom'), connect_obj_anchor=('left', 'front', 'bottom')) +# put chair in front of table +scene.add_object(chair, connect_parent_id='table', connect_parent_anchor=('center', 'front', 'bottom'), connect_obj_anchor=('center', 'center', 'bottom')) + +# randomly place plate and glass on top of table +scene.label_support('table_surface', obj_ids='table') +scene.place_object('plate', synth.procedural_assets.PlateAsset(), support_id='table_surface') +scene.place_object('glass', synth.procedural_assets.GlassAsset(), support_id='table_surface') + +# preview scene in an opengl window +scene.show() + +# export scene in various formats +scene.export('scene.usd') +scene.export('scene.urdf') +``` + ## License The code is released under the [Apache-2.0 license](https://github.com/NVlabs/scene_synthesizer/blob/main/LICENSE). diff --git a/examples/add_random_mdl_materials.py b/examples/add_random_mdl_materials.py index 0db7553..64f4b4d 100644 --- a/examples/add_random_mdl_materials.py +++ b/examples/add_random_mdl_materials.py @@ -138,7 +138,7 @@ # The keys are regular expressions of prim_paths in the USD geometry2material = { "(.*cabinet.*corpus.*|.*cabinet.*door|.*drawer.*board.*|.*cabinet.*closed.*|/world/kitchen_island/.*)|/world/corner_(1|2|3)": "cabinet", - "(.*refrigerator.*|.*range_hood.*|.*oven.*|.*dishwasher.*)": "appliances", + "(.*refrigerator.*|.*range_hood.*|.*range.*|.*dishwasher.*)": "appliances", "/world/oven/corpus/heater.*": 'rusted metal', "/world/oven/corpus/top": 'glossy black', ".*/corpus/sink": 'sink', diff --git a/paper/paper.bib b/paper/paper.bib index a8b5d56..3157828 100644 --- a/paper/paper.bib +++ b/paper/paper.bib @@ -32,7 +32,7 @@ @InProceedings{Mo_2019_CVPR author = {Mo, Kaichun and Zhu, Shilin and Chang, Angel X. and Yi, Li and Tripathi, Subarna and Guibas, Leonidas J. and Su, Hao}, doi = {10.1109/cvpr.2019.00100}, title = {{PartNet}: A Large-Scale Benchmark for Fine-Grained and Hierarchical Part-Level {3D} Object Understanding}, - booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, month = {June}, year = {2019}, } @@ -56,7 +56,7 @@ @inproceedings{ehsani2021manipulathor title={ManipulaTHOR: A Framework for Visual Object Manipulation}, doi={10.1109/cvpr46437.2021.00447}, author={Ehsani, Kiana and Han, Winson and Herrasti, Alvaro and VanderBilt, Eli and Weihs, Luca and Kolve, Eric and Kembhavi, Aniruddha and Mottaghi, Roozbeh}, - booktitle={CVPR}, + booktitle={Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, year={2021} } @@ -64,7 +64,7 @@ @InProceedings{Xiang_2020_SAPIEN author = {Xiang, Fanbo and Qin, Yuzhe and Mo, Kaichun and Xia, Yikuan and Zhu, Hao and Liu, Fangchen and Liu, Minghua and Jiang, Hanxiao and Yuan, Yifu and Wang, He and Yi, Li and Chang, Angel X. and Guibas, Leonidas J. and Su, Hao}, doi = {10.1109/cvpr42600.2020.01111}, title = {{SAPIEN}: A SimulAted Part-based Interactive ENvironment}, - booktitle = {The IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, month = {June}, year = {2020} } @@ -75,7 +75,7 @@ @inproceedings{procthor Winson Han and Eric Kolve and Ali Farhadi and Aniruddha Kembhavi and Roozbeh Mottaghi}, title={{ProcTHOR: Large-Scale Embodied AI Using Procedural Generation}}, - booktitle={NeurIPS}, + booktitle = {Advances in Neural Information Processing Systems (NeurIPS)}, year={2022} } @@ -98,7 +98,7 @@ @inproceedings{infinigen2023infinite title={Infinite Photorealistic Worlds Using Procedural Generation}, doi={10.1109/cvpr52729.2023.01215}, author={Raistrick, Alexander and Lipson, Lahav and Ma, Zeyu and Mei, Lingjie and Wang, Mingzhe and Zuo, Yiming and Kayan, Karhan and Wen, Hongyu and Han, Beining and Wang, Yihan and Newell, Alejandro and Law, Hei and Goyal, Ankit and Yang, Kaiyu and Deng, Jia}, - booktitle={Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition}, + booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, pages={12630--12641}, year={2023} } @@ -123,7 +123,7 @@ @inproceedings{schult24controlroom3d author = {Schult, Jonas and Tsai, Sam and H\"ollein, Lukas and Wu, Bichen and Wang, Jialiang and Ma, Chih-Yao and Li, Kunpeng and Wang, Xiaofang and Wimbauer, Felix and He, Zijian and Zhang, Peizhao and Leibe, Bastian and Vajda, Peter and Hou, Ji}, doi = {10.1109/cvpr52733.2024.00593}, title = {ControlRoom3D: Room Generation using Semantic Proxy Rooms}, - booktitle = {IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, + booktitle = {Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition (CVPR)}, year = {2024}, } @@ -250,4 +250,4 @@ @misc{yang2024physcenephysicallyinteractable3d archivePrefix={arXiv}, primaryClass={cs.CV}, url={https://arxiv.org/abs/2404.09465}, -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 041aac7..331bbd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,5 +35,10 @@ import_heading_localfolder = "Local Folder" line_length = 100 [tool.pytest.ini_options] +# for more details on configuration, see +# https://github.com/pytest-dev/pytest-cov/tree/4732d50f2322a6e0ea480a6c400fbc96f78283bb/examples/src-layout norecursedirs = [".git", ".venv", "deprecated", "dist"] python_files = ["*_test.py"] +testpaths = ["tests"] +addopts = "-n auto --cov-config=.coveragerc --cov=scene_synthesizer --cov-report html" + diff --git a/src/scene_synthesizer/assets.py b/src/scene_synthesizer/assets.py index 1636985..202449d 100644 --- a/src/scene_synthesizer/assets.py +++ b/src/scene_synthesizer/assets.py @@ -1367,7 +1367,7 @@ def _usd_prim_path_to_node_name(path): scale = usd_import.get_scale(mesh_prim) if not np.allclose(scale, [1.0, 1.0, 1.0]): geometry.apply_scale(scale) - + node_name = _usd_prim_path_to_node_name(mesh_path) parent_node_name = _usd_prim_path_to_node_name(mesh_prim.GetParent().GetPath()) if _usd_prim_path_to_node_name(mesh_prim.GetParent().GetPath()) not in s.graph.nodes: @@ -1467,6 +1467,10 @@ def __init__(self, fname, **kwargs): Args: fname (str): File name. + **geom_class_visual (str): Class string in geom element that indicates whether this is visual geometry. Defaults to 'visual'. + **geom_class_collision (str): Class string in geom element that indicates whether this is collision geometry. Defaults to 'collision'. + **geom_groups_visual (list[int]): List of body/geom/group numbers that will be considered a visual geometry. Defaults to all. + **geom_groups_collision (list[int]): List of body/geom/group numbers that will be considered a collision geometry. Defaults to all. Raises: ValueError: Raises exception if file doesn't exist. @@ -1541,33 +1545,34 @@ def _traverse_xml_tree(self, elem, node_name, scene, identifier_fn, rad_conversi elif elem.tag == "geom": log.debug(f"Adding geom {identifier_fn(elem)} to {identifier_fn(elem.parent)}") - unkown_geometry = False - if elem.type == "mesh": + unknown_geometry = False + if elem.type == "mesh" or (hasattr(elem, 'mesh') and elem.mesh is not None): geometry = trimesh.load( trimesh.util.wrap_as_stream(elem.mesh.file.contents), file_type=elem.mesh.file.extension[1:], ) # Set mesh material - if elem.material.texture is not None: - material = trimesh.visual.material.SimpleMaterial( - image=Image.open(trimesh.util.wrap_as_stream(elem.material.texture.file.contents)) - ) - texture = trimesh.visual.TextureVisuals(uv=geometry.visual.uv, material=material) - geometry.visual = texture - else: - specular = getattr(elem.material, 'specular', None) - diffuse = getattr(elem.material, 'rgba', None) - - num_faces = len(geometry.faces) - face_colors = np.tile(diffuse, (num_faces, 1)) - # 'emission' - # 'reflectance' - # 'metallic' - # 'roughness' - # 'rgba' - - geometry.visual = trimesh.visual.ColorVisuals(mesh=geometry, face_colors=face_colors) + if elem.material is not None: + if elem.material.texture is not None: + material = trimesh.visual.material.SimpleMaterial( + image=Image.open(trimesh.util.wrap_as_stream(elem.material.texture.file.contents)) + ) + texture = trimesh.visual.TextureVisuals(uv=geometry.visual.uv, material=material) + geometry.visual = texture + else: + specular = getattr(elem.material, 'specular', None) + diffuse = getattr(elem.material, 'rgba', None) + + num_faces = len(geometry.faces) + face_colors = np.tile(diffuse, (num_faces, 1)) + # 'emission' + # 'reflectance' + # 'metallic' + # 'roughness' + # 'rgba' + + geometry.visual = trimesh.visual.ColorVisuals(mesh=geometry, face_colors=face_colors) elif elem.type == "sphere": # The sphere type defines a sphere. # Only one size parameter is used, specifying the radius of the sphere. @@ -1600,12 +1605,16 @@ def _traverse_xml_tree(self, elem, node_name, scene, identifier_fn, rad_conversi transform=self._get_transform(elem, rad_conversion_fn), ) else: - unkown_geometry = True + unknown_geometry = True + + elem_dclass_visual_label = self._attributes.get("geom_class_visual", 'visual') + elem_dclass_collision_label = self._attributes.get("geom_class_collision", 'collision') + elem_dclass = None if elem.dclass is None else elem.dclass.dclass - if not unkown_geometry and ((use_collision_geometry and elem.dclass.dclass == 'visual') or (use_collision_geometry == False and elem.dclass.dclass == 'collision')): - log.debug(f"Ignore geometry {elem.type} since it is of class '{elem.dclass.dclass}'.") + if not unknown_geometry and ((use_collision_geometry and elem_dclass == elem_dclass_visual_label) or (use_collision_geometry == False and elem_dclass == elem_dclass_collision_label)): + log.debug(f"Ignore geometry {elem.type} since it is of class '{elem_dclass}'.") else: - if not unkown_geometry: + if not unknown_geometry: if elem.mass is not None: if geometry.is_volume: volume = geometry.volume @@ -1622,16 +1631,34 @@ def _traverse_xml_tree(self, elem, node_name, scene, identifier_fn, rad_conversi if elem.friction is not None: # 3D array with sliding, torsional, and rolling friction coefficients pass - - geometry.metadata["layer"] = elem.dclass.dclass - - utils.add_node_to_scene( - scene=scene, - geometry=geometry, - node_name=identifier_fn(elem), - geom_name=identifier_fn(elem), - parent_node_name=identifier_fn(elem.parent), - ) + + ignore_geometry = False + if elem_dclass is not None: + if elem_dclass == elem_dclass_visual_label: + geometry.metadata["layer"] = 'visual' + elif elem_dclass == elem_dclass_collision_label: + geometry.metadata["layer"] = 'collision' + else: + geometry.metadata["layer"] = elem_dclass + else: + if elem.group is not None: + if "geom_groups_collision" in self._attributes: + if elem.group in self._attributes['geom_groups_collision']: + geometry.metadata["layer"] = 'collision' + if "geom_groups_visual" in self._attributes: + if elem.group in self._attributes['geom_groups_visual']: + geometry.metadata["layer"] = 'visual' + if "geom_groups_visual" in self._attributes and "geom_groups_collision" in self._attributes and elem.group not in self._attributes['geom_groups_collision'] and elem.group not in self._attributes['geom_groups_visual']: + ignore_geometry = True + + if not ignore_geometry: + utils.add_node_to_scene( + scene=scene, + geometry=geometry, + node_name=identifier_fn(elem), + geom_name=identifier_fn(elem), + parent_node_name=identifier_fn(elem.parent), + ) else: log.debug(f"Not used: {elem.tag}, {identifier_fn(elem)}") @@ -1714,7 +1741,14 @@ def _load_mjcf(self, fname, model_dir, namespace, use_collision_geometry=None): scene = trimesh.Scene(base_frame=namespace) identifier_fn = self._mjcf_id(namespace=namespace, worldbody=root.worldbody) - self._traverse_xml_tree(elem=root.worldbody, node_name=scene.graph.base_frame, scene=scene, identifier_fn=identifier_fn, rad_conversion_fn=rad_conversion_fn, use_collision_geometry=use_collision_geometry) + self._traverse_xml_tree( + elem=root.worldbody, + node_name=scene.graph.base_frame, + scene=scene, + identifier_fn=identifier_fn, + rad_conversion_fn=rad_conversion_fn, + use_collision_geometry=use_collision_geometry + ) self._add_joints(root, scene, identifier_fn=identifier_fn, rad_conversion_fn=rad_conversion_fn) diff --git a/src/scene_synthesizer/exchange/urdf.py b/src/scene_synthesizer/exchange/urdf.py index f316ebe..fe8004b 100644 --- a/src/scene_synthesizer/exchange/urdf.py +++ b/src/scene_synthesizer/exchange/urdf.py @@ -54,10 +54,10 @@ def export_urdf(scene, if fname is not None and folder is not None: log.warn(f"URDF export: folder={folder} will be ignored since file name is already specified.") - if folder is None or len(folder) == 0: - folder = "." if fname is not None: folder, fname = os.path.split(fname) + if folder is None or len(folder) == 0: + folder = "." # Remember current scene configuration to re-apply it after export current_configuration = scene.get_configuration() diff --git a/src/scene_synthesizer/exchange/usd.py b/src/scene_synthesizer/exchange/usd.py index ad5a00d..e063c35 100644 --- a/src/scene_synthesizer/exchange/usd.py +++ b/src/scene_synthesizer/exchange/usd.py @@ -56,12 +56,13 @@ def export_usd( if fname is not None and folder is not None: log.warn(f"USD export: folder={folder} will be ignored since file name is already specified.") - if folder is None or len(folder) == 0: - folder = "." if fname is not None: folder = os.path.dirname(fname) fname = os.path.basename(fname) + if folder is None or len(folder) == 0: + folder = "." + # Remember current scene configuration to re-apply it after export current_configuration = scene.get_configuration() position_from_joint = dict(zip(scene.get_joint_names(), current_configuration)) @@ -286,10 +287,27 @@ def add_geometry_node(stage, scene_path, g, g_vis, i, transform): if not hasattr(g_vis.material, 'image'): # This is a trimesh.visual.material.PBRMaterial material = g_vis.material + diffuse_texture = None + if material.baseColorTexture is not None: + image_dims = len(np.asarray(material.baseColorTexture).shape) + if image_dims == 3: + diffuse_texture = np.transpose( + np.asarray(material.baseColorTexture) / 255.0, axes=(2, 0, 1) + ) + else: + diffuse_texture = np.array( + [ + np.asarray(material.baseColorTexture) / 255.0, + np.asarray(material.baseColorTexture) / 255.0, + np.asarray(material.baseColorTexture) / 255.0, + ] + ) + material_textures = usd_export.PBRMaterial( - diffuse_color=g_vis.material.baseColorFactor[:3] / 255.0, # what about main_color? + diffuse_color=material.baseColorFactor[:3] / 255.0 if material.baseColorFactor is not None else None, metallic_value=material.metallicFactor, roughness_value=material.roughnessFactor, + diffuse_texture=diffuse_texture, ) elif g_vis.material.image is None: # This is a trimesh.visual.material.SimpleMaterial