diff --git a/pyproject.toml b/pyproject.toml index e69b47d05..2561f6397 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,8 @@ dependencies = [ "antlr4-python3-runtime ~= 4.11", "attrs >= 19.3.0", "dotmap ~= 1.3", + "imageio == 2.19.3", + "imageio-ffmpeg == 0.4.7", "mapbox_earcut >= 0.12.10", "matplotlib ~= 3.2", "manifold3d == 2.3.0", @@ -44,7 +46,7 @@ dependencies = [ "python-fcl >= 0.7", "Rtree ~= 1.0", "rv-ltl ~= 0.1", - "scikit-image ~= 0.21", + "scikit-image ~= 0.20", "scipy ~= 1.7", "shapely ~= 2.0", "trimesh >=4.0.9, <5", diff --git a/src/scenic/simulators/carla/model.scenic b/src/scenic/simulators/carla/model.scenic index 6a9fa954d..a35f563d1 100644 --- a/src/scenic/simulators/carla/model.scenic +++ b/src/scenic/simulators/carla/model.scenic @@ -95,6 +95,9 @@ param weather = Uniform( 'MidRainSunset', 'HardRainSunset' ) +param resolution = '848x480' # '1280x720' +param video_output_path = None +param enable_bird_view = False simulator CarlaSimulator( carla_map=globalParameters.carla_map, @@ -104,7 +107,10 @@ simulator CarlaSimulator( timeout=int(globalParameters.timeout), render=bool(globalParameters.render), record=globalParameters.record, - timestep=float(globalParameters.timestep) + timestep=float(globalParameters.timestep), + resolution=globalParameters.resolution, + video_output_path=globalParameters.video_output_path, + enable_bird_view=globalParameters.enable_bird_view ) class CarlaActor(DrivingObject): diff --git a/src/scenic/simulators/carla/simulator.py b/src/scenic/simulators/carla/simulator.py index 6e7022bae..ebc7df343 100644 --- a/src/scenic/simulators/carla/simulator.py +++ b/src/scenic/simulators/carla/simulator.py @@ -38,6 +38,9 @@ def __init__( record="", timestep=0.1, traffic_manager_port=None, + resolution='1280x720', + video_output_path=None, + enable_bird_view=False ): super().__init__() verbosePrint(f"Connecting to CARLA on port {port}") @@ -72,6 +75,10 @@ def __init__( self.record = record # whether to use the carla recorder self.scenario_number = 0 # Number of the scenario executed + self.resolution = resolution + self.video_output_path = video_output_path + self.enable_bird_view = enable_bird_view + def createSimulation(self, scene, *, timestep, **kwargs): if timestep is not None and timestep != self.timestep: raise RuntimeError( @@ -88,6 +95,9 @@ def createSimulation(self, scene, *, timestep, **kwargs): self.record, self.scenario_number, timestep=self.timestep, + resolution=self.resolution, + video_output_path=self.video_output_path, + enable_bird_view=self.enable_bird_view, **kwargs, ) @@ -101,7 +111,8 @@ def destroy(self): class CarlaSimulation(DrivingSimulation): - def __init__(self, scene, client, tm, render, record, scenario_number, **kwargs): + def __init__(self, scene, client, tm, render, record, scenario_number, + resolution, video_output_path, enable_bird_view, **kwargs): self.client = client self.world = self.client.get_world() self.map = self.world.get_map() @@ -111,6 +122,9 @@ def __init__(self, scene, client, tm, render, record, scenario_number, **kwargs) self.record = record self.scenario_number = scenario_number self.cameraManager = None + self.resolution = resolution + self.video_output_path = video_output_path + self.enable_bird_view = enable_bird_view super().__init__(scene, **kwargs) @@ -151,7 +165,12 @@ def setup(self): camIndex = 0 camPosIndex = 0 egoActor = self.objects[0].carlaActor - self.cameraManager = visuals.CameraManager(self.world, egoActor, self.hud) + self.cameraManager = visuals.CameraManager( + self.world, egoActor, self.hud, + fps=1.0 / self.timestep, + resolution=self.resolution, video_output_path=self.video_output_path, + enable_bird_view=self.enable_bird_view + ) self.cameraManager._transform_index = camPosIndex self.cameraManager.set_sensor(camIndex) self.cameraManager.set_transform(self.camTransform) diff --git a/src/scenic/simulators/carla/utils/visuals.py b/src/scenic/simulators/carla/utils/visuals.py index c73e832e3..7a2d3635a 100644 --- a/src/scenic/simulators/carla/utils/visuals.py +++ b/src/scenic/simulators/carla/utils/visuals.py @@ -27,12 +27,16 @@ import math import weakref +import cv2 +import imageio as iio + import carla from carla import ColorConverter as cc import numpy as np import pygame + def get_actor_display_name(actor, truncate=250): name = " ".join(actor.type_id.replace("_", ".").title().split(".")[1:]) return (name[: truncate - 1] + "\u2026") if len(name) > truncate else name @@ -266,15 +270,23 @@ def _on_collision(weak_self, event): class CameraManager(object): - def __init__(self, world, actor, hud): + def __init__(self, world, actor, hud, fps=30.0, resolution='848x480', video_output_path=None, enable_bird_view=False): self.sensor = None self._surface = None self._actor = actor + self.output_path = video_output_path + self.resolution = resolution + self.recording = video_output_path is not None self._hud = hud self.images = [] + bird_eye_view_transform = carla.Transform( + carla.Location(x=0, y=0, z=10), # 10 meters above the ground + carla.Rotation(pitch=-90, yaw=0, roll=0) # Pointing straight down + ) self._camera_transforms = [ carla.Transform(carla.Location(x=-5.5, z=2.8), carla.Rotation(pitch=-15)), carla.Transform(carla.Location(x=1.6, z=1.7)), + bird_eye_view_transform, ] self._transform_index = 1 self._sensors = [ @@ -299,6 +311,16 @@ def __init__(self, world, actor, hud): ["sensor.lidar.ray_cast", None, "Lidar (Ray-Cast)"], ] self._world = world + self.video_writer = None + if self.recording: + self.resolution = [int(x) for x in resolution.split("x")] + video_file_name = video_output_path.split('.')[0] + if enable_bird_view: + video_file_name += "_bev" + self.bev = True + video_output_path = video_file_name + ".mp4" + self.video_writer = iio.get_writer( + video_output_path, fps=fps) bp_library = self._world.get_blueprint_library() for item in self._sensors: bp = bp_library.find(item[0]) @@ -309,7 +331,7 @@ def __init__(self, world, actor, hud): self._index = None def toggle_camera(self): - set_transform((self._transform_index + 1) % len(self._camera_transforms)) + self.set_transform((self._transform_index + 1) % len(self._camera_transforms)) def set_transform(self, idx): self._transform_index = idx @@ -365,9 +387,25 @@ def _parse_image(weak_self, image): array = array[:, :, :3] array = array[:, :, ::-1] self._surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) + if self.recording: + # make sure we are in bird's eye view + if self.bev: + if self._transform_index != 2: + self.set_transform(2) + + im = np.reshape(np.copy(image.raw_data).astype(np.uint8), (image.height, image.width, 4)) + try: + self.video_writer.append_data(im) + self.cv2_video_writer.write(im) + except Exception as e: + print("Failed to write video:", e) self.images.append(image) def destroy_sensor(self): if self.sensor is not None: self.sensor.stop() self.sensor.destroy() + + def __del__(self): + if self.video_writer: + self.video_writer.close()