Recording

pyproject.toml

@@ -30,6 +30,8 @@ dependencies = [
 	"antlr4-python3-runtime ~= 4.11",
 	"attrs >= 19.3.0",
 	"dotmap ~= 1.3",
+	"imageio == 2.19.3",
+	"imageio-ffmpeg == 0.4.7",
 	"mapbox_earcut >= 0.12.10",
 	"matplotlib ~= 3.2",
 	"manifold3d == 2.3.0",
@@ -44,7 +46,7 @@ dependencies = [
 	"python-fcl >= 0.7",
 	"Rtree ~= 1.0",
 	"rv-ltl ~= 0.1",
-	"scikit-image ~= 0.21",
+	"scikit-image ~= 0.20",
 	"scipy ~= 1.7",
 	"shapely ~= 2.0",
 	"trimesh >=4.0.9, <5",

src/scenic/simulators/carla/model.scenic

@@ -95,6 +95,9 @@ param weather = Uniform(
     'MidRainSunset',
     'HardRainSunset'
 )
+param resolution = '848x480' # '1280x720'
+param video_output_path = None
+param enable_bird_view = False
 
 simulator CarlaSimulator(
     carla_map=globalParameters.carla_map,
@@ -104,7 +107,10 @@ simulator CarlaSimulator(
     timeout=int(globalParameters.timeout),
     render=bool(globalParameters.render),
     record=globalParameters.record,
-    timestep=float(globalParameters.timestep)
+    timestep=float(globalParameters.timestep),
+    resolution=globalParameters.resolution,
+    video_output_path=globalParameters.video_output_path,
+    enable_bird_view=globalParameters.enable_bird_view
 )
 
 class CarlaActor(DrivingObject):

src/scenic/simulators/carla/simulator.py

@@ -38,6 +38,9 @@ def __init__(
         record="",
         timestep=0.1,
         traffic_manager_port=None,
+        resolution='1280x720',
+        video_output_path=None,
+        enable_bird_view=False
     ):
         super().__init__()
         verbosePrint(f"Connecting to CARLA on port {port}")
@@ -72,6 +75,10 @@ def __init__(
         self.record = record  # whether to use the carla recorder
         self.scenario_number = 0  # Number of the scenario executed
 
+        self.resolution = resolution
+        self.video_output_path = video_output_path
+        self.enable_bird_view = enable_bird_view
+
     def createSimulation(self, scene, *, timestep, **kwargs):
         if timestep is not None and timestep != self.timestep:
             raise RuntimeError(
@@ -88,6 +95,9 @@ def createSimulation(self, scene, *, timestep, **kwargs):
             self.record,
             self.scenario_number,
             timestep=self.timestep,
+            resolution=self.resolution,
+            video_output_path=self.video_output_path,
+            enable_bird_view=self.enable_bird_view,
             **kwargs,
         )
 
@@ -101,7 +111,8 @@ def destroy(self):
 
 
 class CarlaSimulation(DrivingSimulation):
-    def __init__(self, scene, client, tm, render, record, scenario_number, **kwargs):
+    def __init__(self, scene, client, tm, render, record, scenario_number, 
+                 resolution, video_output_path, enable_bird_view, **kwargs):
         self.client = client
         self.world = self.client.get_world()
         self.map = self.world.get_map()
@@ -111,6 +122,9 @@ def __init__(self, scene, client, tm, render, record, scenario_number, **kwargs)
         self.record = record
         self.scenario_number = scenario_number
         self.cameraManager = None
+        self.resolution = resolution
+        self.video_output_path = video_output_path
+        self.enable_bird_view = enable_bird_view
 
         super().__init__(scene, **kwargs)
 
@@ -151,7 +165,12 @@ def setup(self):
             camIndex = 0
             camPosIndex = 0
             egoActor = self.objects[0].carlaActor
-            self.cameraManager = visuals.CameraManager(self.world, egoActor, self.hud)
+            self.cameraManager = visuals.CameraManager(
+                self.world, egoActor, self.hud, 
+                fps=1.0 / self.timestep, 
+                resolution=self.resolution, video_output_path=self.video_output_path,
+                enable_bird_view=self.enable_bird_view
+            )
             self.cameraManager._transform_index = camPosIndex
             self.cameraManager.set_sensor(camIndex)
             self.cameraManager.set_transform(self.camTransform)

src/scenic/simulators/carla/utils/visuals.py

@@ -27,12 +27,16 @@
 import math
 import weakref
 
+import cv2
+import imageio as iio
+
 import carla
 from carla import ColorConverter as cc
 import numpy as np
 import pygame
 
 
+
 def get_actor_display_name(actor, truncate=250):
     name = " ".join(actor.type_id.replace("_", ".").title().split(".")[1:])
     return (name[: truncate - 1] + "\u2026") if len(name) > truncate else name
@@ -266,15 +270,23 @@ def _on_collision(weak_self, event):
 
 
 class CameraManager(object):
-    def __init__(self, world, actor, hud):
+    def __init__(self, world, actor, hud, fps=30.0, resolution='848x480', video_output_path=None, enable_bird_view=False):
         self.sensor = None
         self._surface = None
         self._actor = actor
+        self.output_path = video_output_path
+        self.resolution = resolution
+        self.recording = video_output_path is not None
         self._hud = hud
         self.images = []
+        bird_eye_view_transform = carla.Transform(
+            carla.Location(x=0, y=0, z=10),  # 10 meters above the ground
+            carla.Rotation(pitch=-90, yaw=0, roll=0)  # Pointing straight down
+        )
         self._camera_transforms = [
             carla.Transform(carla.Location(x=-5.5, z=2.8), carla.Rotation(pitch=-15)),
             carla.Transform(carla.Location(x=1.6, z=1.7)),
+            bird_eye_view_transform,
         ]
         self._transform_index = 1
         self._sensors = [
@@ -299,6 +311,16 @@ def __init__(self, world, actor, hud):
             ["sensor.lidar.ray_cast", None, "Lidar (Ray-Cast)"],
         ]
         self._world = world
+        self.video_writer = None
+        if self.recording:
+            self.resolution = [int(x) for x in resolution.split("x")]
+            video_file_name = video_output_path.split('.')[0]
+            if enable_bird_view:
+                video_file_name += "_bev"
+                self.bev = True
+            video_output_path = video_file_name + ".mp4"
+            self.video_writer = iio.get_writer(
+                video_output_path, fps=fps)
         bp_library = self._world.get_blueprint_library()
         for item in self._sensors:
             bp = bp_library.find(item[0])
@@ -309,7 +331,7 @@ def __init__(self, world, actor, hud):
         self._index = None
 
     def toggle_camera(self):
-        set_transform((self._transform_index + 1) % len(self._camera_transforms))
+        self.set_transform((self._transform_index + 1) % len(self._camera_transforms))
 
     def set_transform(self, idx):
         self._transform_index = idx
@@ -365,9 +387,25 @@ def _parse_image(weak_self, image):
             array = array[:, :, :3]
             array = array[:, :, ::-1]
             self._surface = pygame.surfarray.make_surface(array.swapaxes(0, 1))
+            if self.recording:
+                # make sure we are in bird's eye view
+                if self.bev:
+                    if self._transform_index != 2:
+                        self.set_transform(2)
+
+                im = np.reshape(np.copy(image.raw_data).astype(np.uint8), (image.height, image.width, 4))
+                try:
+                    self.video_writer.append_data(im)
+                    self.cv2_video_writer.write(im)
+                except Exception as e:
+                    print("Failed to write video:", e)
         self.images.append(image)
 
     def destroy_sensor(self):
         if self.sensor is not None:
             self.sensor.stop()
             self.sensor.destroy()
+
+    def __del__(self):
+        if self.video_writer:
+            self.video_writer.close()