Merge pull request #1 from Berkeley-UAV/airsim

Airsim

VerifAI/examples/scenic/README.md

@@ -15,7 +15,7 @@ It can be set up as follows:
 
 The `falsify_distance.py` example does falsification with a given driving scenario written in Scenic, trying to falsify a specification stating that the ego car always maintains a minimum distance of 5 meters to any other object.
 We have included in the `newtonian` and `carla` subfolders examples of Scenic scenarios that work with this falsifier and run in Scenic's built-in Newtonian simulator and the CARLA driving simulator respectively.
-These examples are [CARLA Challenge scenarios](https://carlachallenge.org/challenge/nhtsa/>) derived from the [National Highway Traffic Safety Administration (NHTSA) pre-crash typology](https://www.nhtsa.gov/sites/nhtsa.gov/files/pre-crash_scenario_typology-final_pdf_version_5-2-07.pdf/>).
+These examples are [CARLA Challenge scenarios](https://web.archive.org/web/20221219223342/https://carlachallenge.org/challenge/nhtsa/) derived from the [National Highway Traffic Safety Administration (NHTSA) pre-crash typology](https://www.nhtsa.gov/sites/nhtsa.gov/files/pre-crash_scenario_typology-final_pdf_version_5-2-07.pdf).
 Many more examples can be found in the [Scenic repo](https://github.com/BerkeleyLearnVerify/Scenic/tree/master/examples).
 
 (Note: the CARLA examples use the CARLA interface provided by Scenic.

VerifAI/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "verifai"
-version = "2.1.0"
+version = "2.1.1"
 description = "A toolkit for the formal design and analysis of systems that include artificial intelligence (AI) and machine learning (ML) components."
 authors = [
 	{ name = "Tommaso Dreossi" },

VerifAI/src/verifai/features/features.py

@@ -721,7 +721,21 @@ def __repr__(self):
         return f'ScalarArray({self.domain}, {self.shape})'
 
 class Struct(Domain):
-    """A domain consisting of named sub-domains."""
+    """A domain consisting of named sub-domains.
+
+    The order of the sub-domains is arbitrary: two Structs are considered equal
+    if they have the same named sub-domains, regardless of order. As the order
+    is an implementation detail, accessing the values of sub-domains in points
+    sampled from a Struct should be done by name:
+
+    >>> struct = Struct({'a': Box((0, 1)), 'b': Box((2, 3))})
+    >>> point = struct.uniformPoint()
+    >>> point.b
+    (2.20215292046797,)
+
+    Within a given version of VerifAI, the sub-domain order is consistent, so
+    that the order of columns in error tables is also consistent.
+    """
 
     def __init__(self, domains):
         self.namedDomains = tuple(sorted(domains.items(), key=lambda i: i[0]))

VerifAI/src/verifai/samplers/scenic_sampler.py

@@ -194,7 +194,10 @@ def spaceForScenario(scenario, ignoredProperties):
     assert scenario.egoObject is scenario.objects[0]
     doms = (domainForObject(obj, ignoredProperties)
             for obj in scenario.objects)
-    objects = Struct({ f'object{i}': dom for i, dom in enumerate(doms) })
+    objects = Struct({
+        ScenicSampler.nameForObject(i): dom
+        for i, dom in enumerate(doms)
+    })
 
     # create domains for global parameters
     paramDoms = {}
@@ -279,15 +282,30 @@ def nextSample(self, feedback=None):
         return self.pointForScene(self.lastScene)
 
     def pointForScene(self, scene):
-        """Convert a sampled Scenic :obj:`Scene` to a point in our feature space."""
+        """Convert a sampled Scenic :obj:`~scenic.core.scenarios.Scene` to a point in our feature space.
+
+        The `FeatureSpace` used by this sampler consists of 2 features:
+
+            * ``objects``, which is a `Struct` consisting of attributes ``object0``,
+              ``object1``, etc. with the properties of the corresponding objects
+              in the Scenic program. The names of these attributes may change in a
+              future version of VerifAI: use the `nameForObject` function to
+              generate them.
+            * ``params``, which is a `Struct` storing the values of the
+              :term:`global parameters` of the Scenic program (use
+              `paramDictForSample` to extract them).
+        """
         lengths, dom = self.space.domains
         assert lengths is None
         assert scene.egoObject is scene.objects[0]
         objDomain = dom.domainNamed['objects']
         assert len(objDomain.domains) == len(scene.objects)
-        objects = (pointForObject(objDomain.domainNamed[f'object{i}'], obj)
-                   for i, obj in enumerate(scene.objects))
-        objPoint = objDomain.makePoint(*objects)
+        objects = {
+            self.nameForObject(i):
+            pointForObject(objDomain.domainNamed[self.nameForObject(i)], obj)
+            for i, obj in enumerate(scene.objects)
+        }
+        objPoint = objDomain.makePoint(**objects)
 
         paramDomain = dom.domainNamed['params']
         params = {}
@@ -298,8 +316,17 @@ def pointForScene(self, scene):
 
         return self.space.makePoint(objects=objPoint, params=paramPoint)
 
+    @staticmethod
+    def nameForObject(i):
+        """Name used in the `FeatureSpace` for the Scenic object with index i.
+
+        That is, if ``scene`` is a :obj:`~scenic.core.scenarios.Scene`, the object
+        ``scene.objects[i]``.
+        """
+        return f'object{i}'
+
     def paramDictForSample(self, sample):
-        """Recover the dict of global parameters from a `ScenicSampler` sample."""
+        """Recover the dict of :term:`global parameters` from a `ScenicSampler` sample."""
         params = sample.params._asdict()
         corrected = {}
         for newName, quotedParam in self.quotedParams.items():

VerifAI/tests/scenic/test_scenic.py

@@ -71,9 +71,15 @@ def test_object_order(new_Object):
     sample = sampler.nextSample()
     objects = sample.objects
     assert len(objects) == 11
-    for i, obj in enumerate(objects):
+    for i in range(len(objects)):
+        name = ScenicSampler.nameForObject(i)
+        obj = getattr(objects, name)
         assert obj.position[:2] == pytest.approx((2*i, 0))
 
+    flat = sampler.space.flatten(sample)
+    unflat = sampler.space.unflatten(flat)
+    assert unflat == sample
+
 ## Active sampling
 
 def test_active_sampling(new_Object):

error_table.csv

@@ -1,2 +1,2 @@
-,point.objects.object0.position[0],point.objects.object0.position[1],point.objects.object0.position[2],rho
-0,6.562315408426532,0.07216232119218624,0.0,False
+,point.objects.object0000.position[0],point.objects.object0000.position[1],point.objects.object0000.position[2],point.objects.object0183.position[0],point.objects.object0183.position[1],point.objects.object0183.position[2],point.objects.object0184.position[0],point.objects.object0184.position[1],point.objects.object0184.position[2],point.objects.object0185.position[0],point.objects.object0185.position[1],point.objects.object0185.position[2],point.objects.object0186.position[0],point.objects.object0186.position[1],point.objects.object0186.position[2],point.objects.object0187.position[0],point.objects.object0187.position[1],point.objects.object0187.position[2],rho
+0,0.0,182.15847435580088,12.0,-31.57378903787832,226.96346121163208,4.980050019073486,41.66522160655998,193.8849795332645,4.980050019073486,37.4566338765801,175.27174471452568,4.980050019073486,-24.38182543180618,229.4351317177681,4.980050019073486,41.66873910321773,193.8825056642247,11.980050019073486,False

examples/airsim/demoDrone.scenic

@@ -1,32 +1,19 @@
 import math
 
 # NOTE: add your world info path here
-param worldInfoPath = "C:/Users/piegu/Scenic/examples/airsim/worldInfo/droneBlocks"
+param worldInfoPath = "examples/airsim/worldInfo/droneBlocks"
 
 model scenic.simulators.airsim.model
 
-# This demo includes 1 adversarial drone, and 1 drone looking for this adversary.
-# Also contains notes on relevant scenic features you might want to use.
+# This demo includes 1 adversarial drone, and 1 drone (ego) looking for this adversary.
 
-# Can define a workspace, by default workspace contains all space
-"""
-workspace_region = RectangularRegion(Vector(0,0,0), 30, 30, 30)
-workspace = Workspace(workspace_region)
-"""
-
-def magnitude(v):
-    return math.hypot(v.x, v.y, v.z)
-
-# Can extend scenic models to add more parameters to them. Generic models are in model.scenic.
-# Scenic models are like templates for the objects to spawn.
-class AdversaryDrone(Drone):
+class AdversaryDrone(Drone): # Drone class defined in model.scenic
     patrolPoints: []
     patrolPointsProb: []
 
 # Find the adversary. drone1 is the adversary target
 behavior FindAdversary(positions, speed = 5):
-    # typical syntax for behaviors in scenarios are try-interrupt statements. It's essentially a nicer while loop
-    # https://scenic-lang.readthedocs.io/en/latest/reference/statements.html#try-interrupt-statement
+    # try-interrupt statements: https://scenic-lang.readthedocs.io/en/latest/reference/statements.html#try-interrupt-statement
     try:
         print("POSSIBLE POSITIONS:")
         print(positions)
@@ -39,59 +26,55 @@ behavior FindAdversary(positions, speed = 5):
             print("EGO CHECKING POSITION:")
             print(selectedPoint)
 
-            do FlyToPosition(selectedPoint, speed=speed, tolerance=1,pidMode=True)
-
-            # resample point since didn't find adversary at that position
+            do FlyToPosition(selectedPoint, speed=speed, tolerance=1,pidMode=True) # FlyToPosition behavior defined in behaviors.scenic
 
-    interrupt when (distance from self to drone1) < 20:
-        # when I see that I am within 20 meters of adversary, follow it
-        print("FOLLOW")
-        do Follow(drone1, speed=10, tolerance=1, offset=(0,0,0))
-    interrupt when distance from self to drone1 < 5:
-        # when I get within 5 meters of adversary, terminate scenario
-        print("TERMINATING")
+    interrupt when (distance from self to drone1) < 15:
+        # when I see that I am within 15 meters of adversary, follow it
+        print("FOLLOWING ADVERSARY")
+        do Follow(drone1, speed=10, tolerance=1, offset=(0,0,0)) # Follow behavior defined in behaviors.scenic
+
+    interrupt when distance from self to drone1 < 7:
+        # when I get within 7 meters of adversary, terminate scenario
+        print("ADVERSARY FOUND")
         terminate
 
-# Adversary behavior. Move to randomly chosen position out of points.
+# Adversary behavior. Patrol through given points.
 behavior AdversaryBehavior(points, speed):
-    do Patrol(points, loop=True, speed=speed)
+    do Patrol(points, loop=True, speed=speed) # Patrol behavior defined in behaviors.scenic
 
 ground = getPrexistingObj("ground")
-centerArea = RectangularRegion(Vector(0,200,30), 0, 70, 70)
-platforms = []
+workspaceArea = RectangularRegion(Vector(0,200,30), 0, 100, 100)
 
+platforms = []
 blockCount = 4
 for i in range(blockCount):
     platforms.append(new StaticObj on ground, 
-        contained in centerArea,
+        contained in workspaceArea,
         with assetName "Cone", # use * to pick a random asset in assets
-        with width Range(3,10),
-        with length Range(3,10),
+        with parentOrientation 0,
+        with width 5,
+        with length 5,
         with height 10)
 
 points = []
 for plat in platforms:
     point = new Point on plat
     points.append(point.position)
 
-pt = Uniform(*points)
+adversarySpawn = Options(points)
 
 # Adversary drone spawning at random point
-drone1 = new AdversaryDrone at pt + (0,0,2),
+drone1 = new AdversaryDrone at adversarySpawn + (0,0,2),
     with behavior AdversaryBehavior(points, speed=5)
-
+# drone1.patrolPoints = possiblePoints
 drone1.patrolPointsProb = [0.4, 0.2, 0.1, 0.3] # Probability distribution on the patrolPoints
 
-ego = new Drone at (0,200,12),
+# ego drone spawning somwhere in the workspace area
+ego = new Drone at (0,Range(150, 250),12),
     with behavior FindAdversary(points, speed=5)
 
 
 # took too long to locate so terminate after x seconds
+terminate after 15 seconds 
 
-terminate after 30 seconds 
-
-
-# can require initial scenario conditions here 
-""" 
-require (distance from ego to drone) > 20
-"""
+record final (distance to drone1) as dist

examples/airsim/demoDroneNH.scenic

@@ -0,0 +1,84 @@
+import math
+
+# NOTE: add your world info path here
+param worldInfoPath = "examples/airsim/worldInfo/nh"
+
+model scenic.simulators.airsim.model
+
+# This demo includes 1 adversarial drone, and 1 drone (ego) looking for this adversary.
+
+class AdversaryDrone(Drone): # Drone class defined in model.scenic
+    patrolPoints: []
+    patrolPointsProb: []
+
+# Find the adversary. drone1 is the adversary target
+behavior FindAdversary(positions, speed = 5):
+    # try-interrupt statements: https://scenic-lang.readthedocs.io/en/latest/reference/statements.html#try-interrupt-statement
+    try:
+        print("POSSIBLE POSITIONS:")
+        print(positions)
+        while ((distance from self to drone1) >= 1):
+            selectedPoint = Discrete({positions[0]:drone1.patrolPointsProb[0], 
+            positions[1]:drone1.patrolPointsProb[1], 
+            positions[2]:drone1.patrolPointsProb[2], 
+            positions[3]:drone1.patrolPointsProb[3]})
+
+            print("EGO CHECKING POSITION:")
+            print(selectedPoint)
+
+            do FlyToPosition(selectedPoint, speed=speed, tolerance=1,pidMode=True) # FlyToPosition behavior defined in behaviors.scenic
+
+    interrupt when (distance from self to drone1) < 15:
+        # when I see that I am within 15 meters of adversary, follow it
+        print("FOLLOWING ADVERSARY")
+        do Follow(drone1, speed=10, tolerance=1, offset=(0,0,0)) # Follow behavior defined in behaviors.scenic
+
+    interrupt when distance from self to drone1 < 7:
+        # when I get within 7 meters of adversary, terminate scenario
+        print("ADVERSARY FOUND")
+        terminate
+
+# Adversary behavior. Patrol through given points.
+behavior AdversaryBehavior(points, speed):
+    do Patrol(points, loop=True, speed=speed) # Patrol behavior defined in behaviors.scenic
+
+ground = getPrexistingObj("nh")
+area1 = RectangularRegion(Vector(0,0,0), 0, 10, 70)
+area2 = RectangularRegion(Vector(0,0,0), 0, 70, 10)
+
+workspaceArea = area1.union(area2)
+
+platforms = []
+blockCount = 4
+for i in range(blockCount):
+    platforms.append(new StaticObj on ground, 
+        contained in workspaceArea,
+        with assetName "Cone", # use * to pick a random asset in assets
+        with parentOrientation 0,
+        with width 5,
+        with length 5,
+        with height 10)
+
+points = []
+for plat in platforms:
+    point = new Point on plat
+    points.append(point.position)
+
+adversarySpawn = Options(points)
+
+# Adversary drone spawning at random point
+drone1 = new AdversaryDrone at adversarySpawn + (0,0,2),
+    with behavior AdversaryBehavior(points, speed=2)
+# drone1.patrolPoints = possiblePoints
+drone1.patrolPointsProb = [0.4, 0.2, 0.1, 0.3] # Probability distribution on the patrolPoints
+
+# ego drone spawning somwhere in the middle of the workspace area
+ego = new Drone at (Range(-5, 5),Range(-5, 5),10),
+    with behavior FindAdversary(points, speed=5)
+
+
+
+# took too long to locate so terminate after x seconds
+terminate after 15 seconds 
+
+record final (distance to drone1) as dist