patch verifai bug, plan b demo ready

VerifAI/src/verifai/samplers/scenic_sampler.py

@@ -194,7 +194,7 @@ def spaceForScenario(scenario, ignoredProperties):
     assert scenario.egoObject is scenario.objects[0]
     doms = (domainForObject(obj, ignoredProperties)
             for obj in scenario.objects)
-    objects = Struct({ f'object{i}': dom for i, dom in enumerate(doms) })
+    objects = Struct({ f'object{i:04d}': dom for i, dom in enumerate(doms) })
 
     # create domains for global parameters
     paramDoms = {}
@@ -285,7 +285,7 @@ def pointForScene(self, scene):
         assert scene.egoObject is scene.objects[0]
         objDomain = dom.domainNamed['objects']
         assert len(objDomain.domains) == len(scene.objects)
-        objects = (pointForObject(objDomain.domainNamed[f'object{i}'], obj)
+        objects = (pointForObject(objDomain.domainNamed[f'object{i:04d}'], obj)
                    for i, obj in enumerate(scene.objects))
         objPoint = objDomain.makePoint(*objects)
 

error_table.csv

@@ -1,2 +1,2 @@
-,point.objects.object0.position[0],point.objects.object0.position[1],point.objects.object0.position[2],rho
-0,6.562315408426532,0.07216232119218624,0.0,False
+,point.objects.object0000.position[0],point.objects.object0000.position[1],point.objects.object0000.position[2],point.objects.object0183.position[0],point.objects.object0183.position[1],point.objects.object0183.position[2],point.objects.object0184.position[0],point.objects.object0184.position[1],point.objects.object0184.position[2],point.objects.object0185.position[0],point.objects.object0185.position[1],point.objects.object0185.position[2],point.objects.object0186.position[0],point.objects.object0186.position[1],point.objects.object0186.position[2],point.objects.object0187.position[0],point.objects.object0187.position[1],point.objects.object0187.position[2],rho
+0,0.0,182.15847435580088,12.0,-31.57378903787832,226.96346121163208,4.980050019073486,41.66522160655998,193.8849795332645,4.980050019073486,37.4566338765801,175.27174471452568,4.980050019073486,-24.38182543180618,229.4351317177681,4.980050019073486,41.66873910321773,193.8825056642247,11.980050019073486,False

examples/airsim/demoDrone.scenic

@@ -1,18 +1,19 @@
 import math
 
 # NOTE: add your world info path here
-param worldInfoPath = "C:/Users/piegu/Scenic/examples/airsim/worldInfo/droneBlocks"
+param worldInfoPath = "examples/airsim/worldInfo/droneBlocks"
 
 model scenic.simulators.airsim.model
 
-# This demo includes 1 adversarial drone, and 1 drone looking for this adversary.
+# This demo includes 1 adversarial drone, and 1 drone (ego) looking for this adversary.
 
-class AdversaryDrone(Drone):
+class AdversaryDrone(Drone): # Drone class defined in model.scenic
     patrolPoints: []
     patrolPointsProb: []
 
 # Find the adversary. drone1 is the adversary target
 behavior FindAdversary(positions, speed = 5):
+    # try-interrupt statements: https://scenic-lang.readthedocs.io/en/latest/reference/statements.html#try-interrupt-statement
     try:
         print("POSSIBLE POSITIONS:")
         print(positions)
@@ -25,51 +26,55 @@ behavior FindAdversary(positions, speed = 5):
             print("EGO CHECKING POSITION:")
             print(selectedPoint)
 
-            do FlyToPosition(selectedPoint, speed=speed, tolerance=1,pidMode=True)
+            do FlyToPosition(selectedPoint, speed=speed, tolerance=1,pidMode=True) # FlyToPosition behavior defined in behaviors.scenic
 
-    interrupt when (distance from self to drone1) < 20:
-        # when I see that I am within 20 meters of adversary, follow it
-        print("FOLLOW")
-        do Follow(drone1, speed=10, tolerance=1, offset=(0,0,0))
-    interrupt when distance from self to drone1 < 5:
-        # when I get within 5 meters of adversary, terminate scenario
-        print("TERMINATING")
+    interrupt when (distance from self to drone1) < 15:
+        # when I see that I am within 15 meters of adversary, follow it
+        print("FOLLOWING ADVERSARY")
+        do Follow(drone1, speed=10, tolerance=1, offset=(0,0,0)) # Follow behavior defined in behaviors.scenic
+
+    interrupt when distance from self to drone1 < 7:
+        # when I get within 7 meters of adversary, terminate scenario
+        print("ADVERSARY FOUND")
         terminate
 
 # Adversary behavior. Patrol through given points.
 behavior AdversaryBehavior(points, speed):
-    do Patrol(points, loop=True, speed=speed)
+    do Patrol(points, loop=True, speed=speed) # Patrol behavior defined in behaviors.scenic
 
 ground = getPrexistingObj("ground")
-centerArea = RectangularRegion(Vector(0,200,30), 0, 70, 70)
-platforms = []
+workspaceArea = RectangularRegion(Vector(0,200,30), 0, 100, 100)
 
+platforms = []
 blockCount = 4
 for i in range(blockCount):
     platforms.append(new StaticObj on ground, 
-        contained in centerArea,
+        contained in workspaceArea,
         with assetName "Cone", # use * to pick a random asset in assets
-        with width Range(3,10),
-        with length Range(3,10),
+        with parentOrientation 0,
+        with width 5,
+        with length 5,
         with height 10)
 
 points = []
 for plat in platforms:
     point = new Point on plat
     points.append(point.position)
 
-pt = Uniform(*points)
+adversarySpawn = Options(points)
 
 # Adversary drone spawning at random point
-drone1 = new AdversaryDrone at pt + (0,0,2),
+drone1 = new AdversaryDrone at adversarySpawn + (0,0,2),
     with behavior AdversaryBehavior(points, speed=5)
-drone1.patrolPoints = points
+# drone1.patrolPoints = possiblePoints
 drone1.patrolPointsProb = [0.4, 0.2, 0.1, 0.3] # Probability distribution on the patrolPoints
 
-ego = new Drone at (0,200,12),
+# ego drone spawning somwhere in the workspace area
+ego = new Drone at (0,Range(150, 250),12),
     with behavior FindAdversary(points, speed=5)
 
 
 # took too long to locate so terminate after x seconds
+terminate after 15 seconds 
 
-terminate after 30 seconds 
+record final (distance to drone1) as dist

safe_table.csv

@@ -0,0 +1,3 @@
+,point.objects.object0000.position[0],point.objects.object0000.position[1],point.objects.object0000.position[2],point.objects.object0183.position[0],point.objects.object0183.position[1],point.objects.object0183.position[2],point.objects.object0184.position[0],point.objects.object0184.position[1],point.objects.object0184.position[2],point.objects.object0185.position[0],point.objects.object0185.position[1],point.objects.object0185.position[2],point.objects.object0186.position[0],point.objects.object0186.position[1],point.objects.object0186.position[2],point.objects.object0187.position[0],point.objects.object0187.position[1],point.objects.object0187.position[2],rho
+0,0.0,162.34998806041605,12.0,32.35322950778509,191.21645519216037,4.980050019073486,32.99792474589637,168.59016143487472,4.980050019073486,6.179574530713317,229.88212673545544,4.980050019073486,-15.075412837133541,226.37856153712312,4.980050019073486,6.175796244525637,229.88111228360398,11.980050019073486,True
+1,0.0,226.35163451147866,12.0,15.166946852274549,229.99158345101318,4.980050019073486,-4.740123482403092,183.77021786853606,4.980050019073486,31.086869979097116,226.28247283256326,4.980050019073486,37.60092206004941,196.69124718720556,4.980050019073486,31.082877943494047,226.28047839062154,11.980050019073486,True

src/scenic/simulators/airsim/simulator.py

@@ -243,7 +243,7 @@ def destroy(self):
         client.reset()
 
         super().destroy()
-        print("canceled simulation")
+        print("Simulation Complete/Destroyed")
 
     def getProperties(self, obj, properties):
         if obj.blueprint == "AirSimPrexisting":

src/scenic/simulators/airsim/verifai_sampler.py

@@ -5,46 +5,42 @@
 from verifai.monitor import specification_monitor
 import math
 
-path_to_scenic_script = 'C:/Users/piegu/Scenic/examples/airsim/test.scenic'
+path_to_scenic_script = 'examples/airsim/demoDrone.scenic'
 sampler = ScenicSampler.fromScenario(path_to_scenic_script)
 
 class confidence_spec(specification_monitor):
     def __init__(self):
         def specification(simulation):
             for traj in simulation.result.trajectory:
-                # success condition is if the drone is within 5 meters of the target
-                dist = math.sqrt((traj[0][0]-traj[1][0])**2 + (traj[0][1]-traj[1][1])**2 + (traj[0][2]-traj[1][2])**2)
-                if dist < 5: 
+                # success condition is if the drone is within 7 meters of the target
+                dist = simulation.result.records['dist']
+                if dist < 7: 
                     return True
+            print("ADVERSARY NOT FOUND")
             return False
 
         super().__init__(specification)
 
-MAX_ITERS = 1
+MAX_ITERS = 3
 PORT = 8888
 MAXREQS = 5
 BUFSIZE = 4096
 falsifier_params = DotMap()
 falsifier_params.n_iters = MAX_ITERS # Number of simulations to run (or None for no limit)
-# falsifier_params.max_time = None # Time limit in seconds, if any
+falsifier_params.max_time = None # Time limit in seconds, if any
 falsifier_params.save_error_table = True # Record samples that violated the monitor/specification
 falsifier_params.save_good_samples = True # Don't record samples that satisfied the monitor/specification
 falsifier_params.error_table_path = 'error_table.csv'
-# falsifier_params.safe_table_path = 'safe_table.csv'
-# falsifier_params.fal_thres = 0.5 # Monitor return value below which a sample is considered a violation
-# falsifier_params.sampler_params = None   # optional DotMap of sampler-specific parameters
+falsifier_params.safe_table_path = 'safe_table.csv'
+falsifier_params.fal_thres = 0.5 # Monitor return value below which a sample is considered a violation
+falsifier_params.sampler_params = None   # optional DotMap of sampler-specific parameters
 
 server_options = DotMap(port=PORT, bufsize=BUFSIZE, maxreqs=MAXREQS, verbosity=1)
 
 falsifier = generic_falsifier(sampler=sampler,
                               falsifier_params=falsifier_params,
-                            #   monitor=confidence_spec(),
+                              monitor=confidence_spec(),
                               server_class=ScenicServer,
                               server_options=server_options)
 falsifier.run_falsifier()
-print("Scenic Samples")
-'''
-for i in falsifier.samples.keys():
-    print("Sample: ", i)
-    print(falsifier.samples[i])
-'''
+print("Sampler Complete")