Merge pull request #918 from lsst-ts/tickets/DM-48100

DM-48100: Update Scheduler addBlock command to allow users to specify how script errors should be handled.

doc/news/interface_changes/DM-48100.scheduler.rst

@@ -0,0 +1,3 @@
+Add failureStrategy parameter to the addBlock command and blockStatus event.
+This parameter allows users to specify how the Scheduler should handle script failures when executing a block.
+

python/lsst/ts/xml/data/sal_interfaces/Scheduler/Scheduler_Commands.xml

@@ -64,6 +64,14 @@
       <Units>unitless</Units>
       <Count>1</Count>
     </item>
+    <item>
+      <EFDB_Name>failureStrategy</EFDB_Name>
+      <Description>What strategy to employ when there is a script failure?</Description>
+      <IDL_Type>long</IDL_Type>
+      <!-- <Enumeration> FailureStrategy </Enumeration> -->
+      <Units>unitless</Units>
+      <Count>1</Count>
+    </item>
   </SALCommand>
   <SALCommand>
     <Subsystem>Scheduler</Subsystem>

python/lsst/ts/xml/data/sal_interfaces/Scheduler/Scheduler_Events.xml

@@ -1,6 +1,8 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <?xml-stylesheet type="text/xsl" href="http://lsst-sal.tuc.noao.edu/schema/SALEventSet.xsl"?>
 <SALEventSet xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://lsst-sal.tuc.noao.edu/schema/SALEventSet.xsd">
+  <Enumeration>DetailedState_IDLE,DetailedState_RUNNING,DetailedState_WAITING_NEXT_TARGET_TIMER_TASK,DetailedState_GENERATING_TARGET_QUEUE,DetailedState_COMPUTING_PREDICTED_SCHEDULE,DetailedState_QUEUEING_TARGET</Enumeration>
+  <Enumeration>FailureStrategy_NONE,FailureStrategy_ON_SCRIPT_FAILURE,FailureStrategy_KEEP_GOING</Enumeration>
   <SALEvent>
     <Subsystem>Scheduler</Subsystem>
     <EFDB_Topic>Scheduler_logevent_detailedState</EFDB_Topic>
@@ -1350,6 +1352,14 @@
       <Units>unitless</Units>
       <Count>1</Count>
     </item>
+    <item>
+      <EFDB_Name>failureStrategy</EFDB_Name>
+      <Description>Strategy used when there is a script failure.</Description>
+      <!-- <Enumeration> FailureStrategy </Enumeration> -->
+      <IDL_Type>long</IDL_Type>
+      <Units>unitless</Units>
+      <Count>1</Count>
+    </item>
   </SALEvent>
   <SALEvent>
     <Subsystem>Scheduler</Subsystem>

python/lsst/ts/xml/enums/Scheduler.py

@@ -21,6 +21,7 @@
 __all__ = [
     "SalIndex",
     "DetailedState",
+    "FailureStrategy",
 ]
 
 import enum
@@ -54,3 +55,23 @@ class DetailedState(enum.IntEnum):
     COMPUTING_PREDICTED_SCHEDULE = enum.auto()
     # Scheduler is queueing targets.
     QUEUEING_TARGET = enum.auto()
+
+
+class FailureStrategy(enum.IntEnum):
+    """Strategy to employ when there is a script failure
+    while executing block.
+    """
+
+    # No error recoverable.
+    # This is the current behavior and causes the Scheduler to remove all
+    # scripts from the queue and mark the execution as failed.
+    NONE = enum.auto()
+    # Will preserve the scripts currently queue and continue to queue them
+    # if the queue makes progress. If a script is stopped the scheduler will
+    # cleanup the remaining scripts, stop the block execution and mark it as
+    # failed.
+    ON_SCRIPT_FAILURE = enum.auto()
+    # This will basically instruct the scheduler to ignore any type of error
+    # or interruption. It will continue to put scripts in the queue until all
+    # scripts from a block finished.
+    KEEP_GOING = enum.auto()