.Net Processes: State Management in Processes Part 1 (#9424)

### Description This is the first one of a few PR that address the State Management in SK Processes. Dividing in smaller PRs since feature contain multiple big components. #### In this PR - **SAVE:** After running a Process, adding capability to extract process state for stateful processes - works also with nested processes - **LOAD:** from a Process State data structure, loading state to ProcessBuilder before `Build()` to apply it to Process and use it on runtime. For now only focusing on Step State only, not focusing on exact replica of processes (using same ids) - **Samples:** Adding more samples in Step02_FoodPrep that load existing State from file and use it to run specific food pre processes, also samples on how to save the state #### Out of Scope but coming up in future PRs - How to deal with processes running that have the same id (processid, stepid) - In the future, ProcessStep State will have versions, so if loading an old state to a process with a new state version there is: - validation - migration strategy placeholder to add custom state migration - A new way to create ProcessBuilders so that on save, processTypeIds can be interpreted instead of needing to pass all steps, and edges when loading a ProcessBuilder from file - current DAPR implementation - More samples showcasing more complex scenarios ### Contribution Checklist  - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone 😄
microsoft · Oct 28, 2024 · d2696a9 · d2696a9
1 parent ea5ceb1
commit d2696a9
Show file tree

Hide file tree

Showing 23 changed files with 587 additions and 51 deletions.
diff --git a/dotnet/samples/GettingStartedWithProcesses/Step03/Processes/FishSandwichProcess.cs b/dotnet/samples/GettingStartedWithProcesses/Step03/Processes/FishSandwichProcess.cs
@@ -46,6 +46,34 @@ public static ProcessBuilder CreateProcess(string processName = "FishSandwichPro
         return processBuilder;
     }
 
+    public static ProcessBuilder CreateProcessWithStatefulSteps(string processName = "FishSandwichWithStatefulStepsProcess")
+    {
+        var processBuilder = new ProcessBuilder(processName);
+        var makeFriedFishStep = processBuilder.AddStepFromProcess(FriedFishProcess.CreateProcessWithStatefulSteps());
+        var addBunsStep = processBuilder.AddStepFromType<AddBunsStep>();
+        var addSpecialSauceStep = processBuilder.AddStepFromType<AddSpecialSauceStep>();
+        // An additional step that is the only one that emits an public event in a process can be added to maintain event names unique
+        var externalStep = processBuilder.AddStepFromType<ExternalFriedFishStep>();
+
+        processBuilder
+            .OnInputEvent(ProcessEvents.PrepareFishSandwich)
+            .SendEventTo(makeFriedFishStep.WhereInputEventIs(FriedFishProcess.ProcessEvents.PrepareFriedFish));
+
+        makeFriedFishStep
+            .OnEvent(FriedFishProcess.ProcessEvents.FriedFishReady)
+            .SendEventTo(new ProcessFunctionTargetBuilder(addBunsStep));
+
+        addBunsStep
+            .OnEvent(AddBunsStep.OutputEvents.BunsAdded)
+            .SendEventTo(new ProcessFunctionTargetBuilder(addSpecialSauceStep));
+
+        addSpecialSauceStep
+            .OnEvent(AddSpecialSauceStep.OutputEvents.SpecialSauceAdded)
+            .SendEventTo(new ProcessFunctionTargetBuilder(externalStep));
+
+        return processBuilder;
+    }
+
     private sealed class AddBunsStep : KernelProcessStep
     {
         public static class Functions

diff --git a/...s/GettingStartedWithProcesses/Step03/ProcessesStates/FishSandwichStateProcessSuccess.json b/...s/GettingStartedWithProcesses/Step03/ProcessesStates/FishSandwichStateProcessSuccess.json
@@ -0,0 +1,42 @@
+{
+  "stepsState": {
+    "FriedFishWithStatefulStepsProcess": {
+      "stepsState": {
+        "GatherFriedFishIngredientsWithStockStep": {
+          "state": {
+            "IngredientsStock": 3
+          },
+          "id": "cfc7d51f3bfa455c980d282dcdd85b13",
+          "name": "GatherFriedFishIngredientsWithStockStep"
+        },
+        "chopStep": {
+          "state": {
+            "KnifeSharpness": 2
+          },
+          "id": "daf4a782df094deeb111ae701431ddbb",
+          "name": "chopStep"
+        },
+        "FryFoodStep": {
+          "id": "e508ffe1d4714097b9716db1fd48b587",
+          "name": "FryFoodStep"
+        }
+      },
+      "id": "a201334a1f534e9db9e1d46dce8345a8",
+      "name": "FriedFishWithStatefulStepsProcess"
+    },
+    "AddBunsStep": {
+      "id": "8a9b2d66e0594ee898d1c94c8bc07d0e",
+      "name": "AddBunsStep"
+    },
+    "AddSpecialSauceStep": {
+      "id": "6b0e92097cb74f5cbac2a71473a4e9c2",
+      "name": "AddSpecialSauceStep"
+    },
+    "ExternalFriedFishStep": {
+      "id": "59ebd4724684469ab19d86d281c205e3",
+      "name": "ExternalFriedFishStep"
+    }
+  },
+  "id": "38e8f477-c022-41fc-89f1-3dd3509d0e83",
+  "name": "FishSandwichWithStatefulStepsProcess"
+}
diff --git a/...gStartedWithProcesses/Step03/ProcessesStates/FishSandwichStateProcessSuccessLowStock.json b/...gStartedWithProcesses/Step03/ProcessesStates/FishSandwichStateProcessSuccessLowStock.json
@@ -0,0 +1,42 @@
+{
+  "stepsState": {
+    "FriedFishWithStatefulStepsProcess": {
+      "stepsState": {
+        "GatherFriedFishIngredientsWithStockStep": {
+          "state": {
+            "IngredientsStock": 1
+          },
+          "id": "cfc7d51f3bfa455c980d282dcdd85b13",
+          "name": "GatherFriedFishIngredientsWithStockStep"
+        },
+        "chopStep": {
+          "state": {
+            "KnifeSharpness": 2
+          },
+          "id": "daf4a782df094deeb111ae701431ddbb",
+          "name": "chopStep"
+        },
+        "FryFoodStep": {
+          "id": "e508ffe1d4714097b9716db1fd48b587",
+          "name": "FryFoodStep"
+        }
+      },
+      "id": "a201334a1f534e9db9e1d46dce8345a8",
+      "name": "FriedFishWithStatefulStepsProcess"
+    },
+    "AddBunsStep": {
+      "id": "8a9b2d66e0594ee898d1c94c8bc07d0e",
+      "name": "AddBunsStep"
+    },
+    "AddSpecialSauceStep": {
+      "id": "6b0e92097cb74f5cbac2a71473a4e9c2",
+      "name": "AddSpecialSauceStep"
+    },
+    "ExternalFriedFishStep": {
+      "id": "59ebd4724684469ab19d86d281c205e3",
+      "name": "ExternalFriedFishStep"
+    }
+  },
+  "id": "38e8f477-c022-41fc-89f1-3dd3509d0e83",
+  "name": "FishSandwichWithStatefulStepsProcess"
+}
diff --git a/...ples/GettingStartedWithProcesses/Step03/ProcessesStates/FriedFishProcessStateSuccess.json b/...ples/GettingStartedWithProcesses/Step03/ProcessesStates/FriedFishProcessStateSuccess.json
@@ -0,0 +1,24 @@
+{
+  "stepsState": {
+    "GatherFriedFishIngredientsWithStockStep": {
+      "state": {
+        "IngredientsStock": 4
+      },
+      "id": "51b39c78e71148c9af0a3c44c83bc6f5",
+      "name": "GatherFriedFishIngredientsWithStockStep"
+    },
+    "chopStep": {
+      "state": {
+        "KnifeSharpness": 4
+      },
+      "id": "6be5207cb71e42c28d7061e45e1127c1",
+      "name": "chopStep"
+    },
+    "FryFoodStep": {
+      "id": "9523d47a97a546908985a6ff028783cd",
+      "name": "FryFoodStep"
+    }
+  },
+  "id": "1c90a002-68ca-4c68-ac3b-a083054ed628",
+  "name": "FriedFishWithStatefulStepsProcess"
+}
diff --git a/...tingStartedWithProcesses/Step03/ProcessesStates/FriedFishProcessStateSuccessLowStock.json b/...tingStartedWithProcesses/Step03/ProcessesStates/FriedFishProcessStateSuccessLowStock.json
@@ -0,0 +1,24 @@
+{
+  "stepsState": {
+    "GatherFriedFishIngredientsWithStockStep": {
+      "state": {
+        "IngredientsStock": 1
+      },
+      "id": "51b39c78e71148c9af0a3c44c83bc6f5",
+      "name": "GatherFriedFishIngredientsWithStockStep"
+    },
+    "chopStep": {
+      "state": {
+        "KnifeSharpness": 4
+      },
+      "id": "6be5207cb71e42c28d7061e45e1127c1",
+      "name": "chopStep"
+    },
+    "FryFoodStep": {
+      "id": "9523d47a97a546908985a6ff028783cd",
+      "name": "FryFoodStep"
+    }
+  },
+  "id": "1c90a002-68ca-4c68-ac3b-a083054ed628",
+  "name": "FriedFishWithStatefulStepsProcess"
+}
diff --git a/...ttingStartedWithProcesses/Step03/ProcessesStates/FriedFishProcessStateSuccessNoStock.json b/...ttingStartedWithProcesses/Step03/ProcessesStates/FriedFishProcessStateSuccessNoStock.json
@@ -0,0 +1,24 @@
+{
+  "stepsState": {
+    "GatherFriedFishIngredientsWithStockStep": {
+      "state": {
+        "IngredientsStock": 0
+      },
+      "id": "51b39c78e71148c9af0a3c44c83bc6f5",
+      "name": "GatherFriedFishIngredientsWithStockStep"
+    },
+    "chopStep": {
+      "state": {
+        "KnifeSharpness": 4
+      },
+      "id": "6be5207cb71e42c28d7061e45e1127c1",
+      "name": "chopStep"
+    },
+    "FryFoodStep": {
+      "id": "9523d47a97a546908985a6ff028783cd",
+      "name": "FryFoodStep"
+    }
+  },
+  "id": "1c90a002-68ca-4c68-ac3b-a083054ed628",
+  "name": "FriedFishWithStatefulStepsProcess"
+}
diff --git a/dotnet/samples/GettingStartedWithProcesses/Step03/Step03a_FoodPreparation.cs b/dotnet/samples/GettingStartedWithProcesses/Step03/Step03a_FoodPreparation.cs
@@ -1,7 +1,9 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Process.Models;
 using Step03.Processes;
+using Utilities;
 
 namespace Step03;
 
@@ -101,7 +103,7 @@ public async Task UsePrepareStatefulPotatoFriesProcessSharedStateAsync()
         Console.WriteLine($"=== End SK Process '{processBuilder.Name}' ===");
     }
 
-    private async Task<KernelProcess> ExecuteProcessWithStateAsync(KernelProcess process, Kernel kernel, string externalTriggerEvent, string orderLabel)
+    private async Task<KernelProcess> ExecuteProcessWithStateAsync(KernelProcess process, Kernel kernel, string externalTriggerEvent, string orderLabel = "Order 1")
     {
         Console.WriteLine($"=== {orderLabel} ===");
         var runningProcess = await process.StartAsync(kernel, new KernelProcessEvent()
@@ -111,6 +113,100 @@ private async Task<KernelProcess> ExecuteProcessWithStateAsync(KernelProcess pro
         });
         return await runningProcess.GetStateAsync();
     }
+
+    #region Running processes and saving Process State Metadata in a file locally
+    [Fact]
+    public async Task RunAndStoreStatefulFriedFishProcessStateAsync()
+    {
+        Kernel kernel = CreateKernelWithChatCompletion();
+        ProcessBuilder builder = FriedFishProcess.CreateProcessWithStatefulSteps();
+        KernelProcess friedFishProcess = builder.Build();
+
+        var executedProcess = await ExecuteProcessWithStateAsync(friedFishProcess, kernel, externalTriggerEvent: FriedFishProcess.ProcessEvents.PrepareFriedFish);
+        var processState = executedProcess.ToProcessStateMetadata();
+        DumpProcessStateMetadataLocally(processState, _statefulFriedFishProcessFilename);
+    }
+
+    [Fact]
+    public async Task RunAndStoreStatefulFishSandwichProcessStateAsync()
+    {
+        Kernel kernel = CreateKernelWithChatCompletion();
+        ProcessBuilder builder = FishSandwichProcess.CreateProcessWithStatefulSteps();
+        KernelProcess friedFishProcess = builder.Build();
+
+        var executedProcess = await ExecuteProcessWithStateAsync(friedFishProcess, kernel, externalTriggerEvent: FishSandwichProcess.ProcessEvents.PrepareFishSandwich);
+        var processState = executedProcess.ToProcessStateMetadata();
+        DumpProcessStateMetadataLocally(processState, _statefulFishSandwichProcessFilename);
+    }
+    #endregion
+
+    #region Reading State from local file and apply to existing ProcessBuilder
+    [Fact]
+    public async Task RunStatefulFriedFishProcessFromFileAsync()
+    {
+        var processState = LoadProcessStateMetadata(this._statefulFriedFishProcessFilename);
+        Assert.NotNull(processState);
+
+        Kernel kernel = CreateKernelWithChatCompletion();
+        ProcessBuilder processBuilder = FriedFishProcess.CreateProcessWithStatefulSteps();
+        KernelProcess processFromFile = processBuilder.Build(processState);
+
+        await ExecuteProcessWithStateAsync(processFromFile, kernel, externalTriggerEvent: FriedFishProcess.ProcessEvents.PrepareFriedFish);
+    }
+
+    [Fact]
+    public async Task RunStatefulFriedFishProcessWithLowStockFromFileAsync()
+    {
+        var processState = LoadProcessStateMetadata(this._statefulFriedFishLowStockProcessFilename);
+        Assert.NotNull(processState);
+
+        Kernel kernel = CreateKernelWithChatCompletion();
+        ProcessBuilder processBuilder = FriedFishProcess.CreateProcessWithStatefulSteps();
+        KernelProcess processFromFile = processBuilder.Build(processState);
+
+        await ExecuteProcessWithStateAsync(processFromFile, kernel, externalTriggerEvent: FriedFishProcess.ProcessEvents.PrepareFriedFish);
+    }
+
+    [Fact]
+    public async Task RunStatefulFriedFishProcessWithNoStockFromFileAsync()
+    {
+        var processState = LoadProcessStateMetadata(this._statefulFriedFishNoStockProcessFilename);
+        Assert.NotNull(processState);
+
+        Kernel kernel = CreateKernelWithChatCompletion();
+        ProcessBuilder processBuilder = FriedFishProcess.CreateProcessWithStatefulSteps();
+        KernelProcess processFromFile = processBuilder.Build(processState);
+
+        await ExecuteProcessWithStateAsync(processFromFile, kernel, externalTriggerEvent: FriedFishProcess.ProcessEvents.PrepareFriedFish);
+    }
+
+    [Fact]
+    public async Task RunStatefulFishSandwichProcessFromFileAsync()
+    {
+        var processState = LoadProcessStateMetadata(this._statefulFishSandwichProcessFilename);
+        Assert.NotNull(processState);
+
+        Kernel kernel = CreateKernelWithChatCompletion();
+        ProcessBuilder processBuilder = FishSandwichProcess.CreateProcessWithStatefulSteps();
+        KernelProcess processFromFile = processBuilder.Build(processState);
+
+        await ExecuteProcessWithStateAsync(processFromFile, kernel, externalTriggerEvent: FishSandwichProcess.ProcessEvents.PrepareFishSandwich);
+    }
+
+    [Fact]
+    public async Task RunStatefulFishSandwichProcessWithLowStockFromFileAsync()
+    {
+        var processState = LoadProcessStateMetadata(this._statefulFishSandwichLowStockProcessFilename);
+        Assert.NotNull(processState);
+
+        Kernel kernel = CreateKernelWithChatCompletion();
+        ProcessBuilder processBuilder = FishSandwichProcess.CreateProcessWithStatefulSteps();
+        KernelProcess processFromFile = processBuilder.Build(processState);
+
+        await ExecuteProcessWithStateAsync(processFromFile, kernel, externalTriggerEvent: FishSandwichProcess.ProcessEvents.PrepareFishSandwich);
+    }
+    #endregion
+
     #endregion
     protected async Task UsePrepareSpecificProductAsync(ProcessBuilder processBuilder, string externalTriggerEvent)
     {
@@ -128,4 +224,29 @@ protected async Task UsePrepareSpecificProductAsync(ProcessBuilder processBuilde
         });
         Console.WriteLine($"=== End SK Process '{processBuilder.Name}' ===");
     }
+
+    // Step03a Utils for saving and loading SK Processes from/to repository
+    private readonly string _step03RelativePath = Path.Combine("Step03", "ProcessesStates");
+    private readonly string _statefulFriedFishProcessFilename = "FriedFishProcessStateSuccess.json";
+    private readonly string _statefulFriedFishLowStockProcessFilename = "FriedFishProcessStateSuccessLowStock.json";
+    private readonly string _statefulFriedFishNoStockProcessFilename = "FriedFishProcessStateSuccessNoStock.json";
+    private readonly string _statefulFishSandwichProcessFilename = "FishSandwichStateProcessSuccess.json";
+    private readonly string _statefulFishSandwichLowStockProcessFilename = "FishSandwichStateProcessSuccessLowStock.json";
+
+    private void DumpProcessStateMetadataLocally(KernelProcessStateMetadata processStateInfo, string jsonFilename)
+    {
+        var sampleRelativePath = GetSampleStep03Filepath(jsonFilename);
+        ProcessStateMetadataUtilities.DumpProcessStateMetadataLocally(processStateInfo, sampleRelativePath);
+    }
+
+    private KernelProcessStateMetadata? LoadProcessStateMetadata(string jsonFilename)
+    {
+        var sampleRelativePath = GetSampleStep03Filepath(jsonFilename);
+        return ProcessStateMetadataUtilities.LoadProcessStateMetadata(sampleRelativePath);
+    }
+
+    private string GetSampleStep03Filepath(string jsonFilename)
+    {
+        return Path.Combine(this._step03RelativePath, jsonFilename);
+    }
 }
diff --git a/dotnet/samples/GettingStartedWithProcesses/Step03/Steps/CutFoodWithSharpeningStep.cs b/dotnet/samples/GettingStartedWithProcesses/Step03/Steps/CutFoodWithSharpeningStep.cs
@@ -92,7 +92,7 @@ private string getActionString(string food, string action)
 /// </summary>
 public sealed class CutFoodWithSharpeningState
 {
-    internal int KnifeSharpness { get; set; } = 5;
+    public int KnifeSharpness { get; set; } = 5;
 
     internal int _needsSharpeningLimit = 3;
     internal int _sharpeningBoost = 5;

diff --git a/dotnet/samples/GettingStartedWithProcesses/Step03/Steps/GatherIngredientsStep.cs b/dotnet/samples/GettingStartedWithProcesses/Step03/Steps/GatherIngredientsStep.cs
@@ -120,5 +120,5 @@ public virtual async Task GatherIngredientsAsync(KernelProcessStepContext contex
 /// </summary>
 public sealed class GatherIngredientsState
 {
-    internal int IngredientsStock { get; set; } = 5;
+    public int IngredientsStock { get; set; } = 5;
 }