Add support for workflow labels (#498)

README.md

@@ -1,23 +1,25 @@
-# Welcome to Cromwell on Azure
-![Logo](/docs/screenshots/logo.png)
-
-[Cromwell](https://cromwell.readthedocs.io/en/stable/) is a workflow management system for scientific workflows, orchestrating the computing tasks needed for genomics analysis. Originally developed by the [Broad Institute](https://github.com/broadinstitute/cromwell), Cromwell is also used in the GATK Best Practices genome analysis pipeline. Cromwell supports running scripts at various scales, including your local machine, a local computing cluster, and on the cloud. <br />
-
-Cromwell on Azure configures all Azure resources needed to run workflows with Cromwell on the Microsoft Azure cloud, and uses the [GA4GH TES](https://cromwell.readthedocs.io/en/develop/backends/TES/) backend for orchestrating the tasks that create a workflow. The installation sets up an Azure Kubernetes cluster to run the Cromwell, TES, and Trigger Service containers, and uses the Azure Batch PaaS service to execute each task in a workflow in its own VM, enabling scale-out to thousands of machines. Cromwell workflows can be written using the [WDL](https://github.com/openwdl/wdl) scripting language. To see examples of WDL scripts - see this ['Learn WDL'](https://github.com/openwdl/learn-wdl) repository on GitHub.<br />
-
-### Latest release
- * https://github.com/microsoft/CromwellOnAzure/releases
-
-### Documentation
-All documentation has been moved to our [wiki](https://github.com/microsoft/CromwellOnAzure/wiki)! 
-
-[Getting Started?](https://github.com/microsoft/CromwellOnAzure/wiki/Getting-Started)
-
-[Got Questions?](https://github.com/microsoft/CromwellOnAzure/wiki/FAQ-And-Troubleshooting)
-
-### Want to Contribute?
-Check out our [contributing guidelines](https://github.com/microsoft/CromwellOnAzure/blob/main/docs/contributing.md) and [Code of Conduct](https://github.com/microsoft/CromwellOnAzure/blob/main/CODE_OF_CONDUCT.md) and submit a PR! We'd love to have you.
-
-## Related Projects
-
-[Genomics Data Analysis with Jupyter Notebooks on Azure](https://github.com/microsoft/genomicsnotebook)<br/>
+# Welcome to Cromwell on Azure
+![Logo](/docs/screenshots/logo.png)
+
+[Cromwell](https://cromwell.readthedocs.io/en/stable/) is a workflow management system for scientific workflows, orchestrating the computing tasks needed for genomics analysis. Originally developed by the [Broad Institute](https://github.com/broadinstitute/cromwell), Cromwell is also used in the GATK Best Practices genome analysis pipeline. Cromwell supports running scripts at various scales, including your local machine, a local computing cluster, and on the cloud. <br />
+
+Cromwell on Azure configures all Azure resources needed to run workflows with Cromwell on the Microsoft Azure cloud, and uses the [GA4GH TES](https://cromwell.readthedocs.io/en/develop/backends/TES/) backend for orchestrating the tasks that create a workflow. The installation sets up an Azure Kubernetes cluster to run the Cromwell, TES, and Trigger Service containers, and uses the Azure Batch PaaS service to execute each task in a workflow in its own VM, enabling scale-out to thousands of machines. Cromwell workflows can be written using the [WDL](https://github.com/openwdl/wdl) scripting language. To see examples of WDL scripts - see this ['Learn WDL'](https://github.com/openwdl/learn-wdl) repository on GitHub.<br />
+
+### Latest release
+ * https://github.com/microsoft/CromwellOnAzure/releases
+
+### Documentation
+All documentation has been moved to our [wiki](https://github.com/microsoft/CromwellOnAzure/wiki)!
+
+
+
+[Getting Started?](https://github.com/microsoft/CromwellOnAzure/wiki/Getting-Started)
+
+[Got Questions?](https://github.com/microsoft/CromwellOnAzure/wiki/FAQ-And-Troubleshooting)
+
+### Want to Contribute?
+Check out our [contributing guidelines](https://github.com/microsoft/CromwellOnAzure/blob/main/docs/contributing.md) and [Code of Conduct](https://github.com/microsoft/CromwellOnAzure/blob/main/CODE_OF_CONDUCT.md) and submit a PR! We'd love to have you.
+
+## Related Projects
+
+[Genomics Data Analysis with Jupyter Notebooks on Azure](https://github.com/microsoft/genomicsnotebook)<br/>

src/Common/Workflow.cs

@@ -12,6 +12,7 @@ public class Workflow
         public List<string> WorkflowInputsUrls { get; set; }
         public string WorkflowOptionsUrl { get; set; }
         public string WorkflowDependenciesUrl { get; set; }
+        public string WorkflowLabelsUrl { get; set; }
         public WorkflowFailureInfo WorkflowFailureInfo { get; set; }
         public List<TaskWarning> TaskWarnings { get; set; }
 

src/CromwellApiClient/CromwellApiClient.cs

@@ -63,7 +63,9 @@ public async Task<PostWorkflowResponse> PostWorkflowAsync(
             string workflowOptionsFilename = null,
             byte[] workflowOptionsData = null,
             string workflowDependenciesFilename = null,
-            byte[] workflowDependenciesData = null)
+            byte[] workflowDependenciesData = null,
+            string workflowLabelsFilename = null,
+            byte[] workflowLabelsData = null)
         {
             var files = AccumulatePostFiles(
                 workflowSourceFilename,
@@ -73,7 +75,9 @@ public async Task<PostWorkflowResponse> PostWorkflowAsync(
                 workflowOptionsFilename,
                 workflowOptionsData,
                 workflowDependenciesFilename,
-                workflowDependenciesData);
+                workflowDependenciesData,
+                workflowLabelsFilename,
+                workflowLabelsData);
             return await PostAsync<PostWorkflowResponse>(string.Empty, files);
         }
 
@@ -85,7 +89,9 @@ internal static List<FileToPost> AccumulatePostFiles(
             string workflowOptionsFilename = null,
             byte[] workflowOptionsData = null,
             string workflowDependenciesFilename = null,
-            byte[] workflowDependenciesData = null)
+            byte[] workflowDependenciesData = null,
+            string workflowLabelsFilename = null,
+            byte[] workflowLabelsData = null)
         {
             var files = new List<FileToPost> {
                 new(workflowSourceFilename, workflowSourceData, "workflowSource", removeTabs: true)
@@ -107,6 +113,11 @@ internal static List<FileToPost> AccumulatePostFiles(
                 files.Add(new(workflowDependenciesFilename, workflowDependenciesData, "workflowDependencies"));
             }
 
+            if (workflowLabelsFilename is not null && workflowLabelsData is not null)
+            {
+                files.Add(new FileToPost(workflowLabelsFilename, workflowLabelsData, "workflowLabels"));
+            }
+
             return files;
         }
 

src/CromwellApiClient/ICromwellApiClient.cs

@@ -14,7 +14,8 @@ Task<PostWorkflowResponse> PostWorkflowAsync(
             string workflowSourceFilename, byte[] workflowSourceData,
             List<string> workflowInputsFilename, List<byte[]> workflowInputsData,
             string workflowOptionsFilename = null, byte[] workflowOptionsData = null,
-            string workflowDependenciesFilename = null, byte[] workflowDependenciesData = null);
+            string workflowDependenciesFilename = null, byte[] workflowDependenciesData = null,
+            string workflowLabelsFilename = null, byte[] workflowLabelsData = null);
         Task<GetStatusResponse> GetStatusAsync(Guid id);
         Task<GetOutputsResponse> GetOutputsAsync(Guid id);
         Task<GetMetadataResponse> GetMetadataAsync(Guid id);

src/TriggerService.Tests/CromwellOnAzureEnvironmentTests.cs

@@ -225,6 +225,29 @@ public async Task ProcessBlobTrigger_WorkflowUrlNull()
                 }"
             );
 
+        [TestMethod]
+        public async Task ProcessBlobTrigger_OnlyWorkflowUrlSpecified()
+        {
+            var triggerFileContent =
+                @"{
+                    ""WorkflowUrl"":""" + fakeAzureWdl + @"""
+                }";
+
+            await ExecuteTriggerFileTest(triggerFileContent, 0);
+        }
+
+        [TestMethod]
+        public async Task ProcessBlobTrigger_OnlyWorkflowUrlAndInputsUrlSpecified()
+        {
+            var triggerFileContent =
+                @"{
+                    ""WorkflowUrl"":""" + fakeAzureWdl + @""",
+                    ""WorkflowInputsUrl"":""" + fakeAzureInput + @"""
+                }";
+
+            await ExecuteTriggerFileTest(triggerFileContent, 1);
+        }
+
         [TestMethod]
         public async Task ProcessBlobTrigger_NoInput()
         {
@@ -233,7 +256,8 @@ public async Task ProcessBlobTrigger_NoInput()
                     ""WorkflowUrl"":""" + fakeAzureWdl + @""",
                     ""WorkflowInputsUrl"":null,
                     ""WorkflowOptionsUrl"":null,
-                    ""WorkflowDependenciesUrl"":null
+                    ""WorkflowDependenciesUrl"":null,
+                    ""WorkflowLabelsUrl"":null
                 }";
 
             await ExecuteTriggerFileTest(triggerFileContent, 0);
@@ -247,7 +271,8 @@ public async Task ProcessBlobTrigger_SingleInput()
                     ""WorkflowUrl"":""" + fakeAzureWdl + @""",
                     ""WorkflowInputsUrl"":""" + fakeAzureInput + @""",
                     ""WorkflowOptionsUrl"":null,
-                    ""WorkflowDependenciesUrl"":null
+                    ""WorkflowDependenciesUrl"":null,
+                    ""WorkflowLabelsUrl"":null
                 }";
 
             await ExecuteTriggerFileTest(triggerFileContent, 1);
@@ -261,7 +286,8 @@ public async Task ProcessBlobTrigger_MultiInput()
                     ""WorkflowUrl"":""" + fakeAzureWdl + @""",
                     ""WorkflowInputsUrls"":" + JsonConvert.SerializeObject(fakeAzureInputs) + @",
                     ""WorkflowOptionsUrl"":null,
-                    ""WorkflowDependenciesUrl"":null
+                    ""WorkflowDependenciesUrl"":null,
+                    ""WorkflowLabelsUrl"":null
                 }";
 
             await ExecuteTriggerFileTest(triggerFileContent, fakeAzureInputs.Count);
@@ -276,7 +302,8 @@ public async Task ProcessBlobTrigger_CombinedInputs()
                     ""WorkflowInputsUrl"":""" + fakeAzureInput + @""",
                     ""WorkflowInputsUrls"":" + JsonConvert.SerializeObject(fakeAzureInputs) + @",
                     ""WorkflowOptionsUrl"":null,
-                    ""WorkflowDependenciesUrl"":null
+                    ""WorkflowDependenciesUrl"":null,
+                    ""WorkflowLabelsUrl"":null
                 }";
 
             await ExecuteTriggerFileTest(triggerFileContent, fakeAzureInputs.Count + 1);
@@ -291,7 +318,8 @@ public async Task ProcessBlobTrigger_SingleInputWithNull()
                     ""WorkflowInputsUrl"":""" + fakeAzureInput + @""",
                     ""WorkflowInputsUrls"":null,
                     ""WorkflowOptionsUrl"":null,
-                    ""WorkflowDependenciesUrl"":null
+                    ""WorkflowDependenciesUrl"":null,
+                    ""WorkflowLabelsUrl"":null
                 }";
 
             await ExecuteTriggerFileTest(triggerFileContent, 1);
@@ -306,7 +334,8 @@ public async Task ProcessBlobTrigger_MultiInputWithNull()
                     ""WorkflowInputsUrl"":null,
                     ""WorkflowInputsUrls"":" + JsonConvert.SerializeObject(fakeAzureInputs) + @",
                     ""WorkflowOptionsUrl"":null,
-                    ""WorkflowDependenciesUrl"":null
+                    ""WorkflowDependenciesUrl"":null,
+                    ""WorkflowLabelsUrl"":null
                 }";
 
             await ExecuteTriggerFileTest(triggerFileContent, fakeAzureInputs.Count);
@@ -321,7 +350,8 @@ public async Task ProcessBlobTrigger_AllInputsNull()
                     ""WorkflowInputsUrl"":null,
                     ""WorkflowInputsUrls"":null,
                     ""WorkflowOptionsUrl"":null,
-                    ""WorkflowDependenciesUrl"":null
+                    ""WorkflowDependenciesUrl"":null,
+                    ""WorkflowLabelsUrl"":null
                 }";
 
             await ExecuteTriggerFileTest(triggerFileContent, 0);

src/TriggerService.Tests/ProcessNewWorkflowTests.cs

@@ -32,7 +32,7 @@ public async Task NewWorkflowsAreMovedToInProgressSubdirectory()
             var cromwellApiClient = new Mock<ICromwellApiClient>();
 
             cromwellApiClient
-                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
+                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
                 .Returns(Task.FromResult(new PostWorkflowResponse { Id = workflowId }));
 
             var (newTriggerName, newTriggerContent) = await ProcessNewWorkflowAsync(cromwellApiClient.Object);
@@ -47,7 +47,7 @@ public async Task NewWorkflowsThatFailToPostToCromwellAreMovedToFailedSubdirecto
             var exceptionMessage = "Error submitting new workflow";
 
             cromwellApiClient
-                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
+                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
                 .Throws(new Exception(exceptionMessage));
 
             var (newTriggerName, newTriggerContent) = await ProcessNewWorkflowAsync(cromwellApiClient.Object);
@@ -143,7 +143,7 @@ public async Task NewWorkflowsThatFailToParseAsJsonAreAnotatedAndMovedToFailedSu
 
             var triesToPost = false;
             cromwellApiClient
-                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
+                .Setup(ac => ac.PostWorkflowAsync(It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<List<string>>(), It.IsAny<List<byte[]>>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>(), It.IsAny<string>(), It.IsAny<byte[]>()))
                 .Callback(() => triesToPost = true)
                 .Throws(new Exception("Should never get here."));
 

src/TriggerService/ProcessedTriggerInfo.cs

@@ -11,14 +11,16 @@ public class ProcessedTriggerInfo
         public List<ProcessedWorkflowItem> WorkflowInputs { get; private set; }
         public ProcessedWorkflowItem WorkflowOptions { get; private set; }
         public ProcessedWorkflowItem WorkflowDependencies { get; private set; }
+        public ProcessedWorkflowItem WorkflowLabels { get; private set; }
 
         public ProcessedTriggerInfo(ProcessedWorkflowItem workflowSource, List<ProcessedWorkflowItem> workflowInputs,
-            ProcessedWorkflowItem workflowOptions, ProcessedWorkflowItem workflowDependencies)
+            ProcessedWorkflowItem workflowOptions, ProcessedWorkflowItem workflowDependencies, ProcessedWorkflowItem workflowLabels)
         {
             this.WorkflowSource = workflowSource;
             this.WorkflowInputs = workflowInputs;
             this.WorkflowOptions = workflowOptions;
             this.WorkflowDependencies = workflowDependencies;
+            this.WorkflowLabels = workflowLabels;
         }
     }
 

src/TriggerService/TriggerHostedService.cs

@@ -192,8 +192,9 @@ internal async Task<ProcessedTriggerInfo> ProcessBlobTrigger(string blobTriggerJ
 
             var workflowOptions = await GetBlobFileNameAndData(triggerInfo.WorkflowOptionsUrl);
             var workflowDependencies = await GetBlobFileNameAndData(triggerInfo.WorkflowDependenciesUrl);
+            var workflowLabels = await GetBlobFileNameAndData(triggerInfo.WorkflowLabelsUrl);
 
-            return new(workflowSource, workflowInputs, workflowOptions, workflowDependencies);
+            return new(workflowSource, workflowInputs, workflowOptions, workflowDependencies, workflowLabels);
         }
 
         public async Task UpdateWorkflowStatusesAsync()
@@ -329,7 +330,7 @@ public async Task<ProcessedWorkflowItem> GetBlobFileNameAndData(string url)
 
             if (string.IsNullOrEmpty(blobName))
             {
-                throw new ArgumentException($"url object submitted ({url}) is not valid URL");
+                throw new ArgumentException($"Invalid URL: {url}");
             }
 
             byte[] data;