From 4274f78830a8a4c68e639429ae541165e9192eda Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Thu, 28 Nov 2024 13:56:49 +0530 Subject: [PATCH 01/15] create a function to validate the required fields --- .../kubernetes/deploy.ingress.yaml.template | 35 +++++++++++++++++-- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/Deployment/kubernetes/deploy.ingress.yaml.template b/Deployment/kubernetes/deploy.ingress.yaml.template index 1492ed68..81bb0604 100644 --- a/Deployment/kubernetes/deploy.ingress.yaml.template +++ b/Deployment/kubernetes/deploy.ingress.yaml.template @@ -1,3 +1,7 @@ +import yaml + +# Load the YAML file +ingress_yaml = """ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -21,7 +25,7 @@ spec: port: number: 5900 rules: - - host: {{ fqdn }} + - host: "{{ fqdn }}" http: paths: - path: /backend(/|$)(.*) @@ -40,5 +44,30 @@ spec: number: 5900 tls: - hosts: - - {{ fqdn }} - secretName: secret-kmgs \ No newline at end of file + - "{{ fqdn }}" + secretName: secret-kmgs +""" + +# Function to validate the required fields +def validate_ingress(ingress): + required_fields = { + 'fqdn': ingress['spec']['rules'][0]['host'], + 'cert-manager.io/cluster-issuer': ingress['metadata']['annotations'].get('cert-manager.io/cluster-issuer'), + 'ingressClassName': ingress['spec'].get('ingressClassName'), + 'frontapp-service': ingress['spec']['defaultBackend']['service']['name'], + 'aiservice-service': ingress['spec']['rules'][0]['http']['paths'][0]['backend']['service']['name'], + 'tls-secret-name': ingress['spec']['tls'][0].get('secretName') + } + + # Check each required field + for field, value in required_fields.items(): + if value in [None, '', '{{ fqdn }}']: + print(f"ERROR: The field '{field}' is missing or empty.") + else: + print(f"Field '{field}' is valid: {value}") + +# Load YAML data +ingress_data = yaml.safe_load(ingress_yaml) + +# Validate +validate_ingress(ingress_data) \ No newline at end of file From 8bc5dbc842c4004fae1058050e1f17d0d04bd1ba Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Thu, 28 Nov 2024 17:11:05 +0530 Subject: [PATCH 02/15] Revert the changes --- .../kubernetes/deploy.ingress.yaml.template | 35 ++----------------- 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/Deployment/kubernetes/deploy.ingress.yaml.template b/Deployment/kubernetes/deploy.ingress.yaml.template index 81bb0604..1492ed68 100644 --- a/Deployment/kubernetes/deploy.ingress.yaml.template +++ b/Deployment/kubernetes/deploy.ingress.yaml.template @@ -1,7 +1,3 @@ -import yaml - -# Load the YAML file -ingress_yaml = """ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: @@ -25,7 +21,7 @@ spec: port: number: 5900 rules: - - host: "{{ fqdn }}" + - host: {{ fqdn }} http: paths: - path: /backend(/|$)(.*) @@ -44,30 +40,5 @@ spec: number: 5900 tls: - hosts: - - "{{ fqdn }}" - secretName: secret-kmgs -""" - -# Function to validate the required fields -def validate_ingress(ingress): - required_fields = { - 'fqdn': ingress['spec']['rules'][0]['host'], - 'cert-manager.io/cluster-issuer': ingress['metadata']['annotations'].get('cert-manager.io/cluster-issuer'), - 'ingressClassName': ingress['spec'].get('ingressClassName'), - 'frontapp-service': ingress['spec']['defaultBackend']['service']['name'], - 'aiservice-service': ingress['spec']['rules'][0]['http']['paths'][0]['backend']['service']['name'], - 'tls-secret-name': ingress['spec']['tls'][0].get('secretName') - } - - # Check each required field - for field, value in required_fields.items(): - if value in [None, '', '{{ fqdn }}']: - print(f"ERROR: The field '{field}' is missing or empty.") - else: - print(f"Field '{field}' is valid: {value}") - -# Load YAML data -ingress_data = yaml.safe_load(ingress_yaml) - -# Validate -validate_ingress(ingress_data) \ No newline at end of file + - {{ fqdn }} + secretName: secret-kmgs \ No newline at end of file From 11b7f393a99feb5d165c6e5fa850194456d68c57 Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Thu, 28 Nov 2024 19:37:45 +0530 Subject: [PATCH 03/15] Implemented the required validation --- Deployment/resourcedeployment.ps1 | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/Deployment/resourcedeployment.ps1 b/Deployment/resourcedeployment.ps1 index ced60993..af21f250 100644 --- a/Deployment/resourcedeployment.ps1 +++ b/Deployment/resourcedeployment.ps1 @@ -605,12 +605,33 @@ try { # 6-2. Generate Unique backend API fqdn Name - esgdocanalysis-3 digit random number with padding 0 $dnsName = "kmgs$($(Get-Random -Minimum 0 -Maximum 9999).ToString("D4"))" + # Validate if the resource group name, public IP name and dns name are provided + if ([string]::IsNullOrEmpty($aksResourceGroupName)) { + Write-Host "Error: Resource Group name is null or empty." -ForegroundColor Red + exit 1 # Exit the script if the resource group name is missing + } + if ([string]::IsNullOrEmpty($publicIpName)) { + Write-Host "Error: Public IP name is null or empty." -ForegroundColor Red + exit 1 # Exit the script if the public IP name is missing + } + if ([string]::IsNullOrEmpty($dnsName)) { + Write-Host "Error: DNS name is null or empty." -ForegroundColor Red + exit 1 # Exit the script if the dns name is missing + } # 6-3. Assign DNS Name to the public IP address az network public-ip update --resource-group $aksResourceGroupName --name $publicIpName --dns-name $dnsName - # 6-4. Get FQDN for the public IP address - $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv - Write-Host "FQDN for the public IP address is: $fqdn" -ForegroundColor Green + # 6-4. Get FQDN for the public IP address + $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv + + # Validate if the FQDN is null or empty + if ([string]::IsNullOrEmpty($fqdn)) { + Write-Host "No FQDN is associated with the public IP address." -ForegroundColor Red + Exit 1 + } else { + Write-Host "FQDN for the public IP address is: $fqdn" -ForegroundColor Green + } + # 7. Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup Write-Host "Assign the role for aks system assigned managed identity to App Configuration Data Reader role" -ForegroundColor Green # Get vmss resource group name From f6e4ce290714e627d7b2a30a84a46076de556709 Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Mon, 2 Dec 2024 16:47:40 +0530 Subject: [PATCH 04/15] Implemented the validation with failure banner. --- Deployment/resourcedeployment.ps1 | 116 ++++++++++++++++++++++++++++-- 1 file changed, 110 insertions(+), 6 deletions(-) diff --git a/Deployment/resourcedeployment.ps1 b/Deployment/resourcedeployment.ps1 index af21f250..3037b42a 100644 --- a/Deployment/resourcedeployment.ps1 +++ b/Deployment/resourcedeployment.ps1 @@ -41,6 +41,23 @@ function successBanner(){ Write-Host " |_| |___/ " } +function failureBanner(){ + Write-Host " _____ _ _ " + Write-Host "| __ \ | | | | " + Write-Host "| | | | ___ _ __ | | ___ _ _ _ __ ___ ___ _ __ | |_ " + Write-Host "| | | |/ _ \ '_ \| |/ _ \| | | | '_ ` _ \ / _ \ '_ \| __| " + Write-Host "| |__| | __/ |_) | | (_) | |_| | | | | | | __/ | | | |_ " + Write-Host "|_____/ \___| .__/|_|\___/ \__, |_| |_| |_|\___|_| |_|\__| " + Write-Host " | | __/ | " + Write-Host " ______ _|_| _ |___/ " + Write-Host "| ____| (_) | | | " + Write-Host "| |__ __ _ _| | ___ __| | " + Write-Host "| __/ _` | | |/ _ \/ _` | " + Write-Host "| | | (_| | | | __/ (_| | " + Write-Host "|_| \__,_|_|_|\___|\__,_| " +} + + # Function to prompt for parameters with kind messages function PromptForParameters { param( @@ -395,10 +412,61 @@ try { ############################################################### # Get the storage account key $storageAccountKey = az storage account keys list --account-name $deploymentResult.StorageAccountName --resource-group $deploymentResult.ResourceGroupName --query "[0].value" -o tsv + # Validate if the storage account key is empty or null + if ([string]::IsNullOrEmpty($storageAccountKey)) { + Write-Host "Storage account key is empty. Exiting and Stopping execution..." -ForegroundColor Red + failureBanner + exit 1 # Exit the script due to Storage account key is empty. + } else { + Write-Host "Storage account key retrieved successfully." -ForegroundColor Green + } ## Construct the connection string manually $storageAccountConnectionString = "DefaultEndpointsProtocol=https;AccountName=$($deploymentResult.StorageAccountName);AccountKey=$storageAccountKey;EndpointSuffix=core.windows.net" + # Validate if the Storage Account Connection String is empty or null + if ([string]::IsNullOrEmpty($storageAccountConnectionString)) { + Write-Host "Storage Account Connection String is empty. Exiting and Stopping execution..." -ForegroundColor Red + failureBanner + exit 1 # Exit the script due to Storage Account Connection String is empty. + } else { + Write-Host "Storage Account Connection String retrieved successfully." -ForegroundColor Green + } ## Assign the connection string to the deployment result object - $deploymentResult.StorageAccountConnectionString = $storageAccountConnectionString + $deploymentResult.StorageAccountConnectionString = $storageAccountConnectionString + + # Check if ResourceGroupName is valid + if ([string]::IsNullOrEmpty($deploymentResult.ResourceGroupName)) { + Write-Host "Error: Resource group name is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } + + # Check if AzCosmosDBName is valid + if ([string]::IsNullOrEmpty($deploymentResult.AzCosmosDBName)) { + Write-Host "Error: Cosmos DB name is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } + + # Check if AzCognitiveServiceName is valid + if ([string]::IsNullOrEmpty($deploymentResult.AzCognitiveServiceName)) { + Write-Host "Error: Cognitive Service name is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } + + # Check if AzSearchServiceName is valid + if ([string]::IsNullOrEmpty($deploymentResult.AzSearchServiceName)) { + Write-Host "Error: Azure Search Service name is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } + + # Check if AzOpenAiServiceName is valid + if ([string]::IsNullOrEmpty($deploymentResult.AzOpenAiServiceName)) { + Write-Host "Error: OpenAI Service name is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } # Get MongoDB connection string $deploymentResult.AzCosmosDBConnectionString = az cosmosdb keys list --name $deploymentResult.AzCosmosDBName --resource-group $deploymentResult.ResourceGroupName --type connection-strings --query "connectionStrings[0].connectionString" -o tsv # Get Azure Cognitive Service API Key @@ -601,21 +669,22 @@ try { # 6-1. Get Az Network resource Name with the public IP address Write-Host "Assign DNS Name to the public IP address" -ForegroundColor Green $publicIpName=$(az network public-ip list --query "[?ipAddress=='$externalIP'].name" --output tsv) - # 6-2. Generate Unique backend API fqdn Name - esgdocanalysis-3 digit random number with padding 0 $dnsName = "kmgs$($(Get-Random -Minimum 0 -Maximum 9999).ToString("D4"))" - # Validate if the resource group name, public IP name and dns name are provided if ([string]::IsNullOrEmpty($aksResourceGroupName)) { - Write-Host "Error: Resource Group name is null or empty." -ForegroundColor Red + Write-Host "Error: aks Resource Group name is null or empty." -ForegroundColor Red + failureBanner exit 1 # Exit the script if the resource group name is missing } if ([string]::IsNullOrEmpty($publicIpName)) { Write-Host "Error: Public IP name is null or empty." -ForegroundColor Red + failureBanner exit 1 # Exit the script if the public IP name is missing } if ([string]::IsNullOrEmpty($dnsName)) { Write-Host "Error: DNS name is null or empty." -ForegroundColor Red + failureBanner exit 1 # Exit the script if the dns name is missing } # 6-3. Assign DNS Name to the public IP address @@ -623,10 +692,10 @@ try { # 6-4. Get FQDN for the public IP address $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv - # Validate if the FQDN is null or empty if ([string]::IsNullOrEmpty($fqdn)) { Write-Host "No FQDN is associated with the public IP address." -ForegroundColor Red + failureBanner Exit 1 } else { Write-Host "FQDN for the public IP address is: $fqdn" -ForegroundColor Green @@ -634,14 +703,49 @@ try { # 7. Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup Write-Host "Assign the role for aks system assigned managed identity to App Configuration Data Reader role" -ForegroundColor Green + # Ensure that the required fields are not null or empty + if ([string]::IsNullOrEmpty($deploymentResult.ResourceGroupName)) { + Write-Host "Error: Resource group name for AKS deployment is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } + if ([string]::IsNullOrEmpty($deploymentResult.AksName)) { + Write-Host "Error: AKS cluster name is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } + # Get vmss resource group name $vmssResourceGroupName = $(az aks show --resource-group $deploymentResult.ResourceGroupName --name $deploymentResult.AksName --query nodeResourceGroup --output tsv) + + # Validate if vmss Resource Group Name is null or empty + if ([string]::IsNullOrEmpty($vmssResourceGroupName)) { + Write-Host "Error: Unable to retrieve the VMSS resource group name." -ForegroundColor Red + failureBanner + exit 1 + } + # Get vmss name $vmssName = $(az vmss list --resource-group $vmssResourceGroupName --query "[0].name" --output tsv) + + # Validate if vmss Name is null or empty + if ([string]::IsNullOrEmpty($vmssName)) { + Write-Host "Error: Unable to retrieve the VMSS name." -ForegroundColor Red + failureBanner + exit 1 + } + # Create System Assigned Managed Identity $systemAssignedIdentity = $(az vmss identity assign --resource-group $vmssResourceGroupName --name $vmssName --query systemAssignedIdentity --output tsv) - + # Validate if System Assigned Identity is null or empty + if ([string]::IsNullOrEmpty($systemAssignedIdentity)) { + Write-Host "Error: Failed to assign system-assigned managed identity to the VMSS." -ForegroundColor Red + failureBanner + exit 1 + } else { + Write-Host "Retrived System Assigned Identity is: $systemAssignedIdentity" -ForegroundColor Green + } # Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup az role assignment create --assignee $systemAssignedIdentity --role "App Configuration Data Reader" --scope $deploymentResult.ResourceGroupId From 766ff86a4305bf351264cd37f51e0eacb3beaa74 Mon Sep 17 00:00:00 2001 From: Johnson Tseng Date: Wed, 4 Dec 2024 17:10:16 -0500 Subject: [PATCH 05/15] refactor to allowKeywordExtractingHandler to read from prompt txt --- .../service/Abstractions/Constants.cs | 1 + App/kernel-memory/service/Core/Core.csproj | 1 + .../Core/Handlers/KeywordExtractingHandler.cs | 32 +++++-------------- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/App/kernel-memory/service/Abstractions/Constants.cs b/App/kernel-memory/service/Abstractions/Constants.cs index 53d5535e..d2e516f7 100644 --- a/App/kernel-memory/service/Abstractions/Constants.cs +++ b/App/kernel-memory/service/Abstractions/Constants.cs @@ -163,4 +163,5 @@ public static class Summary // Standard prompt names public const string PromptNamesSummarize = "summarize"; public const string PromptNamesAnswerWithFacts = "answer-with-facts"; + public const string PromptNamesExtractKeywords = "extract-keywords"; } diff --git a/App/kernel-memory/service/Core/Core.csproj b/App/kernel-memory/service/Core/Core.csproj index 71cec67b..c3ab4dd3 100644 --- a/App/kernel-memory/service/Core/Core.csproj +++ b/App/kernel-memory/service/Core/Core.csproj @@ -42,6 +42,7 @@ + diff --git a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs index b5f95c9b..df66dedf 100644 --- a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs @@ -11,6 +11,7 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using static Microsoft.KernelMemory.Pipeline.DataPipeline; +using Microsoft.KernelMemory.Prompts; namespace Microsoft.KernelMemory.Handlers; @@ -21,11 +22,13 @@ public sealed class KeywordExtractingHandler : IPipelineStepHandler private readonly IPipelineOrchestrator _orchestrator; private readonly Kernel _kernel; private readonly KernelMemoryConfig? _config = null; + private readonly string _extractKeywordPrompt; public KeywordExtractingHandler( string stepName, IPipelineOrchestrator orchestrator, KernelMemoryConfig config = null, + IPromptProvider? promptProvider = null, ILoggerFactory? loggerFactory = null ) { @@ -34,6 +37,10 @@ public KeywordExtractingHandler( this._orchestrator = orchestrator; this._config = config; + promptProvider ??= new EmbeddedPromptProvider(); + + this._extractKeywordPrompt = promptProvider.ReadPrompt(Constants.PromptNamesExtractKeywords); + //init Semantic Kernel this._kernel = Kernel.CreateBuilder() .AddAzureOpenAIChatCompletion(deploymentName: (string)this._config.Services["AzureOpenAIText"]["Deployment"], @@ -79,30 +86,7 @@ public KeywordExtractingHandler( var chat = this._kernel.GetRequiredService(); var chatHistory = new ChatHistory(); - var systemMessage = """ - You are an assistant to analyze Content and Extract Tags by Content. - [EXTRACT TAGS RULES] - IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND TAGS - TAGS SHOULD BE CATEGORY SPECIFIC - TAGS SHOULD BE A LIST OF STRINGS - TAGS COUNT CAN BE UP TO 10 UNDER A CATEGORY - CATEGORY COUNT CAN BE UP TO 10 - DON'T ADD ANY MARKDOWN EXPRESSION IN YOUR RESPONSE - [END RULES] - - [EXAMPLE] - [ - { - [category1": ["tag1", "tag2", "tag3"] - }, - { - "category2": ["tag1", "tag2", "tag3"] - } - ] - [END EXAMPLE] - """; - - chatHistory.AddSystemMessage(systemMessage); + chatHistory.AddSystemMessage(this._extractKeywordPrompt); chatHistory.AddUserMessage($"Extract tags from this content : {extactedFileContent} \n The format should be Json but Markdown expression."); var executionParam = new PromptExecutionSettings() From 8d2d66caeac103176765e1d70e3b68f9bcc88a22 Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Thu, 5 Dec 2024 15:51:57 +0530 Subject: [PATCH 06/15] Added a common function to validate the required variables --- Deployment/resourcedeployment.ps1 | 143 ++++++++++-------------------- 1 file changed, 46 insertions(+), 97 deletions(-) diff --git a/Deployment/resourcedeployment.ps1 b/Deployment/resourcedeployment.ps1 index 3037b42a..d68ed05a 100644 --- a/Deployment/resourcedeployment.ps1 +++ b/Deployment/resourcedeployment.ps1 @@ -57,7 +57,19 @@ function failureBanner(){ Write-Host "|_| \__,_|_|_|\___|\__,_| " } - +# Common function to check if a variable is null or empty +function ValidateVariableIsNullOrEmpty { + param ( + [string]$variableValue, + [string]$variableName + ) + + if ([string]::IsNullOrEmpty($variableValue)) { + Write-Host "Error: $variableName is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } +} # Function to prompt for parameters with kind messages function PromptForParameters { param( @@ -412,61 +424,33 @@ try { ############################################################### # Get the storage account key $storageAccountKey = az storage account keys list --account-name $deploymentResult.StorageAccountName --resource-group $deploymentResult.ResourceGroupName --query "[0].value" -o tsv + # Validate if the storage account key is empty or null - if ([string]::IsNullOrEmpty($storageAccountKey)) { - Write-Host "Storage account key is empty. Exiting and Stopping execution..." -ForegroundColor Red - failureBanner - exit 1 # Exit the script due to Storage account key is empty. - } else { - Write-Host "Storage account key retrieved successfully." -ForegroundColor Green - } + ValidateVariableIsNullOrEmpty -variableValue $storageAccountKey -variableName "Storage account key" + ## Construct the connection string manually $storageAccountConnectionString = "DefaultEndpointsProtocol=https;AccountName=$($deploymentResult.StorageAccountName);AccountKey=$storageAccountKey;EndpointSuffix=core.windows.net" # Validate if the Storage Account Connection String is empty or null - if ([string]::IsNullOrEmpty($storageAccountConnectionString)) { - Write-Host "Storage Account Connection String is empty. Exiting and Stopping execution..." -ForegroundColor Red - failureBanner - exit 1 # Exit the script due to Storage Account Connection String is empty. - } else { - Write-Host "Storage Account Connection String retrieved successfully." -ForegroundColor Green - } + ValidateVariableIsNullOrEmpty -variableValue $storageAccountConnectionString -variableName "Storage Account Connection String" + ## Assign the connection string to the deployment result object $deploymentResult.StorageAccountConnectionString = $storageAccountConnectionString - # Check if ResourceGroupName is valid - if ([string]::IsNullOrEmpty($deploymentResult.ResourceGroupName)) { - Write-Host "Error: Resource group name is null or empty." -ForegroundColor Red - failureBanner - exit 1 - } - + # Check if ResourceGroupName is valid + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name" + # Check if AzCosmosDBName is valid - if ([string]::IsNullOrEmpty($deploymentResult.AzCosmosDBName)) { - Write-Host "Error: Cosmos DB name is null or empty." -ForegroundColor Red - failureBanner - exit 1 - } - + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzCosmosDBName -variableName "Az Cosmos DB name" + # Check if AzCognitiveServiceName is valid - if ([string]::IsNullOrEmpty($deploymentResult.AzCognitiveServiceName)) { - Write-Host "Error: Cognitive Service name is null or empty." -ForegroundColor Red - failureBanner - exit 1 - } - + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzCognitiveServiceName -variableName "Az Cognitive Service name" + # Check if AzSearchServiceName is valid - if ([string]::IsNullOrEmpty($deploymentResult.AzSearchServiceName)) { - Write-Host "Error: Azure Search Service name is null or empty." -ForegroundColor Red - failureBanner - exit 1 - } - + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzSearchServiceName -variableName "Az Search Service name" + # Check if AzOpenAiServiceName is valid - if ([string]::IsNullOrEmpty($deploymentResult.AzOpenAiServiceName)) { - Write-Host "Error: OpenAI Service name is null or empty." -ForegroundColor Red - failureBanner - exit 1 - } + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzOpenAiServiceName -variableName "Az OpenAI Service name" + # Get MongoDB connection string $deploymentResult.AzCosmosDBConnectionString = az cosmosdb keys list --name $deploymentResult.AzCosmosDBName --resource-group $deploymentResult.ResourceGroupName --type connection-strings --query "connectionStrings[0].connectionString" -o tsv # Get Azure Cognitive Service API Key @@ -671,81 +655,46 @@ try { $publicIpName=$(az network public-ip list --query "[?ipAddress=='$externalIP'].name" --output tsv) # 6-2. Generate Unique backend API fqdn Name - esgdocanalysis-3 digit random number with padding 0 $dnsName = "kmgs$($(Get-Random -Minimum 0 -Maximum 9999).ToString("D4"))" - # Validate if the resource group name, public IP name and dns name are provided - if ([string]::IsNullOrEmpty($aksResourceGroupName)) { - Write-Host "Error: aks Resource Group name is null or empty." -ForegroundColor Red - failureBanner - exit 1 # Exit the script if the resource group name is missing - } - if ([string]::IsNullOrEmpty($publicIpName)) { - Write-Host "Error: Public IP name is null or empty." -ForegroundColor Red - failureBanner - exit 1 # Exit the script if the public IP name is missing - } - if ([string]::IsNullOrEmpty($dnsName)) { - Write-Host "Error: DNS name is null or empty." -ForegroundColor Red - failureBanner - exit 1 # Exit the script if the dns name is missing - } + + # Validate if the AKS Resource Group Name, Public IP name and DNS Name are provided + ValidateVariableIsNullOrEmpty -variableValue $aksResourceGroupName -variableName "AKS Resource Group name" + + ValidateVariableIsNullOrEmpty -variableValue $publicIpName -variableName "Public IP name" + + ValidateVariableIsNullOrEmpty -variableValue $dnsName -variableName "DNS Name" + # 6-3. Assign DNS Name to the public IP address az network public-ip update --resource-group $aksResourceGroupName --name $publicIpName --dns-name $dnsName # 6-4. Get FQDN for the public IP address $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv # Validate if the FQDN is null or empty - if ([string]::IsNullOrEmpty($fqdn)) { - Write-Host "No FQDN is associated with the public IP address." -ForegroundColor Red - failureBanner - Exit 1 - } else { - Write-Host "FQDN for the public IP address is: $fqdn" -ForegroundColor Green - } + ValidateVariableIsNullOrEmpty -variableValue $fqdn -variableName "FQDN" # 7. Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup Write-Host "Assign the role for aks system assigned managed identity to App Configuration Data Reader role" -ForegroundColor Green # Ensure that the required fields are not null or empty - if ([string]::IsNullOrEmpty($deploymentResult.ResourceGroupName)) { - Write-Host "Error: Resource group name for AKS deployment is null or empty." -ForegroundColor Red - failureBanner - exit 1 - } - if ([string]::IsNullOrEmpty($deploymentResult.AksName)) { - Write-Host "Error: AKS cluster name is null or empty." -ForegroundColor Red - failureBanner - exit 1 - } + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name for AKS deployment" + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AksName -variableName "AKS cluster name" + # Get vmss resource group name $vmssResourceGroupName = $(az aks show --resource-group $deploymentResult.ResourceGroupName --name $deploymentResult.AksName --query nodeResourceGroup --output tsv) # Validate if vmss Resource Group Name is null or empty - if ([string]::IsNullOrEmpty($vmssResourceGroupName)) { - Write-Host "Error: Unable to retrieve the VMSS resource group name." -ForegroundColor Red - failureBanner - exit 1 - } - + ValidateVariableIsNullOrEmpty -variableValue $vmssResourceGroupName -variableName "VMSS resource group" + # Get vmss name $vmssName = $(az vmss list --resource-group $vmssResourceGroupName --query "[0].name" --output tsv) # Validate if vmss Name is null or empty - if ([string]::IsNullOrEmpty($vmssName)) { - Write-Host "Error: Unable to retrieve the VMSS name." -ForegroundColor Red - failureBanner - exit 1 - } - + ValidateVariableIsNullOrEmpty -variableValue $vmssName -variableName "VMSS name" + # Create System Assigned Managed Identity $systemAssignedIdentity = $(az vmss identity assign --resource-group $vmssResourceGroupName --name $vmssName --query systemAssignedIdentity --output tsv) # Validate if System Assigned Identity is null or empty - if ([string]::IsNullOrEmpty($systemAssignedIdentity)) { - Write-Host "Error: Failed to assign system-assigned managed identity to the VMSS." -ForegroundColor Red - failureBanner - exit 1 - } else { - Write-Host "Retrived System Assigned Identity is: $systemAssignedIdentity" -ForegroundColor Green - } + ValidateVariableIsNullOrEmpty -variableValue $systemAssignedIdentity -variableName "System-assigned managed identity" # Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup az role assignment create --assignee $systemAssignedIdentity --role "App Configuration Data Reader" --scope $deploymentResult.ResourceGroupId From bb89f62fdeb6b7aaa757600a046eeae55966cc16 Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Thu, 5 Dec 2024 19:17:01 +0530 Subject: [PATCH 07/15] Added the correct variable name --- Deployment/resourcedeployment.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Deployment/resourcedeployment.ps1 b/Deployment/resourcedeployment.ps1 index d68ed05a..b3806cd5 100644 --- a/Deployment/resourcedeployment.ps1 +++ b/Deployment/resourcedeployment.ps1 @@ -667,14 +667,15 @@ try { az network public-ip update --resource-group $aksResourceGroupName --name $publicIpName --dns-name $dnsName # 6-4. Get FQDN for the public IP address - $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv + #$fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv + $fqdn = $null # Validate if the FQDN is null or empty ValidateVariableIsNullOrEmpty -variableValue $fqdn -variableName "FQDN" # 7. Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup Write-Host "Assign the role for aks system assigned managed identity to App Configuration Data Reader role" -ForegroundColor Green # Ensure that the required fields are not null or empty - ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name for AKS deployment" + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name" ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AksName -variableName "AKS cluster name" From 928971b24f328ba2a8873bca84d18738946f5106 Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Fri, 6 Dec 2024 13:29:18 +0530 Subject: [PATCH 08/15] remove $fqdn= $null --- Deployment/resourcedeployment.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Deployment/resourcedeployment.ps1 b/Deployment/resourcedeployment.ps1 index b3806cd5..35a5342a 100644 --- a/Deployment/resourcedeployment.ps1 +++ b/Deployment/resourcedeployment.ps1 @@ -667,8 +667,8 @@ try { az network public-ip update --resource-group $aksResourceGroupName --name $publicIpName --dns-name $dnsName # 6-4. Get FQDN for the public IP address - #$fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv - $fqdn = $null + $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv + # Validate if the FQDN is null or empty ValidateVariableIsNullOrEmpty -variableValue $fqdn -variableName "FQDN" From 20b93abad13c28b29cd3728a7784e3780db1a085 Mon Sep 17 00:00:00 2001 From: Prasanjeet-Microsoft Date: Wed, 11 Dec 2024 00:51:46 +0530 Subject: [PATCH 09/15] Added CodeOwners, PRTemplate and StaleBot --- .github/CODEOWNERS | 5 +++++ .github/PULL_REQUEST_TEMPLATE.md | 37 ++++++++++++++++++++++++++++++++ .github/workflows/stale-bot.yml | 19 ++++++++++++++++ 3 files changed, 61 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/stale-bot.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..396d5c29 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in the repo. +* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..22b04b72 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,37 @@ +## Purpose + +* ... + +## Does this introduce a breaking change? + + +- [ ] Yes +- [ ] No + + + +## Golden Path Validation +- [ ] I have tested the primary workflows (the "golden path") to ensure they function correctly without errors. + +## Deployment Validation +- [ ] I have validated the deployment process successfully and all services are running as expected with this change. + +## What to Check +Verify that the following are valid +* ... + +## Other Information + \ No newline at end of file diff --git a/.github/workflows/stale-bot.yml b/.github/workflows/stale-bot.yml new file mode 100644 index 00000000..e31059ab --- /dev/null +++ b/.github/workflows/stale-bot.yml @@ -0,0 +1,19 @@ +name: 'Close stale issues and PRs' +on: + schedule: + - cron: '30 1 * * *' + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue is stale because it has been open 180 days with no activity. Remove stale label or comment or this will be closed in 30 days.' + days-before-stale: 180 + days-before-close: 30 From f8ca0001ec6ddf1d2f9eb44dbec8595955da5bdd Mon Sep 17 00:00:00 2001 From: Prasanjeet-Microsoft Date: Wed, 11 Dec 2024 22:50:27 +0530 Subject: [PATCH 10/15] Updated CodeOwners --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 396d5c29..fded4435 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,4 @@ # Each line is a file pattern followed by one or more owners. # These owners will be the default owners for everything in the repo. -* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft \ No newline at end of file +* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @dongbumlee \ No newline at end of file From 72e1cbb9d3770604c8691c162056a4ba06af116f Mon Sep 17 00:00:00 2001 From: Prasanjeet-Microsoft Date: Fri, 13 Dec 2024 12:49:11 +0530 Subject: [PATCH 11/15] Added ISSUE_TEMPLATE --- .github/ISSUE_TEMPLATE/bug_report.md | 45 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 32 ++++++++++++++++ .github/ISSUE_TEMPLATE/subtask.md | 22 +++++++++++ 3 files changed, 99 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/subtask.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..882ebd79 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,45 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +# Describe the bug +A clear and concise description of what the bug is. + +# Expected behavior +A clear and concise description of what you expected to happen. + +# How does this bug make you feel? +_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel_ + +--- + +# Debugging information + +## Steps to reproduce +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +## Screenshots +If applicable, add screenshots to help explain your problem. + +## Logs + +If applicable, add logs to help the engineer debug the problem. + +--- + +# Tasks + +_To be filled in by the engineer picking up the issue_ + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..3496fc82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +# Motivation + +A clear and concise description of why this feature would be useful and the value it would bring. +Explain any alternatives considered and why they are not sufficient. + +# How would you feel if this feature request was implemented? + +_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel. Format: ![alt_text](https://media.giphy.com/media/xxx/giphy.gif)_ + +# Requirements + +A list of requirements to consider this feature delivered +- Requirement 1 +- Requirement 2 +- ... + +# Tasks + +_To be filled in by the engineer picking up the issue_ + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/subtask.md b/.github/ISSUE_TEMPLATE/subtask.md new file mode 100644 index 00000000..9f86c843 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/subtask.md @@ -0,0 +1,22 @@ +--- +name: Sub task +about: A sub task +title: '' +labels: subtask +assignees: '' + +--- + +Required by + +# Description + +A clear and concise description of what this subtask is. + +# Tasks + +_To be filled in by the engineer picking up the subtask + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... From d13581a9951a037c08f86d26c1fc3bdabc920cb1 Mon Sep 17 00:00:00 2001 From: Prasanjeet-Microsoft Date: Mon, 16 Dec 2024 21:20:35 +0530 Subject: [PATCH 12/15] Align extract-keywords.txt Prompt with Hardcoded System Message --- .../service/Core/Prompts/extract-keywords.txt | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/App/kernel-memory/service/Core/Prompts/extract-keywords.txt b/App/kernel-memory/service/Core/Prompts/extract-keywords.txt index 7ecd1689..9dd6485c 100644 --- a/App/kernel-memory/service/Core/Prompts/extract-keywords.txt +++ b/App/kernel-memory/service/Core/Prompts/extract-keywords.txt @@ -1,21 +1,25 @@ -[EXTRACT KEYWORDS RULES] -IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND KEYWORDS -KEYWORDS SHOULD BE CATEGORY SPECIFIC -KEYWORDS SHOULD BE A LIST OF STRINGS -KEYWORDS COUNT CAN BE UP TO 50 +You are an assistant to analyze Content and Extract Tags by Content. +[EXTRACT TAGS RULES] +IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND TAGS +TAGS SHOULD BE CATEGORY SPECIFIC +TAGS SHOULD BE A LIST OF STRINGS +TAGS COUNT CAN BE UP TO 10 UNDER A CATEGORY CATEGORY COUNT CAN BE UP TO 10 +DON'T ADD ANY MARKDOWN EXPRESSION IN YOUR RESPONSE [END RULES] - [EXAMPLE] [ { - "category1": ["keyword1", "keyword2", "keyword3"] + "category1": ["tag1", "tag2", "tag3"] }, { - "category2": ["keyword1", "keyword2", "keyword3"] + "category2": ["tag1", "tag2", "tag3"] } ] [END EXAMPLE] -Extract Keywords from this : -{{$input}} \ No newline at end of file +Extract Tags from this : +{{$input}} + + + From e49ed7e0b9757d6da4523d54e5041fae58fa9995 Mon Sep 17 00:00:00 2001 From: Prasanjeet-Microsoft Date: Thu, 19 Dec 2024 01:16:42 +0530 Subject: [PATCH 13/15] Added PR title checker GitHub Actions workflow --- .github/workflows/pr-title-checker.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/pr-title-checker.yml diff --git a/.github/workflows/pr-title-checker.yml b/.github/workflows/pr-title-checker.yml new file mode 100644 index 00000000..5cbbae1a --- /dev/null +++ b/.github/workflows/pr-title-checker.yml @@ -0,0 +1,22 @@ +name: "pr-title-checker" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + merge_group: + +permissions: + pull-requests: read + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + if: ${{ github.event_name != 'merge_group' }} + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 9b9c8e848eb002ccc9863928052879ddc93799f4 Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Tue, 24 Dec 2024 16:13:33 +0530 Subject: [PATCH 14/15] Update dspapi_build.yaml for code coverage report --- App/backend-api/pipelines/dspapi_build.yaml | 49 ++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/App/backend-api/pipelines/dspapi_build.yaml b/App/backend-api/pipelines/dspapi_build.yaml index 681122d0..f36d101c 100644 --- a/App/backend-api/pipelines/dspapi_build.yaml +++ b/App/backend-api/pipelines/dspapi_build.yaml @@ -15,7 +15,7 @@ stages: - task: UseDotNet@2 inputs: packageType: 'sdk' - version: '9.0.x' # Use the appropriate .NET 9 version + version: '9.0.x' # Use the appropriate .NET 9 version installationPath: $(Agent.ToolsDirectory)/dotnet includePreviewVersions: true @@ -41,3 +41,50 @@ stages: Dockerfile: '$(Build.SourcesDirectory)/Dockerfile' tags: | $(Build.BuildId) + +- stage: TestAndCoverage + jobs: + - job: Test + steps: + - task: UseDotNet@2 + inputs: + packageType: 'sdk' + version: '9.0.x' # Use the appropriate .NET 9 version + installationPath: $(Agent.ToolsDirectory)/dotnet + includePreviewVersions: true + + - script: | + dotnet restore + displayName: 'Restore dependencies' + + - script: | + dotnet test --configuration Release --collect:"XPlat Code Coverage" --results-directory $(Build.ArtifactStagingDirectory)/TestResults + displayName: 'Run tests and collect code coverage' + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory)/TestResults + artifactName: 'CodeCoverage' + publishLocation: 'Container' + + - task: Docker@2 + displayName: Login in ACR + inputs: + command: login + containerRegistry: 'dps-acr-connection' + + - task: Docker@2 + inputs: + containerRegistry: 'dps-acr-connection' + repository: '$(imageName)' + command: 'buildAndPush' + Dockerfile: '$(Build.SourcesDirectory)/Dockerfile' + tags: | + $(Build.BuildId) + + - task: Codecov@1 + inputs: + token: $(CODECOV_TOKEN) # Add your Codecov token as a secret + file: '$(Build.ArtifactStagingDirectory)/TestResults/*/coverage.cobertura.xml' # Path to coverage file + commit: $(Build.SourceVersion) + name: $(Build.BuildId) From de60329c47895867ff50297e60b6deca6f6881c5 Mon Sep 17 00:00:00 2001 From: pradeepjha-microsoft Date: Fri, 27 Dec 2024 15:43:02 +0530 Subject: [PATCH 15/15] Remove TestAndCoverage stage from pipeline --- App/backend-api/pipelines/dspapi_build.yaml | 49 +-------------------- 1 file changed, 1 insertion(+), 48 deletions(-) diff --git a/App/backend-api/pipelines/dspapi_build.yaml b/App/backend-api/pipelines/dspapi_build.yaml index f36d101c..681122d0 100644 --- a/App/backend-api/pipelines/dspapi_build.yaml +++ b/App/backend-api/pipelines/dspapi_build.yaml @@ -15,7 +15,7 @@ stages: - task: UseDotNet@2 inputs: packageType: 'sdk' - version: '9.0.x' # Use the appropriate .NET 9 version + version: '9.0.x' # Use the appropriate .NET 9 version installationPath: $(Agent.ToolsDirectory)/dotnet includePreviewVersions: true @@ -41,50 +41,3 @@ stages: Dockerfile: '$(Build.SourcesDirectory)/Dockerfile' tags: | $(Build.BuildId) - -- stage: TestAndCoverage - jobs: - - job: Test - steps: - - task: UseDotNet@2 - inputs: - packageType: 'sdk' - version: '9.0.x' # Use the appropriate .NET 9 version - installationPath: $(Agent.ToolsDirectory)/dotnet - includePreviewVersions: true - - - script: | - dotnet restore - displayName: 'Restore dependencies' - - - script: | - dotnet test --configuration Release --collect:"XPlat Code Coverage" --results-directory $(Build.ArtifactStagingDirectory)/TestResults - displayName: 'Run tests and collect code coverage' - - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: $(Build.ArtifactStagingDirectory)/TestResults - artifactName: 'CodeCoverage' - publishLocation: 'Container' - - - task: Docker@2 - displayName: Login in ACR - inputs: - command: login - containerRegistry: 'dps-acr-connection' - - - task: Docker@2 - inputs: - containerRegistry: 'dps-acr-connection' - repository: '$(imageName)' - command: 'buildAndPush' - Dockerfile: '$(Build.SourcesDirectory)/Dockerfile' - tags: | - $(Build.BuildId) - - - task: Codecov@1 - inputs: - token: $(CODECOV_TOKEN) # Add your Codecov token as a secret - file: '$(Build.ArtifactStagingDirectory)/TestResults/*/coverage.cobertura.xml' # Path to coverage file - commit: $(Build.SourceVersion) - name: $(Build.BuildId)