diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..fded4435 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in the repo. +* @Avijit-Microsoft @Roopan-Microsoft @Prajwal-Microsoft @dongbumlee \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..882ebd79 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,45 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: bug +assignees: '' + +--- + +# Describe the bug +A clear and concise description of what the bug is. + +# Expected behavior +A clear and concise description of what you expected to happen. + +# How does this bug make you feel? +_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel_ + +--- + +# Debugging information + +## Steps to reproduce +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +## Screenshots +If applicable, add screenshots to help explain your problem. + +## Logs + +If applicable, add logs to help the engineer debug the problem. + +--- + +# Tasks + +_To be filled in by the engineer picking up the issue_ + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..3496fc82 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,32 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +# Motivation + +A clear and concise description of why this feature would be useful and the value it would bring. +Explain any alternatives considered and why they are not sufficient. + +# How would you feel if this feature request was implemented? + +_Share a gif from [giphy](https://giphy.com/) to tells us how you'd feel. Format: ![alt_text](https://media.giphy.com/media/xxx/giphy.gif)_ + +# Requirements + +A list of requirements to consider this feature delivered +- Requirement 1 +- Requirement 2 +- ... + +# Tasks + +_To be filled in by the engineer picking up the issue_ + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/ISSUE_TEMPLATE/subtask.md b/.github/ISSUE_TEMPLATE/subtask.md new file mode 100644 index 00000000..9f86c843 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/subtask.md @@ -0,0 +1,22 @@ +--- +name: Sub task +about: A sub task +title: '' +labels: subtask +assignees: '' + +--- + +Required by + +# Description + +A clear and concise description of what this subtask is. + +# Tasks + +_To be filled in by the engineer picking up the subtask + +- [ ] Task 1 +- [ ] Task 2 +- [ ] ... diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..22b04b72 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,37 @@ +## Purpose + +* ... + +## Does this introduce a breaking change? + + +- [ ] Yes +- [ ] No + + + +## Golden Path Validation +- [ ] I have tested the primary workflows (the "golden path") to ensure they function correctly without errors. + +## Deployment Validation +- [ ] I have validated the deployment process successfully and all services are running as expected with this change. + +## What to Check +Verify that the following are valid +* ... + +## Other Information + \ No newline at end of file diff --git a/.github/workflows/pr-title-checker.yml b/.github/workflows/pr-title-checker.yml new file mode 100644 index 00000000..5cbbae1a --- /dev/null +++ b/.github/workflows/pr-title-checker.yml @@ -0,0 +1,22 @@ +name: "pr-title-checker" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + merge_group: + +permissions: + pull-requests: read + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + if: ${{ github.event_name != 'merge_group' }} + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/stale-bot.yml b/.github/workflows/stale-bot.yml new file mode 100644 index 00000000..e31059ab --- /dev/null +++ b/.github/workflows/stale-bot.yml @@ -0,0 +1,19 @@ +name: 'Close stale issues and PRs' +on: + schedule: + - cron: '30 1 * * *' + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: 'This issue is stale because it has been open 180 days with no activity. Remove stale label or comment or this will be closed in 30 days.' + days-before-stale: 180 + days-before-close: 30 diff --git a/App/kernel-memory/service/Abstractions/Constants.cs b/App/kernel-memory/service/Abstractions/Constants.cs index 53d5535e..d2e516f7 100644 --- a/App/kernel-memory/service/Abstractions/Constants.cs +++ b/App/kernel-memory/service/Abstractions/Constants.cs @@ -163,4 +163,5 @@ public static class Summary // Standard prompt names public const string PromptNamesSummarize = "summarize"; public const string PromptNamesAnswerWithFacts = "answer-with-facts"; + public const string PromptNamesExtractKeywords = "extract-keywords"; } diff --git a/App/kernel-memory/service/Core/Core.csproj b/App/kernel-memory/service/Core/Core.csproj index 71cec67b..c3ab4dd3 100644 --- a/App/kernel-memory/service/Core/Core.csproj +++ b/App/kernel-memory/service/Core/Core.csproj @@ -42,6 +42,7 @@ + diff --git a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs index b5f95c9b..df66dedf 100644 --- a/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs +++ b/App/kernel-memory/service/Core/Handlers/KeywordExtractingHandler.cs @@ -11,6 +11,7 @@ using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.ChatCompletion; using static Microsoft.KernelMemory.Pipeline.DataPipeline; +using Microsoft.KernelMemory.Prompts; namespace Microsoft.KernelMemory.Handlers; @@ -21,11 +22,13 @@ public sealed class KeywordExtractingHandler : IPipelineStepHandler private readonly IPipelineOrchestrator _orchestrator; private readonly Kernel _kernel; private readonly KernelMemoryConfig? _config = null; + private readonly string _extractKeywordPrompt; public KeywordExtractingHandler( string stepName, IPipelineOrchestrator orchestrator, KernelMemoryConfig config = null, + IPromptProvider? promptProvider = null, ILoggerFactory? loggerFactory = null ) { @@ -34,6 +37,10 @@ public KeywordExtractingHandler( this._orchestrator = orchestrator; this._config = config; + promptProvider ??= new EmbeddedPromptProvider(); + + this._extractKeywordPrompt = promptProvider.ReadPrompt(Constants.PromptNamesExtractKeywords); + //init Semantic Kernel this._kernel = Kernel.CreateBuilder() .AddAzureOpenAIChatCompletion(deploymentName: (string)this._config.Services["AzureOpenAIText"]["Deployment"], @@ -79,30 +86,7 @@ public KeywordExtractingHandler( var chat = this._kernel.GetRequiredService(); var chatHistory = new ChatHistory(); - var systemMessage = """ - You are an assistant to analyze Content and Extract Tags by Content. - [EXTRACT TAGS RULES] - IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND TAGS - TAGS SHOULD BE CATEGORY SPECIFIC - TAGS SHOULD BE A LIST OF STRINGS - TAGS COUNT CAN BE UP TO 10 UNDER A CATEGORY - CATEGORY COUNT CAN BE UP TO 10 - DON'T ADD ANY MARKDOWN EXPRESSION IN YOUR RESPONSE - [END RULES] - - [EXAMPLE] - [ - { - [category1": ["tag1", "tag2", "tag3"] - }, - { - "category2": ["tag1", "tag2", "tag3"] - } - ] - [END EXAMPLE] - """; - - chatHistory.AddSystemMessage(systemMessage); + chatHistory.AddSystemMessage(this._extractKeywordPrompt); chatHistory.AddUserMessage($"Extract tags from this content : {extactedFileContent} \n The format should be Json but Markdown expression."); var executionParam = new PromptExecutionSettings() diff --git a/App/kernel-memory/service/Core/Prompts/extract-keywords.txt b/App/kernel-memory/service/Core/Prompts/extract-keywords.txt index 7ecd1689..9dd6485c 100644 --- a/App/kernel-memory/service/Core/Prompts/extract-keywords.txt +++ b/App/kernel-memory/service/Core/Prompts/extract-keywords.txt @@ -1,21 +1,25 @@ -[EXTRACT KEYWORDS RULES] -IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND KEYWORDS -KEYWORDS SHOULD BE CATEGORY SPECIFIC -KEYWORDS SHOULD BE A LIST OF STRINGS -KEYWORDS COUNT CAN BE UP TO 50 +You are an assistant to analyze Content and Extract Tags by Content. +[EXTRACT TAGS RULES] +IT SHOULD BE A LIST OF DICTIONARIES WITH CATEGORY AND TAGS +TAGS SHOULD BE CATEGORY SPECIFIC +TAGS SHOULD BE A LIST OF STRINGS +TAGS COUNT CAN BE UP TO 10 UNDER A CATEGORY CATEGORY COUNT CAN BE UP TO 10 +DON'T ADD ANY MARKDOWN EXPRESSION IN YOUR RESPONSE [END RULES] - [EXAMPLE] [ { - "category1": ["keyword1", "keyword2", "keyword3"] + "category1": ["tag1", "tag2", "tag3"] }, { - "category2": ["keyword1", "keyword2", "keyword3"] + "category2": ["tag1", "tag2", "tag3"] } ] [END EXAMPLE] -Extract Keywords from this : -{{$input}} \ No newline at end of file +Extract Tags from this : +{{$input}} + + + diff --git a/Deployment/resourcedeployment.ps1 b/Deployment/resourcedeployment.ps1 index ced60993..35a5342a 100644 --- a/Deployment/resourcedeployment.ps1 +++ b/Deployment/resourcedeployment.ps1 @@ -41,6 +41,35 @@ function successBanner(){ Write-Host " |_| |___/ " } +function failureBanner(){ + Write-Host " _____ _ _ " + Write-Host "| __ \ | | | | " + Write-Host "| | | | ___ _ __ | | ___ _ _ _ __ ___ ___ _ __ | |_ " + Write-Host "| | | |/ _ \ '_ \| |/ _ \| | | | '_ ` _ \ / _ \ '_ \| __| " + Write-Host "| |__| | __/ |_) | | (_) | |_| | | | | | | __/ | | | |_ " + Write-Host "|_____/ \___| .__/|_|\___/ \__, |_| |_| |_|\___|_| |_|\__| " + Write-Host " | | __/ | " + Write-Host " ______ _|_| _ |___/ " + Write-Host "| ____| (_) | | | " + Write-Host "| |__ __ _ _| | ___ __| | " + Write-Host "| __/ _` | | |/ _ \/ _` | " + Write-Host "| | | (_| | | | __/ (_| | " + Write-Host "|_| \__,_|_|_|\___|\__,_| " +} + +# Common function to check if a variable is null or empty +function ValidateVariableIsNullOrEmpty { + param ( + [string]$variableValue, + [string]$variableName + ) + + if ([string]::IsNullOrEmpty($variableValue)) { + Write-Host "Error: $variableName is null or empty." -ForegroundColor Red + failureBanner + exit 1 + } +} # Function to prompt for parameters with kind messages function PromptForParameters { param( @@ -395,10 +424,33 @@ try { ############################################################### # Get the storage account key $storageAccountKey = az storage account keys list --account-name $deploymentResult.StorageAccountName --resource-group $deploymentResult.ResourceGroupName --query "[0].value" -o tsv + + # Validate if the storage account key is empty or null + ValidateVariableIsNullOrEmpty -variableValue $storageAccountKey -variableName "Storage account key" + ## Construct the connection string manually $storageAccountConnectionString = "DefaultEndpointsProtocol=https;AccountName=$($deploymentResult.StorageAccountName);AccountKey=$storageAccountKey;EndpointSuffix=core.windows.net" + # Validate if the Storage Account Connection String is empty or null + ValidateVariableIsNullOrEmpty -variableValue $storageAccountConnectionString -variableName "Storage Account Connection String" + ## Assign the connection string to the deployment result object - $deploymentResult.StorageAccountConnectionString = $storageAccountConnectionString + $deploymentResult.StorageAccountConnectionString = $storageAccountConnectionString + + # Check if ResourceGroupName is valid + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name" + + # Check if AzCosmosDBName is valid + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzCosmosDBName -variableName "Az Cosmos DB name" + + # Check if AzCognitiveServiceName is valid + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzCognitiveServiceName -variableName "Az Cognitive Service name" + + # Check if AzSearchServiceName is valid + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzSearchServiceName -variableName "Az Search Service name" + + # Check if AzOpenAiServiceName is valid + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AzOpenAiServiceName -variableName "Az OpenAI Service name" + # Get MongoDB connection string $deploymentResult.AzCosmosDBConnectionString = az cosmosdb keys list --name $deploymentResult.AzCosmosDBName --resource-group $deploymentResult.ResourceGroupName --type connection-strings --query "connectionStrings[0].connectionString" -o tsv # Get Azure Cognitive Service API Key @@ -601,26 +653,49 @@ try { # 6-1. Get Az Network resource Name with the public IP address Write-Host "Assign DNS Name to the public IP address" -ForegroundColor Green $publicIpName=$(az network public-ip list --query "[?ipAddress=='$externalIP'].name" --output tsv) - # 6-2. Generate Unique backend API fqdn Name - esgdocanalysis-3 digit random number with padding 0 $dnsName = "kmgs$($(Get-Random -Minimum 0 -Maximum 9999).ToString("D4"))" + + # Validate if the AKS Resource Group Name, Public IP name and DNS Name are provided + ValidateVariableIsNullOrEmpty -variableValue $aksResourceGroupName -variableName "AKS Resource Group name" + + ValidateVariableIsNullOrEmpty -variableValue $publicIpName -variableName "Public IP name" + ValidateVariableIsNullOrEmpty -variableValue $dnsName -variableName "DNS Name" + # 6-3. Assign DNS Name to the public IP address az network public-ip update --resource-group $aksResourceGroupName --name $publicIpName --dns-name $dnsName - # 6-4. Get FQDN for the public IP address - $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv - Write-Host "FQDN for the public IP address is: $fqdn" -ForegroundColor Green + # 6-4. Get FQDN for the public IP address + $fqdn = az network public-ip show --resource-group $aksResourceGroupName --name $publicIpName --query "dnsSettings.fqdn" --output tsv + + # Validate if the FQDN is null or empty + ValidateVariableIsNullOrEmpty -variableValue $fqdn -variableName "FQDN" + # 7. Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup Write-Host "Assign the role for aks system assigned managed identity to App Configuration Data Reader role" -ForegroundColor Green + # Ensure that the required fields are not null or empty + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.ResourceGroupName -variableName "Resource group name" + + ValidateVariableIsNullOrEmpty -variableValue $deploymentResult.AksName -variableName "AKS cluster name" + # Get vmss resource group name $vmssResourceGroupName = $(az aks show --resource-group $deploymentResult.ResourceGroupName --name $deploymentResult.AksName --query nodeResourceGroup --output tsv) + + # Validate if vmss Resource Group Name is null or empty + ValidateVariableIsNullOrEmpty -variableValue $vmssResourceGroupName -variableName "VMSS resource group" + # Get vmss name $vmssName = $(az vmss list --resource-group $vmssResourceGroupName --query "[0].name" --output tsv) + + # Validate if vmss Name is null or empty + ValidateVariableIsNullOrEmpty -variableValue $vmssName -variableName "VMSS name" + # Create System Assigned Managed Identity $systemAssignedIdentity = $(az vmss identity assign --resource-group $vmssResourceGroupName --name $vmssName --query systemAssignedIdentity --output tsv) - + # Validate if System Assigned Identity is null or empty + ValidateVariableIsNullOrEmpty -variableValue $systemAssignedIdentity -variableName "System-assigned managed identity" # Assign the role for aks system assigned managed identity to App Configuration Data Reader role with the scope of Resourcegroup az role assignment create --assignee $systemAssignedIdentity --role "App Configuration Data Reader" --scope $deploymentResult.ResourceGroupId