diff --git a/.github/workflows/deploy-aca-revision-and-traffic-management.yaml b/.github/workflows/deploy-aca-revision-and-traffic-management.yaml index 295a824..b7148e5 100644 --- a/.github/workflows/deploy-aca-revision-and-traffic-management.yaml +++ b/.github/workflows/deploy-aca-revision-and-traffic-management.yaml @@ -1,3 +1,4 @@ +# This workflow provisions all the necessary resources for the demo application from aca-revision-and-traffic-management folder. # Traffic splitting can be automated further, with revision name generation, health checks and full re-routing to the new version upon successful checks. # Microsoft has created a useful repository with a sample implementation of this process that can be used for reference and inspiration: https://github.com/Azure-Samples/containerapps-blue-green name: deploy-aca-revision-and-traffic-management-apps diff --git a/aca-revision-and-traffic-management/README.md b/aca-revision-and-traffic-management/README.md index a53230f..c5c06b2 100644 --- a/aca-revision-and-traffic-management/README.md +++ b/aca-revision-and-traffic-management/README.md @@ -1 +1,18 @@ # Revision and traffic management in Azure Container Apps + +This folder contains Bicep code for provisioning a demo application that can be used to see multiple revisions and traffic splitting for Azure Container Apps in action. Demo application itself is a simple Hello World application that was initially created by Microsoft for AKS demos, but why not re-use it for Azure Container Apps as well?😼 + +## Deployment instructions + +1. Deploy code as-is first (after adjusting parameters as per your use case) - initially in ```aca-public-apps.bicep``` it's defined that application will be deployed in multi-revision mode, but when we start from nothing only one, first, revision will be deployed. Due to that in ```*.bicepparam``` file traffic distribution is configured to send 100% traffic to the latest revision, which will be the app's very first revision. + +2. Let's make a change to the application to create a new revision - in ```aca-public-apps.bicep``` update ```TITLE``` environment variable with a new value that can identify new app revision. Next, let's update traffic distribution: + 2.1. Get name of the currently active, first app revision by running following Azure CLI command (update ```resource-group``` parameter with the one defined in the respective ```.bicepparam``` file): ```az containerapp revision list --name aca-helloworld --resource-group --query [0].name -o tsv``` + 2.2. In the respective ```.bicepparam``` file update ```trafficDistribution``` array: update weight number for ```latestRevision``` object - this object represents every new revision that's being provisioned. Uncomment second object and update ```revisionName``` value with the one retrieved in step 2.1. Then update ```weight``` value with the amount of traffic you want to send to the previous/initial revision. **Please note that weight for all revisions combined must be 100.** +3. Re-provision resources with the new changes. Go to the public URL of the app and do a bunch of refreshes to verify that traffic is now routed to both versions/revisions of the application. + +### GitHub Actions Workflow + +Example of a GitHub Actions Workflow has been set up for you to use in your own repository to provision resources in this folder. Workflow is available in ```.github/workflows/deploy-aca-revision-and-traffic-management.yaml``` file in the root of the repository. Please note that you need to configure GitHub secrets for the workflow to be able to log into your Azure subscription and provision resources to it. I would recommend setting up a managed identity with federated credential for this purpose and give it Contributor permissions on the subscription level (resource group provisioning is part of the Bicep code, but you can also provision resource group outside of this deployment and then only give the identity permissions on the respective resource group's level). + +Please refer following Microsoft documentation on how to set up managed identity with federated credentials for usage in GitHub Actions worfklow: [Use GitHub Actions to connect to Azure](https://learn.microsoft.com/en-us/azure/developer/github/connect-from-azure?tabs=azure-portal%2Clinux) diff --git a/aca-revision-and-traffic-management/main.bicep b/aca-revision-and-traffic-management/main.bicep index dd0b495..3fe8c7b 100644 --- a/aca-revision-and-traffic-management/main.bicep +++ b/aca-revision-and-traffic-management/main.bicep @@ -1,9 +1,18 @@ targetScope='subscription' +@description('Resource group name where all resources for the deployment will be provisioned.') param acaResourceGroupName string + +@description('Environment name (dev, test, prod)') param environment string + +@description('Location where resources will be provisioned') param location string + +@description('Tags to be applied to all resources in this deployment') param tags object + +@description('Array that represents desired traffic distribution between container apps revisions') param trafficDistribution array resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { @@ -11,8 +20,9 @@ resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { location: location } +@description('Module that provisions common resources that will be re-used by other resources in the deployment, like managed identities') module common 'modules/common.bicep' = { - name: 'common' + name: 'common-resources' scope: rg params: { environment: environment @@ -21,29 +31,29 @@ module common 'modules/common.bicep' = { } } -module acaenvironment 'modules/aca-environment.bicep' = { - name: 'aca-environment' +@description('Module that provisions common overall resources for Azure Container Apps, like Azure Container Apps environment.') +module aca_common 'modules/aca-common.bicep' = { + name: 'aca-common' scope: rg params: { location: location managedIdentityId: common.outputs.managedIdentityId tags: tags } - dependsOn: [common] } -module aca 'modules/aca.bicep' = { - name: 'aca' +@description('Module that provisions publicly accessible applications as Azure Container Apps.') +module public_apps 'modules/aca-public-apps.bicep' = { + name: 'public-apps' scope: rg params: { - environmentId: acaenvironment.outputs.environmentId + environmentId: aca_common.outputs.environmentId location: location managedIdentityId: common.outputs.managedIdentityId tags: tags trafficDistribution: trafficDistribution } - dependsOn: [acaenvironment] } -@description('URL for store application') -output storeUrl string = aca.outputs.helloWorldAppUri +@description('URL for accessing Hello World application') +output helloWorldUrl string = public_apps.outputs.helloWorldAppUri diff --git a/aca-revision-and-traffic-management/modules/aca-environment.bicep b/aca-revision-and-traffic-management/modules/aca-common.bicep similarity index 100% rename from aca-revision-and-traffic-management/modules/aca-environment.bicep rename to aca-revision-and-traffic-management/modules/aca-common.bicep diff --git a/aca-revision-and-traffic-management/modules/aca.bicep b/aca-revision-and-traffic-management/modules/aca-public-apps.bicep similarity index 99% rename from aca-revision-and-traffic-management/modules/aca.bicep rename to aca-revision-and-traffic-management/modules/aca-public-apps.bicep index 09c8f69..ccd6b7d 100644 --- a/aca-revision-and-traffic-management/modules/aca.bicep +++ b/aca-revision-and-traffic-management/modules/aca-public-apps.bicep @@ -39,7 +39,7 @@ resource helloworld 'Microsoft.App/containerApps@2023-05-02-preview' = { env: [ { name: 'TITLE' - value: 'Hello World from Azure Container Apps (ACA) - V2!' + value: 'Hello World from Azure Container Apps (ACA)!' } ] probes: [ diff --git a/aca-revision-and-traffic-management/parameters/dev.bicepparam b/aca-revision-and-traffic-management/parameters/dev.bicepparam index 2a435e8..78f9f7e 100644 --- a/aca-revision-and-traffic-management/parameters/dev.bicepparam +++ b/aca-revision-and-traffic-management/parameters/dev.bicepparam @@ -11,12 +11,12 @@ param tags = { param trafficDistribution = [ { latestRevision: true - weight: 50 + weight: 100 } - { - revisionName: 'aca-helloworld--f8u0hny' + /*{ + revisionName: '' weight: 50 - } + }*/ ] -// Command to get revision name: az containerapp revision list --name aca-helloworld --resource-group rg-aca-helloworld-neu-dev --query [0].name -o tsv +// Command to get revision names: az containerapp revision list --name aca-helloworld --resource-group rg-aca-helloworld-neu-dev --query [].name -o tsv diff --git a/aca-revision-and-traffic-management/parameters/prod.bicepparam b/aca-revision-and-traffic-management/parameters/prod.bicepparam index 5f91c19..32d50de 100644 --- a/aca-revision-and-traffic-management/parameters/prod.bicepparam +++ b/aca-revision-and-traffic-management/parameters/prod.bicepparam @@ -19,3 +19,5 @@ param trafficDistribution = [ weight: 50 }*/ ] + +// Command to get revision names: az containerapp revision list --name aca-helloworld --resource-group rg-aca-helloworld-neu-dev --query [].name -o tsv diff --git a/aks-store-on-aca/README.md b/aks-store-on-aca/README.md index 1f7c8a0..a8c60ba 100644 --- a/aks-store-on-aca/README.md +++ b/aks-store-on-aca/README.md @@ -1,18 +1,20 @@ # Implementation of AKS Store Demo App with Azure Container Apps -This folder contains Bicep code for provisioning [aks-store-demo](https://github.com/Azure-Samples/aks-store-demo) but on Azure Container Apps. Deployment also is created in a manner that's closer to an actual production scenario, including security hardening configuration. +This folder contains Bicep code for provisioning [aks-store-demo](https://github.com/Azure-Samples/aks-store-demo), but on Azure Container Apps. Deployment also is created in a manner that's closer to an actual production scenario, including security hardening configuration. Below you may find the solution architecture diagram: TODO -Implementation includes following modules: (TODO: add details) +Implementation includes following modules: -* ```common``` -* ```azure-monitor``` -* ```network``` -* ```keyvault``` -* ```ai``` -* ```aca-common``` -* ```aca-public-apps``` -* ```aca-internal-apps``` +* ```common```: includes common, shared resources that are used by other resources in the deployment. For example, managed identities or deployment-specific Azure Policy assignments. +* ```network```: includes network-related resources. For example, virtual networks, subnets and network security groups. +* ```dns```: includes DNS-related resources. For example, private DNS zones. +* ```vnet_links```: includes virtual network link resources for mapping of virtual networks with private DNS zones, which is required for the private endpoints to function properly. +* ```kv```: includes Azure Key Vault resources, with enabled RBAC and configuration for secure access to the resources with private endpoints. +* ```azure_monitor```: includes observability-related resources, like Log Analytics, Application Insights, etc. It also includes Azure Monitor Private Link Scope (AMPLS) and related resources for configuration of secure access to Azure Monitor services. +* ```ai```: includes cognitive services, like Azure OpenAI with respective model deployments and configuration for secure access to the resources with private endpoints. +* ```aca_common```: includes resources that are common for Azure Container Apps, like Azure Container Apps environment and network configuration for secure communication to and between apps. +* ```internal_apps```: includes container apps that are not publicly accessible, i.e. internal services. +* ```public_apps```: includes container apps that are publicly accessible. diff --git a/aks-store-on-aca/functions.bicep b/aks-store-on-aca/functions.bicep index e7b9c68..dc58163 100644 --- a/aks-store-on-aca/functions.bicep +++ b/aks-store-on-aca/functions.bicep @@ -1,4 +1,6 @@ -// Re-used from https://github.com/Azure/bicep/issues/5703#issuecomment-2004230485 +// Re-used from https://github.com/Azure/bicep/issues/5703#issuecomment-2004230485 + +@description('User-defined, re-usable function that can be used to replace multiple strings in a specific string, which is currently not supported out of the box by the replace() function in Bicep.') @export() func replaceMultipleStrings(input string, replacements { *: string }) string => reduce( items(replacements), input, (cur, next) => replace(string(cur), next.key, next.value)) diff --git a/aks-store-on-aca/main.bicep b/aks-store-on-aca/main.bicep index 6595ba1..973d185 100644 --- a/aks-store-on-aca/main.bicep +++ b/aks-store-on-aca/main.bicep @@ -1,16 +1,50 @@ targetScope='subscription' +@description('Resource group name where Azure Container Apps resources will be provisioned') param acaResourceGroupName string + +@description('Resource group name where resources that are shared across different resource group are provisioned') param commonResourceGroupName string + +@description('Name of the common Azure Key Vault that contains secrets that can\'t be uploaded as part of the Bicep code') param commonKeyVaultName string + +@description('Environment name (dev, test, prod)') param environment string + +@description('Location where resources will be provisioned') param location string + +@description('Prefix for the location name (e.g. "neu" for "northeurope")') param locationPrefix string + +@description('Location where Azure OpenAI resources will be provisioned') param openAILocation string + +@description('Subnets for the virtual network used by the Azure Container Apps resources') param subnets array + +@description('Tags to be applied to all resources in this deployment') param tags object + +@description('IP range for the virtual network that will be utilized by the Azure Container Apps resources') param vnetIpRange string +@description('List of Azure Monitor Private Link Scope (AMPLS) private DNS zones that will be used by the private endpoints in the deployment') +var amplsPrivateDnsZones = [ + 'privatelink.monitor.azure.com' + 'privatelink.oms.opinsights.azure.com' + 'privatelink.ods.opinsights.azure.com' + 'privatelink.agentsvc.azure-automation.net' + 'privatelink.blob.${az.environment().name}' +] + +@description('List of non-AMPLS private DNS zones that will be used by the private endpoints in the deployment') +var otherPrivateDnsZones = [ + 'privatelink.vaultcore.azure.net' + 'privatelink.openai.azure.com' +] + resource keyVaultCommon 'Microsoft.KeyVault/vaults@2022-07-01' existing = { scope: resourceGroup(commonResourceGroupName) name: commonKeyVaultName @@ -21,10 +55,9 @@ resource rg 'Microsoft.Resources/resourceGroups@2022-09-01' = { location: location } -/* Common resources that will be shared across services in the resource group */ - +@description('Module that provisions common resources that will be re-used by other resources in the deployment, like managed identities') module common 'modules/common.bicep' = { - name: 'common' + name: 'common-resources' scope: rg params: { environment: environment @@ -33,15 +66,11 @@ module common 'modules/common.bicep' = { } } -/* Network resources, including private DNS zones with virtual network links for the private endpoints */ -// TODO: look into iterating a list of DNS zones provided as parameter from the params file -module vnet 'modules/network.bicep' = { - name: 'vnet' +@description('Module that provisions network-related resources, like virtual network, subnets and network security groups') +module network 'modules/network.bicep' = { + name: 'network-resources' scope: rg - params: { - dnsZoneNameFile: 'privatelink.file.${az.environment().suffixes.storage}' - dnsZoneNameKeyVault: 'privatelink.vaultcore.azure.net' - dnsZoneNameOpenAI: 'privatelink.openai.azure.com' + params: { environment: environment location: location locationPrefix: locationPrefix @@ -51,101 +80,130 @@ module vnet 'modules/network.bicep' = { } } -/* Azure Key Vault resources, including respective access control and private endpoint configuration */ -module keyvault 'modules/keyvault.bicep' = { - name: 'keyvault' +@description('Module that provisions DNS-related resources, like private DNS zones') +module dns 'modules/dns.bicep' = { + name: 'dns-resources' + scope: rg + params: { + privateDnsZones: union(amplsPrivateDnsZones, otherPrivateDnsZones) + tags: tags + } +} + +@description('Module that provisions virtual network links for mapping respective virtual network resources with the private DNS zones') +module vnet_links 'modules/virtual-network-links.bicep' = [for (zone, i) in union(amplsPrivateDnsZones, otherPrivateDnsZones): { + name: '${zone}-vnetlink-deploy' scope: rg params: { - dnsZoneKeyVault: vnet.outputs.dnsZoneKeyVaultId + vnetId: network.outputs.vnetId + dnsZoneName: zone + tags: tags + } + dependsOn: [dns] +}] + +@description('Module that provisions Azure Key Vault that will be used by Azure Container Apps, with enabled RBAC and restricted access configuration, including access only through private endpoint.') +module kv 'modules/keyvault.bicep' = { + name: 'key-vault' + scope: rg + params: { + keyVaultDnsZoneName: otherPrivateDnsZones[0] location: location managedIdentityName: common.outputs.managedIdentityName - subnetId: vnet.outputs.acaSubnetId + subnetId: network.outputs.acaSubnetId tags: tags } - dependsOn: [vnet] + dependsOn: [dns] } -module azuremonitor 'modules/azure-monitor.bicep' = { - name: 'azuremonitor' +@description('Module that provisions Azure Monitor resources, like Log Analytics workspace and Application Insights, including Azure Monitor Private Link Scope (AMPLS) for secure access to observability resources.') +module azure_monitor 'modules/azure-monitor.bicep' = { + name: 'azure-monitor' scope: rg params: { + amplsPrivateDnsZones: amplsPrivateDnsZones environment: environment - keyVaultName: keyvault.outputs.keyVaultName - location: location + keyVaultName: kv.outputs.keyVaultName + location: location + locationPrefix: locationPrefix + managedIdentityId: common.outputs.managedIdentityId + subnetId: network.outputs.acaSubnetId tags: tags } - dependsOn: [vnet, keyvault] + dependsOn: [dns, vnet_links] } -resource keyVaultACAShared 'Microsoft.KeyVault/vaults@2022-07-01' existing = { - name: keyvault.outputs.keyVaultName +resource keyVaultACA 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + name: kv.outputs.keyVaultName scope: rg } -// AI-related services, like Azure OpenAI +@description('Module that provisions AI-related resources, like Azure OpenAI with respective model deployments, with restricted access configuration, like resource access only through private endpoint.') module ai 'modules/ai.bicep' = { name: 'ai' scope: rg params: { - dnsZoneOpenAIId: vnet.outputs.dnsZoneOpenAIId - keyVaultName: keyvault.outputs.keyVaultName + openAIDnsZoneName: otherPrivateDnsZones[1] + keyVaultName: kv.outputs.keyVaultName location: location managedIdentityId: common.outputs.managedIdentityId openAILocation: openAILocation - subnetId: vnet.outputs.acaSubnetId - tags: tags + subnetId: network.outputs.acaSubnetId + tags: tags } - dependsOn: [vnet, keyvault] + dependsOn: [dns] } -module acacommon 'modules/aca-common.bicep' = { +@description('Module that provisions common overall resources for Azure Container Apps, like Azure Container Apps environment, with restricted access configuration, like resource access only for defined subnets and ports.') +module aca_common 'modules/aca-common.bicep' = { name: 'aca-common' scope: rg params: { - appInsightsConnectionString: keyVaultACAShared.getSecret(azuremonitor.outputs.appInsightsConnectionString) location: location - logAnalyticsCustomerId: azuremonitor.outputs.logAnalyticsCustomerId - logAnalyticsKey: keyVaultACAShared.getSecret(azuremonitor.outputs.logAnalyticsKey) + logAnalyticsCustomerId: azure_monitor.outputs.logAnalyticsCustomerId + logAnalyticsKey: keyVaultACA.getSecret(azure_monitor.outputs.logAnalyticsKey) managedIdentityId: common.outputs.managedIdentityId - nsgName: vnet.outputs.nsgName - subnetId: vnet.outputs.acaSubnetId + nsgName: network.outputs.nsgName + subnetId: network.outputs.acaSubnetId tags: tags } - dependsOn: [common, keyvault, vnet] } -module backend 'modules/aca-internal-apps.bicep' = { - name: 'backend' +@description('Module that provisions internal applications as Azure Container Apps.') +module internal_apps 'modules/aca-internal-apps.bicep' = { + name: 'internal-apps' scope: rg params: { - location: location - environmentId: acacommon.outputs.environmentId - openAIEndpoint: keyVaultACAShared.getSecret('cogaEndpoint') + environmentId: aca_common.outputs.environmentId + location: location + managedIdentityId: common.outputs.managedIdentityId + openAIDeploymentName: ai.outputs.openAIDeploymentName + openAIEndpoint: keyVaultACA.getSecret('cogaEndpoint') + openAIKey: keyVaultACA.getSecret('cogaKey') queueUsername: keyVaultCommon.getSecret('queue-username') queuePass: keyVaultCommon.getSecret('queue-password') - subnetIpRange: vnet.outputs.acaSubnetIpRange + subnetIpRange: network.outputs.acaSubnetIpRange tags: tags } - dependsOn: [acacommon, keyvault] } -module frontend 'modules/aca-public-apps.bicep' = { - name: 'frontend' +@description('Module that provisions publicly accessible applications as Azure Container Apps.') +module public_apps 'modules/aca-public-apps.bicep' = { + name: 'public-apps' scope: rg params: { - environmentId: acacommon.outputs.environmentId + environmentId: aca_common.outputs.environmentId location: location - makelineServiceUri: backend.outputs.makelineServiceUri + makelineServiceUri: internal_apps.outputs.makelineServiceUri managedIdentityId: common.outputs.managedIdentityId - orderServiceUri: backend.outputs.orderServiceUri - productServiceUri: backend.outputs.productServiceUri + orderServiceUri: internal_apps.outputs.orderServiceUri + productServiceUri: internal_apps.outputs.productServiceUri tags: tags } - dependsOn: [acacommon, keyvault, backend] } -@description('URL for store application') -output storeUrl string = frontend.outputs.storeFrontUri +@description('URL for accessing store application') +output storeUrl string = public_apps.outputs.storeFrontUri -@description('URL for store admin application') -output storeAdminUrl string = frontend.outputs.storeAdminUri +@description('URL for accessing store admin application') +output storeAdminUrl string = public_apps.outputs.storeAdminUri diff --git a/aks-store-on-aca/modules/aca-common.bicep b/aks-store-on-aca/modules/aca-common.bicep index 9dcd5d3..649b6eb 100644 --- a/aks-store-on-aca/modules/aca-common.bicep +++ b/aks-store-on-aca/modules/aca-common.bicep @@ -5,14 +5,9 @@ param nsgName string param subnetId string param tags object -@secure() -param appInsightsConnectionString string - @secure() param logAnalyticsKey string - - resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-11-02-preview' = { name: 'cae-aca-store' location: location @@ -21,11 +16,8 @@ resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-11-02- userAssignedIdentities: { '${managedIdentityId}' : {} } - } + } properties: { - appInsightsConfiguration: { - connectionString: appInsightsConnectionString - } appLogsConfiguration: { destination: 'log-analytics' logAnalyticsConfiguration: { @@ -40,7 +32,6 @@ resource containerAppsEnvironment 'Microsoft.App/managedEnvironments@2023-11-02- tags: tags } - resource containerAppsInboundNsgRule 'Microsoft.Network/networkSecurityGroups/securityRules@2023-05-01' = { name: '${nsgName}/AllowInternet443FrontendInbound' properties: { diff --git a/aks-store-on-aca/modules/aca-internal-apps.bicep b/aks-store-on-aca/modules/aca-internal-apps.bicep index 6bb082f..7d5668e 100644 --- a/aks-store-on-aca/modules/aca-internal-apps.bicep +++ b/aks-store-on-aca/modules/aca-internal-apps.bicep @@ -1,24 +1,34 @@ param environmentId string param location string +param managedIdentityId string +param openAIDeploymentName string param subnetIpRange string param tags object @secure() param openAIEndpoint string +@secure() +param openAIKey string + @secure() param queueUsername string @secure() param queuePass string +/* ConfigMap isn't supported in Azure Container Apps, but an alternative way is to mount a file containing the needed information to the RabbitMQ container app. */ var rabbitmqPluginsConf = loadTextContent('rabbitmq_enabled_plugins') -// MongoDB instance for persisted data - resource mongodb 'Microsoft.App/containerApps@2023-05-02-preview' = { name: 'mongodb' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { managedEnvironmentId: environmentId configuration: { @@ -59,6 +69,7 @@ resource mongodb 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 } } } @@ -68,6 +79,12 @@ resource mongodb 'Microsoft.App/containerApps@2023-05-02-preview' = { resource rabbitmq 'Microsoft.App/containerApps@2023-05-02-preview' = { name: 'rabbitmq' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { managedEnvironmentId: environmentId configuration: { @@ -127,6 +144,7 @@ resource rabbitmq 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 } volumes: [ { @@ -148,6 +166,12 @@ resource rabbitmq 'Microsoft.App/containerApps@2023-05-02-preview' = { resource orderservice 'Microsoft.App/containerApps@2023-05-02-preview' = { name: 'order-service' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { managedEnvironmentId: environmentId configuration: { @@ -155,7 +179,7 @@ resource orderservice 'Microsoft.App/containerApps@2023-05-02-preview' = { external: false targetPort: 3000 transport: 'http' - allowInsecure: true + allowInsecure: true // required for this service due to limitations in the original application ipSecurityRestrictions: [ { name: 'AllowSnet' @@ -252,6 +276,7 @@ resource orderservice 'Microsoft.App/containerApps@2023-05-02-preview' = { //] scale: { minReplicas: 1 + maxReplicas: 3 } } } @@ -261,6 +286,12 @@ resource orderservice 'Microsoft.App/containerApps@2023-05-02-preview' = { resource makelineservice 'Microsoft.App/containerApps@2023-05-02-preview' = { name: 'makeline-service' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { managedEnvironmentId: environmentId configuration: { @@ -268,7 +299,7 @@ resource makelineservice 'Microsoft.App/containerApps@2023-05-02-preview' = { external: false targetPort: 3001 transport: 'http' - allowInsecure: true + allowInsecure: true // required for this service due to limitations in the original application ipSecurityRestrictions: [ { name: 'AllowSnet' @@ -344,6 +375,7 @@ resource makelineservice 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 } } } @@ -353,6 +385,12 @@ resource makelineservice 'Microsoft.App/containerApps@2023-05-02-preview' = { resource productservice 'Microsoft.App/containerApps@2023-05-02-preview' = { name: 'product-service' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { managedEnvironmentId: environmentId configuration: { @@ -414,6 +452,96 @@ resource productservice 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 + } + } + } + tags: tags +} + +resource aiservice 'Microsoft.App/containerApps@2023-05-02-preview' = { + name: 'ai-service' + location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } + properties: { + managedEnvironmentId: environmentId + configuration: { + ingress: { + external: false + targetPort: 5001 + transport: 'http' + allowInsecure: false + clientCertificateMode: 'accept' + ipSecurityRestrictions: [ + { + name: 'AllowSnet' + description: 'Allow access from main subnet' + action: 'Allow' + ipAddressRange: subnetIpRange + } + ] + } + } + template: { + containers: [ + { + image: 'ghcr.io/azure-samples/aks-store-demo/ai-service:latest' + name: 'product-service' + resources: { + cpu: json('0.25') + memory: '0.5Gi' + } + env: [ + { + name: 'USE_AZURE_OPENAI' + value: 'true' + } + { + name: 'AZURE_OPENAI_DEPLOYMENT_NAME' + value: openAIDeploymentName + } + { + name: 'AZURE_OPENAI_ENDPOINT' + value: openAIEndpoint + } + { + name: 'OPENAI_API_KEY' + value: openAIKey + } + ] + probes: [ + { + type: 'Liveness' + httpGet: { + path: '/health' + port: 5001 + } + initialDelaySeconds: 3 + periodSeconds: 3 + failureThreshold: 5 + } + { + type: 'Readiness' + httpGet: { + path: '/health' + port: 5001 + } + initialDelaySeconds: 3 + periodSeconds: 5 + failureThreshold: 3 + } + ] + + } + ] + scale: { + minReplicas: 1 + maxReplicas: 3 } } } @@ -423,6 +551,12 @@ resource productservice 'Microsoft.App/containerApps@2023-05-02-preview' = { resource virtualcustomer 'Microsoft.App/containerApps@2023-05-02-preview' = { name: 'virtual-customer' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { managedEnvironmentId: environmentId template: { @@ -448,6 +582,7 @@ resource virtualcustomer 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 } } } @@ -457,6 +592,12 @@ resource virtualcustomer 'Microsoft.App/containerApps@2023-05-02-preview' = { resource virtualworker 'Microsoft.App/containerApps@2023-05-02-preview' = { name: 'virtual-worker' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { managedEnvironmentId: environmentId template: { @@ -482,6 +623,7 @@ resource virtualworker 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 } } } diff --git a/aks-store-on-aca/modules/aca-public-apps.bicep b/aks-store-on-aca/modules/aca-public-apps.bicep index 2af7a3d..c09a25c 100644 --- a/aks-store-on-aca/modules/aca-public-apps.bicep +++ b/aks-store-on-aca/modules/aca-public-apps.bicep @@ -8,6 +8,9 @@ param orderServiceUri string param productServiceUri string param tags object +/* Due to hard-coded URLs and port numbers in the NGINX configuration in the original source code, instead of opening up additional unused ports in Azure Container Apps to support this + * NGINX configuration is overriden in a way that would work more natively for Azure Container Apps. NGINX configuration files for store-front and store-admin apps are stored in .conf files in the current folder. +*/ var storeFrontNginxConfReplacements = { '{ORDER_SERVICE_URI}': orderServiceUri '{PRODUCT_SERVICE_URI}': productServiceUri @@ -113,6 +116,7 @@ resource storefront 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 } volumes: [ { @@ -223,6 +227,7 @@ resource storeadmin 'Microsoft.App/containerApps@2023-05-02-preview' = { ] scale: { minReplicas: 1 + maxReplicas: 3 } volumes: [ { diff --git a/aks-store-on-aca/modules/ai.bicep b/aks-store-on-aca/modules/ai.bicep index 1f8b20b..a16fd68 100644 --- a/aks-store-on-aca/modules/ai.bicep +++ b/aks-store-on-aca/modules/ai.bicep @@ -1,4 +1,4 @@ -param dnsZoneOpenAIId string +param openAIDnsZoneName string param keyVaultName string param location string param managedIdentityId string @@ -8,10 +8,14 @@ param tags object var cognitiveAccountName = 'coga-${uniqueString('cognitive', resourceGroup().id)}' -resource keyVaultACAShared 'Microsoft.KeyVault/vaults@2022-07-01' existing = { +resource keyVaultACA 'Microsoft.KeyVault/vaults@2022-07-01' existing = { name: keyVaultName } +resource openAIDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: openAIDnsZoneName +} + resource cognitiveAccount 'Microsoft.CognitiveServices/accounts@2023-05-01' = { name: cognitiveAccountName location: openAILocation @@ -34,18 +38,18 @@ resource cognitiveAccount 'Microsoft.CognitiveServices/accounts@2023-05-01' = { tags: tags } -resource cognitiveAccountDeploymentGpt432k 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = { +resource cognitiveAccountDeploymentGpt35Turbo 'Microsoft.CognitiveServices/accounts/deployments@2023-05-01' = { parent: cognitiveAccount - name: 'gpt-4-32k' + name: 'gpt-35-turbo' sku: { name: 'Standard' - capacity: 60 + capacity: 240 } properties: { model: { format: 'OpenAI' - name: 'gpt-4-32k' - version: '0613' + name: 'gpt-35-turbo' + version: '0301' } } } @@ -81,7 +85,7 @@ resource cogaPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZ { name: 'config1' properties: { - privateDnsZoneId: dnsZoneOpenAIId + privateDnsZoneId: openAIDnsZone.id } } ] @@ -89,7 +93,7 @@ resource cogaPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZ } resource openAIKeySecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { - parent: keyVaultACAShared + parent: keyVaultACA name: 'cogaKey' properties: { value: cognitiveAccount.listKeys().key1 @@ -97,12 +101,11 @@ resource openAIKeySecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { } resource cognitiveAccountEndpoint 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { - parent: keyVaultACAShared + parent: keyVaultACA name: 'cogaEndpoint' properties: { value: cognitiveAccount.properties.endpoint } } -output openAIEndpoint string = cognitiveAccountEndpoint.properties.secretUri -output openAIKey string = openAIKeySecret.properties.secretUri +output openAIDeploymentName string = cognitiveAccountDeploymentGpt35Turbo.name diff --git a/aks-store-on-aca/modules/azure-monitor.bicep b/aks-store-on-aca/modules/azure-monitor.bicep index 12cf2f5..2b4fbe4 100644 --- a/aks-store-on-aca/modules/azure-monitor.bicep +++ b/aks-store-on-aca/modules/azure-monitor.bicep @@ -1,51 +1,98 @@ +param amplsPrivateDnsZones array param environment string param keyVaultName string param location string +param locationPrefix string +param managedIdentityId string +param subnetId string param tags object -resource keyVaultACAShared 'Microsoft.KeyVault/vaults@2022-07-01' existing = { + +var privateDnsZoneConfigs = [ for zone in amplsPrivateDnsZones : { + name: replace(zone, '.', '-') + properties: { + privateDnsZoneId: resourceId('Microsoft.Network/privateDnsZones', zone) + } +}] + +resource keyVaultACA 'Microsoft.KeyVault/vaults@2022-07-01' existing = { name: keyVaultName } +resource ampls 'microsoft.insights/privateLinkScopes@2021-07-01-preview' = { + name: 'ampls-${locationPrefix}-${environment}' + location: 'global' + tags: tags + properties: { + accessModeSettings: { + ingestionAccessMode: 'PrivateOnly' + queryAccessMode: 'Open' + } + } +} + resource logAnalytics 'Microsoft.OperationalInsights/workspaces@2022-10-01' = { name: 'log-aca-${environment}' location: location + identity: { + type: 'UserAssigned' + userAssignedIdentities: { + '${managedIdentityId}' : {} + } + } properties: { retentionInDays: 30 sku: { name: 'PerGB2018' } + publicNetworkAccessForIngestion: 'Disabled' + publicNetworkAccessForQuery: 'Enabled' } tags: tags } -resource appInsights 'Microsoft.Insights/components@2020-02-02' = { - name: 'appi-aca-${environment}' - location: location - kind: 'web' +resource logAnalyticsAMPLSConnection 'Microsoft.Insights/privateLinkScopes/scopedResources@2021-07-01-preview' = { + name: 'ampls-${logAnalytics.name}' + parent: ampls properties: { - Application_Type: 'web' - IngestionMode: 'LogAnalytics' - RetentionInDays: 30 - WorkspaceResourceId: logAnalytics.id + linkedResourceId: logAnalytics.id } - tags: tags } -resource appInsightsConnStringSecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { - parent: keyVaultACAShared - name: '${appInsights.name}-connection-string' +resource privateEndpointAMPLS 'Microsoft.Network/privateEndpoints@2023-04-01' = { + name: 'pe-${ampls.name}' + location: location properties: { - attributes: { - enabled: true + customNetworkInterfaceName: '${ampls.name}-nic-deluxe' + privateLinkServiceConnections: [ + { + name: 'psc-${ampls.name}' + properties: { + privateLinkServiceId: ampls.id + groupIds: [ + 'azuremonitor' + ] + } + } + ] + subnet: { + id: subnetId } - value: appInsights.properties.ConnectionString } + dependsOn: [logAnalyticsAMPLSConnection] tags: tags } +resource amplsPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-04-01' = { + name: 'default' + parent: privateEndpointAMPLS + properties: { + privateDnsZoneConfigs: privateDnsZoneConfigs + } +} + resource logAnalyticsKeySecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = { - parent: keyVaultACAShared + parent: keyVaultACA name: '${logAnalytics.name}-key' properties: { attributes: { @@ -58,7 +105,7 @@ resource logAnalyticsKeySecret 'Microsoft.KeyVault/vaults/secrets@2022-07-01' = resource kvDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01-preview' = { name: keyVaultName - scope: keyVaultACAShared + scope: keyVaultACA properties: { workspaceId: logAnalytics.id logs: [ @@ -70,7 +117,6 @@ resource kvDiagnosticSettings 'Microsoft.Insights/diagnosticSettings@2021-05-01- } } -output appInsightsConnectionString string = appInsightsConnStringSecret.name output logAnalyticsWorkspaceId string = logAnalytics.id output logAnalyticsCustomerId string = logAnalytics.properties.customerId output logAnalyticsKey string = logAnalyticsKeySecret.name diff --git a/aks-store-on-aca/modules/dns.bicep b/aks-store-on-aca/modules/dns.bicep new file mode 100644 index 0000000..6e71422 --- /dev/null +++ b/aks-store-on-aca/modules/dns.bicep @@ -0,0 +1,9 @@ +param privateDnsZones array +param tags object + +resource dnsZones 'Microsoft.Network/privateDnsZones@2020-06-01' = [for zone in privateDnsZones : { + name: zone + location: 'global' + properties: {} + tags: tags +}] diff --git a/aks-store-on-aca/modules/keyvault.bicep b/aks-store-on-aca/modules/keyvault.bicep index 8dda8a1..65ead1b 100644 --- a/aks-store-on-aca/modules/keyvault.bicep +++ b/aks-store-on-aca/modules/keyvault.bicep @@ -1,10 +1,10 @@ -param dnsZoneKeyVault string +param keyVaultDnsZoneName string param location string param managedIdentityName string param subnetId string param tags object -param tenantId string = subscription().tenantId +var tenantId = subscription().tenantId resource keyVaultSecretsOfficerRoleDefinition 'Microsoft.Authorization/roleDefinitions@2018-01-01-preview' existing = { scope: subscription() @@ -15,6 +15,10 @@ resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018- name: managedIdentityName } +resource keyVaultDnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: keyVaultDnsZoneName +} + resource keyVault 'Microsoft.KeyVault/vaults@2021-10-01' = { name: 'kv-${uniqueString('keyvault', resourceGroup().id, deployment().name)}' location: location @@ -54,7 +58,7 @@ resource keyVaultroleAssignment 'Microsoft.Authorization/roleAssignments@2020-10 resource privateEndpointKeyVault 'Microsoft.Network/privateEndpoints@2023-04-01' = { name: 'pe-${keyVault.name}' - location: location + location: location properties: { customNetworkInterfaceName: '${keyVault.name}-nic-deluxe' privateLinkServiceConnections: [ @@ -83,7 +87,7 @@ resource keyVaultPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/private { name: 'config1' properties: { - privateDnsZoneId: dnsZoneKeyVault + privateDnsZoneId: keyVaultDnsZone.id } } ] diff --git a/aks-store-on-aca/modules/network.bicep b/aks-store-on-aca/modules/network.bicep index 63dc6df..71a7996 100644 --- a/aks-store-on-aca/modules/network.bicep +++ b/aks-store-on-aca/modules/network.bicep @@ -1,6 +1,3 @@ -param dnsZoneNameFile string -param dnsZoneNameKeyVault string -param dnsZoneNameOpenAI string param environment string param location string param locationPrefix string @@ -130,69 +127,7 @@ resource nsg 'Microsoft.Network/networkSecurityGroups@2023-05-01' = { tags: tags } -resource privateDnsZoneOpenAI 'Microsoft.Network/privateDnsZones@2020-06-01' = { - name: dnsZoneNameOpenAI - location: 'global' - properties: {} - tags: tags -} - -resource privateDnsZoneFile 'Microsoft.Network/privateDnsZones@2020-06-01' = { - name: dnsZoneNameFile - location: 'global' - properties: {} - tags: tags -} - -resource privateDnsZoneKeyVault 'Microsoft.Network/privateDnsZones@2020-06-01' = { - name: dnsZoneNameKeyVault - location: 'global' - properties: {} - tags: tags -} - -resource privateDnsZoneLinkOpenAI 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { - name: '${privateDnsZoneOpenAI.name}-link' - parent: privateDnsZoneOpenAI - location: 'global' - properties: { - registrationEnabled: false - virtualNetwork: { - id: vnet.id - } - } - tags: tags -} - -resource privateDnsZoneLinkFile 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { - name: '${privateDnsZoneFile.name}-link' - parent: privateDnsZoneFile - location: 'global' - properties: { - registrationEnabled: false - virtualNetwork: { - id: vnet.id - } - } - tags: tags -} - -resource privateDnsZoneLinkKeyVault 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { - name: '${privateDnsZoneKeyVault.name}-link' - parent: privateDnsZoneKeyVault - location: 'global' - properties: { - registrationEnabled: false - virtualNetwork: { - id: vnet.id - } - } - tags: tags -} - output acaSubnetId string = vnet.properties.subnets[0].id output acaSubnetIpRange string = vnet.properties.subnets[0].properties.addressPrefix -output dnsZoneOpenAIId string = privateDnsZoneOpenAI.id -output dnsZoneFileId string = privateDnsZoneFile.id -output dnsZoneKeyVaultId string = privateDnsZoneKeyVault.id output nsgName string = nsg.name +output vnetId string = vnet.id diff --git a/aks-store-on-aca/modules/virtual-network-links.bicep b/aks-store-on-aca/modules/virtual-network-links.bicep new file mode 100644 index 0000000..a48d9c9 --- /dev/null +++ b/aks-store-on-aca/modules/virtual-network-links.bicep @@ -0,0 +1,20 @@ +param dnsZoneName string +param tags object +param vnetId string + +resource dnsZone 'Microsoft.Network/privateDnsZones@2020-06-01' existing = { + name: dnsZoneName +} + +resource vnetLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2020-06-01' = { + name: '${dnsZone.name}-link' + parent: dnsZone + location: 'global' + properties: { + registrationEnabled: false + virtualNetwork: { + id: vnetId + } + } + tags: tags +}