From 7cd1829cbf71486b572ab4829247ecc117c67870 Mon Sep 17 00:00:00 2001 From: zjhe Date: Mon, 14 Aug 2023 11:24:57 +0800 Subject: [PATCH] try to fix 223 --- .../k8s_workload.tf | 104 +++++++++++ examples/application_gateway_ingress/main.tf | 176 ++++++++++++++++++ .../application_gateway_ingress/outputs.tf | 4 + .../application_gateway_ingress/providers.tf | 38 ++++ .../application_gateway_ingress/variables.tf | 14 ++ locals.tf | 15 +- log_analytics.tf | 65 +++++++ main.tf | 121 +----------- role_assignments.tf | 97 ++++++++++ test/e2e/terraform_aks_test.go | 43 +++++ variables.tf | 20 +- 11 files changed, 571 insertions(+), 126 deletions(-) create mode 100644 examples/application_gateway_ingress/k8s_workload.tf create mode 100644 examples/application_gateway_ingress/main.tf create mode 100644 examples/application_gateway_ingress/outputs.tf create mode 100644 examples/application_gateway_ingress/providers.tf create mode 100644 examples/application_gateway_ingress/variables.tf create mode 100644 log_analytics.tf create mode 100644 role_assignments.tf diff --git a/examples/application_gateway_ingress/k8s_workload.tf b/examples/application_gateway_ingress/k8s_workload.tf new file mode 100644 index 00000000..032bb4f1 --- /dev/null +++ b/examples/application_gateway_ingress/k8s_workload.tf @@ -0,0 +1,104 @@ +resource "kubernetes_namespace_v1" "example" { + metadata { + name = "example" + } +} + +resource "kubernetes_pod" "aspnet_app" { + #checkov:skip=CKV_K8S_8:We don't need readiness probe for this simple example. + #checkov:skip=CKV_K8S_9:We don't need readiness probe for this simple example. + #checkov:skip=CKV_K8S_22:readOnlyRootFilesystem would block our pod from working + #checkov:skip=CKV_K8S_28:capabilities would block our pod from working + metadata { + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + labels = { + app = "aspnetapp" + } + } + spec { + container { + name = "aspnetapp-image" + image = "mcr.microsoft.com/dotnet/samples@sha256:7070894cc10d2b1e68e72057cca22040c5984cfae2ec3e079e34cf0a4da7fcea" + image_pull_policy = "Always" + security_context {} + port { + container_port = 80 + protocol = "TCP" + } + resources { + requests = { + cpu = "250m" + memory = "256Mi" + } + limits = { + cpu = "250m" + memory = "256Mi" + } + } + } + } +} + +resource "kubernetes_service" "svc" { + metadata { + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + } + spec { + selector = { + app = "aspnetapp" + } + port { + port = 80 + target_port = 80 + protocol = "TCP" + } + } +} + +resource "kubernetes_ingress_v1" "ing" { + metadata { + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + annotations = { + "kubernetes.io/ingress.class" : "azure/application-gateway" + } + } + spec { + rule { + http { + path { + path = "/" + backend { + service { + name = "aspnetapp" + port { + number = 80 + } + } + } + path_type = "Exact" + } + } + } + } + depends_on = [ + module.aks, + ] +} + +resource "time_sleep" "wait_one_minute" { + create_duration = "1m" + + depends_on = [kubernetes_ingress_v1.ing] +} + +data "kubernetes_ingress_v1" "ing" { + metadata { + name = "aspnetapp" + namespace = kubernetes_namespace_v1.example.metadata[0].name + } + + depends_on = [time_sleep.wait_one_minute] +} \ No newline at end of file diff --git a/examples/application_gateway_ingress/main.tf b/examples/application_gateway_ingress/main.tf new file mode 100644 index 00000000..7530ee55 --- /dev/null +++ b/examples/application_gateway_ingress/main.tf @@ -0,0 +1,176 @@ +resource "random_id" "prefix" { + byte_length = 8 +} + +resource "random_id" "name" { + byte_length = 8 +} + +resource "azurerm_resource_group" "main" { + count = var.create_resource_group ? 1 : 0 + + location = var.location + name = coalesce(var.resource_group_name, "${random_id.prefix.hex}-rg") +} + +locals { + resource_group = { + name = var.create_resource_group ? azurerm_resource_group.main[0].name : var.resource_group_name + location = var.location + } +} + +resource "azurerm_virtual_network" "test" { + address_space = ["10.52.0.0/16"] + location = local.resource_group.location + name = "${random_id.prefix.hex}-vn" + resource_group_name = local.resource_group.name +} + +resource "azurerm_subnet" "test" { + address_prefixes = ["10.52.0.0/24"] + name = "${random_id.prefix.hex}-sn" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name +} + +resource "azurerm_subnet" "appgw" { + address_prefixes = ["10.52.1.0/24"] + name = "${random_id.prefix.hex}-gw" + resource_group_name = local.resource_group.name + virtual_network_name = azurerm_virtual_network.test.name +} + +# Locals block for hardcoded names +locals { + backend_address_pool_name = "${azurerm_virtual_network.test.name}-beap" + frontend_ip_configuration_name = "${azurerm_virtual_network.test.name}-feip" + frontend_port_name = "${azurerm_virtual_network.test.name}-feport" + http_setting_name = "${azurerm_virtual_network.test.name}-be-htst" + listener_name = "${azurerm_virtual_network.test.name}-httplstn" + request_routing_rule_name = "${azurerm_virtual_network.test.name}-rqrt" +} + +resource "azurerm_public_ip" "pip" { + allocation_method = "Static" + location = local.resource_group.location + name = "appgw-pip" + resource_group_name = local.resource_group.name + sku = "Standard" +} + +resource "azurerm_application_gateway" "appgw" { + location = local.resource_group.location + #checkov:skip=CKV_AZURE_120:We don't need the WAF for this simple example + name = "ingress" + resource_group_name = local.resource_group.name + + backend_address_pool { + name = local.backend_address_pool_name + } + backend_http_settings { + cookie_based_affinity = "Disabled" + name = local.http_setting_name + port = 80 + protocol = "Http" + request_timeout = 1 + } + frontend_ip_configuration { + name = local.frontend_ip_configuration_name + public_ip_address_id = azurerm_public_ip.pip.id + } + frontend_port { + name = local.frontend_port_name + port = 80 + } + gateway_ip_configuration { + name = "appGatewayIpConfig" + subnet_id = azurerm_subnet.appgw.id + } + http_listener { + frontend_ip_configuration_name = local.frontend_ip_configuration_name + frontend_port_name = local.frontend_port_name + name = local.listener_name + protocol = "Http" + } + request_routing_rule { + http_listener_name = local.listener_name + name = local.request_routing_rule_name + rule_type = "Basic" + backend_address_pool_name = local.backend_address_pool_name + backend_http_settings_name = local.http_setting_name + priority = 1 + } + sku { + name = "Standard_v2" + tier = "Standard_v2" + capacity = 1 + } + + lifecycle { + ignore_changes = [ + tags, + backend_address_pool, + backend_http_settings, + http_listener, + probe, + request_routing_rule, + url_path_map, + ] + } +} + +module "aks" { + #checkov:skip=CKV_AZURE_141:We enable admin account here so we can provision K8s resources directly in this simple example + source = "../.." + + prefix = random_id.name.hex + resource_group_name = local.resource_group.name + kubernetes_version = "1.26" # don't specify the patch version! + automatic_channel_upgrade = "patch" + agents_availability_zones = ["1", "2"] + agents_count = null + agents_max_count = 2 + agents_max_pods = 100 + agents_min_count = 1 + agents_pool_name = "testnodepool" + agents_pool_linux_os_configs = [ + { + transparent_huge_page_enabled = "always" + sysctl_configs = [ + { + fs_aio_max_nr = 65536 + fs_file_max = 100000 + fs_inotify_max_user_watches = 1000000 + } + ] + } + ] + agents_type = "VirtualMachineScaleSets" + azure_policy_enabled = true + enable_auto_scaling = true + enable_host_encryption = true + http_application_routing_enabled = true + ingress_application_gateway_enabled = true + application_gateway_for_ingress = { + id = azurerm_application_gateway.appgw.id + subnet_id = azurerm_subnet.appgw.id + } + local_account_disabled = false + log_analytics_workspace_enabled = false + net_profile_dns_service_ip = "10.0.0.10" + net_profile_service_cidr = "10.0.0.0/16" + network_plugin = "azure" + network_policy = "azure" + os_disk_size_gb = 60 + private_cluster_enabled = false + public_network_access_enabled = true + rbac_aad = true + rbac_aad_managed = true + role_based_access_control_enabled = true + sku_tier = "Standard" + vnet_subnet_id = azurerm_subnet.test.id + depends_on = [ + azurerm_subnet.test, + ] +} diff --git a/examples/application_gateway_ingress/outputs.tf b/examples/application_gateway_ingress/outputs.tf new file mode 100644 index 00000000..75b02f54 --- /dev/null +++ b/examples/application_gateway_ingress/outputs.tf @@ -0,0 +1,4 @@ +output "ingress_endpoint" { + depends_on = [time_sleep.wait_one_minute] + value = "http://${data.kubernetes_ingress_v1.ing.status[0].load_balancer[0].ingress[0].ip}" +} diff --git a/examples/application_gateway_ingress/providers.tf b/examples/application_gateway_ingress/providers.tf new file mode 100644 index 00000000..ba7f5bd2 --- /dev/null +++ b/examples/application_gateway_ingress/providers.tf @@ -0,0 +1,38 @@ +terraform { + required_version = ">=1.3" + required_providers { + azurerm = { + source = "hashicorp/azurerm" + version = ">= 3.51, < 4.0" + } + kubernetes = { + source = "hashicorp/kubernetes" + version = "2.22.0" + } + random = { + source = "hashicorp/random" + version = "3.3.2" + } + time = { + source = "hashicorp/time" + version = "0.9.1" + } + } +} + +provider "azurerm" { + features { + resource_group { + prevent_deletion_if_contains_resources = false + } + } +} + +provider "kubernetes" { + host = module.aks.admin_host + client_certificate = base64decode(module.aks.admin_client_certificate) + client_key = base64decode(module.aks.admin_client_key) + cluster_ca_certificate = base64decode(module.aks.admin_cluster_ca_certificate) +} + +provider "random" {} \ No newline at end of file diff --git a/examples/application_gateway_ingress/variables.tf b/examples/application_gateway_ingress/variables.tf new file mode 100644 index 00000000..8336610e --- /dev/null +++ b/examples/application_gateway_ingress/variables.tf @@ -0,0 +1,14 @@ +variable "create_resource_group" { + type = bool + default = true + nullable = false +} + +variable "location" { + default = "eastus" +} + +variable "resource_group_name" { + type = string + default = null +} diff --git a/locals.tf b/locals.tf index b5c09745..496ea1aa 100644 --- a/locals.tf +++ b/locals.tf @@ -1,4 +1,14 @@ locals { + # Application Gateway ID: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mygroup1/providers/Microsoft.Network/applicationGateways/myGateway1 + application_gateway_for_ingress_id = var.application_gateway_for_ingress == null ? null : var.application_gateway_for_ingress.id + application_gateway_resource_group_for_ingress = var.application_gateway_for_ingress == null ? null : local.application_gateway_segments_for_ingress[4] + application_gateway_segments_for_ingress = var.application_gateway_for_ingress == null ? null : split("/", local.application_gateway_for_ingress_id) + application_gateway_subnet_resource_group_name = try(local.application_gateway_subnet_segments[4], null) + # Subnet ID: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/mygroup1/providers/Microsoft.Network/virtualNetworks/myvnet1/subnets/mysubnet1 + application_gateway_subnet_segments = try(split("/", var.application_gateway_for_ingress.subnet_id), []) + application_gateway_subnet_subscription_id_for_ingress = try(local.application_gateway_subnet_segments[2], null) + application_gateway_subnet_vnet_name = try(local.application_gateway_subnet_segments[8], null) + application_gateway_subscription_id_for_ingress = var.application_gateway_for_ingress == null ? null : local.application_gateway_segments_for_ingress[2] # Abstract if auto_scaler_profile_scale_down_delay_after_delete is not set or null we should use the scan_interval. auto_scaler_profile_scale_down_delay_after_delete = var.auto_scaler_profile_scale_down_delay_after_delete == null ? var.auto_scaler_profile_scan_interval : var.auto_scaler_profile_scale_down_delay_after_delete # automatic upgrades are either: @@ -10,8 +20,9 @@ locals { (contains(["rapid", "stable", "node-image"], var.automatic_channel_upgrade) && var.kubernetes_version == null && var.orchestrator_version == null) ) # Abstract the decision whether to create an Analytics Workspace or not. - create_analytics_solution = var.log_analytics_workspace_enabled && var.log_analytics_solution == null - create_analytics_workspace = var.log_analytics_workspace_enabled && var.log_analytics_workspace == null + create_analytics_solution = var.log_analytics_workspace_enabled && var.log_analytics_solution == null + create_analytics_workspace = var.log_analytics_workspace_enabled && var.log_analytics_workspace == null + create_role_assignments_for_application_gateway = try(var.application_gateway_for_ingress.create_role_assignments, false) # Abstract the decision whether to use an Analytics Workspace supplied via vars, provision one ourselves or leave it null. # This guarantees that local.log_analytics_workspace will contain a valid `id` and `name` IFF log_analytics_workspace_enabled # is set to `true`. diff --git a/log_analytics.tf b/log_analytics.tf new file mode 100644 index 00000000..bdc0298e --- /dev/null +++ b/log_analytics.tf @@ -0,0 +1,65 @@ +resource "azurerm_log_analytics_workspace" "main" { + count = local.create_analytics_workspace ? 1 : 0 + + location = coalesce(var.location, data.azurerm_resource_group.main.location) + name = coalesce(var.cluster_log_analytics_workspace_name, trim("${var.prefix}-workspace", "-")) + resource_group_name = coalesce(var.log_analytics_workspace_resource_group_name, var.resource_group_name) + retention_in_days = var.log_retention_in_days + sku = var.log_analytics_workspace_sku + tags = merge(var.tags, (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { + avm_git_commit = "0ae8a663f1dc1dc474b14c10d9c94c77a3d1e234" + avm_git_file = "main.tf" + avm_git_last_modified_at = "2023-06-05 02:21:33" + avm_git_org = "Azure" + avm_git_repo = "terraform-azurerm-aks" + avm_yor_trace = "9bb3ab45-1155-4bea-bc68-6b7d9aa73fbc" + } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/), (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { + avm_yor_name = "main" + } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/)) + + lifecycle { + precondition { + condition = can(coalesce(var.cluster_log_analytics_workspace_name, var.prefix)) + error_message = "You must set one of `var.cluster_log_analytics_workspace_name` and `var.prefix` to create `azurerm_log_analytics_workspace.main`." + } + } +} + +locals { + azurerm_log_analytics_workspace_id = try(azurerm_log_analytics_workspace.main[0].id, null) + azurerm_log_analytics_workspace_location = try(azurerm_log_analytics_workspace.main[0].location, null) + azurerm_log_analytics_workspace_name = try(azurerm_log_analytics_workspace.main[0].name, null) + azurerm_log_analytics_workspace_resource_group_name = try(azurerm_log_analytics_workspace.main[0].resource_group_name, null) +} + +data "azurerm_log_analytics_workspace" "main" { + count = local.query_datasource_for_log_analytics_workspace_location ? 1 : 0 + + name = var.log_analytics_workspace.name + resource_group_name = local.log_analytics_workspace.resource_group_name +} + +resource "azurerm_log_analytics_solution" "main" { + count = local.create_analytics_solution ? 1 : 0 + + location = coalesce(local.log_analytics_workspace.location, try(data.azurerm_log_analytics_workspace.main[0].location, null)) + resource_group_name = local.log_analytics_workspace.resource_group_name + solution_name = "ContainerInsights" + workspace_name = local.log_analytics_workspace.name + workspace_resource_id = local.log_analytics_workspace.id + tags = merge(var.tags, (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { + avm_git_commit = "886c26d95843149cc2a58ae72edb31478faa2a8c" + avm_git_file = "main.tf" + avm_git_last_modified_at = "2023-07-20 06:04:07" + avm_git_org = "Azure" + avm_git_repo = "terraform-azurerm-aks" + avm_yor_trace = "72af332c-2eac-4d8e-b895-bf85e31f0e23" + } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/), (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { + avm_yor_name = "main" + } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/)) + + plan { + product = "OMSGallery/ContainerInsights" + publisher = "Microsoft" + } +} diff --git a/main.tf b/main.tf index d7c53a5a..0c07c882 100644 --- a/main.tf +++ b/main.tf @@ -261,9 +261,7 @@ resource "azurerm_kubernetes_cluster" "main" { } } dynamic "api_server_access_profile" { - for_each = var.api_server_authorized_ip_ranges != null || var.api_server_subnet_id != null ? [ - "api_server_access_profile" - ] : [] + for_each = var.api_server_authorized_ip_ranges != null || var.api_server_subnet_id != null ? ["api_server_access_profile"] : [] content { authorized_ip_ranges = var.api_server_authorized_ip_ranges @@ -333,7 +331,7 @@ resource "azurerm_kubernetes_cluster" "main" { for_each = var.ingress_application_gateway_enabled ? ["ingress_application_gateway"] : [] content { - gateway_id = var.ingress_application_gateway_id + gateway_id = try(var.application_gateway_for_ingress.id, null) gateway_name = var.ingress_application_gateway_name subnet_cidr = var.ingress_application_gateway_subnet_cidr subnet_id = var.ingress_application_gateway_subnet_id @@ -437,9 +435,7 @@ resource "azurerm_kubernetes_cluster" "main" { service_cidr = var.net_profile_service_cidr dynamic "load_balancer_profile" { - for_each = var.load_balancer_profile_enabled && var.load_balancer_sku == "standard" ? [ - "load_balancer_profile" - ] : [] + for_each = var.load_balancer_profile_enabled && var.load_balancer_sku == "standard" ? ["load_balancer_profile"] : [] content { idle_timeout_in_minutes = var.load_balancer_profile_idle_timeout_in_minutes @@ -745,81 +741,6 @@ resource "null_resource" "pool_name_keeper" { } } -resource "azurerm_log_analytics_workspace" "main" { - count = local.create_analytics_workspace ? 1 : 0 - - location = coalesce(var.location, data.azurerm_resource_group.main.location) - name = coalesce(var.cluster_log_analytics_workspace_name, trim("${var.prefix}-workspace", "-")) - resource_group_name = coalesce(var.log_analytics_workspace_resource_group_name, var.resource_group_name) - retention_in_days = var.log_retention_in_days - sku = var.log_analytics_workspace_sku - tags = merge(var.tags, (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { - avm_git_commit = "0ae8a663f1dc1dc474b14c10d9c94c77a3d1e234" - avm_git_file = "main.tf" - avm_git_last_modified_at = "2023-06-05 02:21:33" - avm_git_org = "Azure" - avm_git_repo = "terraform-azurerm-aks" - avm_yor_trace = "8f2ac5c0-289d-4e27-a3e3-e605f51e7454" - } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/), (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { - avm_yor_name = "main" - } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/)) - - lifecycle { - precondition { - condition = can(coalesce(var.cluster_log_analytics_workspace_name, var.prefix)) - error_message = "You must set one of `var.cluster_log_analytics_workspace_name` and `var.prefix` to create `azurerm_log_analytics_workspace.main`." - } - } -} - -locals { - azurerm_log_analytics_workspace_id = try(azurerm_log_analytics_workspace.main[0].id, null) - azurerm_log_analytics_workspace_location = try(azurerm_log_analytics_workspace.main[0].location, null) - azurerm_log_analytics_workspace_name = try(azurerm_log_analytics_workspace.main[0].name, null) - azurerm_log_analytics_workspace_resource_group_name = try(azurerm_log_analytics_workspace.main[0].resource_group_name, null) -} - -data "azurerm_log_analytics_workspace" "main" { - count = local.query_datasource_for_log_analytics_workspace_location ? 1 : 0 - - name = var.log_analytics_workspace.name - resource_group_name = local.log_analytics_workspace.resource_group_name -} - -resource "azurerm_log_analytics_solution" "main" { - count = local.create_analytics_solution ? 1 : 0 - - location = coalesce(local.log_analytics_workspace.location, try(data.azurerm_log_analytics_workspace.main[0].location, null)) - resource_group_name = local.log_analytics_workspace.resource_group_name - solution_name = "ContainerInsights" - workspace_name = local.log_analytics_workspace.name - workspace_resource_id = local.log_analytics_workspace.id - tags = merge(var.tags, (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { - avm_git_commit = "886c26d95843149cc2a58ae72edb31478faa2a8c" - avm_git_file = "main.tf" - avm_git_last_modified_at = "2023-07-20 06:04:07" - avm_git_org = "Azure" - avm_git_repo = "terraform-azurerm-aks" - avm_yor_trace = "b15b12f7-337b-4ead-897a-49d07a0f26b3" - } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/), (/**/ (var.tracing_tags_enabled ? { for k, v in /**/ { - avm_yor_name = "main" - } /**/ : replace(k, "avm_", var.tracing_tags_prefix) => v } : {}) /**/)) - - plan { - product = "OMSGallery/ContainerInsights" - publisher = "Microsoft" - } -} - -resource "azurerm_role_assignment" "acr" { - for_each = var.attached_acr_id_map - - principal_id = azurerm_kubernetes_cluster.main.kubelet_identity[0].object_id - scope = each.value - role_definition_name = "AcrPull" - skip_service_principal_aad_check = true -} - # /subscriptions/00000000-0000-0000-0000-000000000000/resourcegroups/acceptanceTestResourceGroup1/providers/Microsoft.ManagedIdentity/userAssignedIdentities/testIdentity data "azurerm_user_assigned_identity" "cluster_identity" { count = (var.client_id == "" || var.client_secret == "") && var.identity_type == "UserAssigned" ? 1 : 0 @@ -827,39 +748,3 @@ data "azurerm_user_assigned_identity" "cluster_identity" { name = split("/", var.identity_ids[0])[8] resource_group_name = split("/", var.identity_ids[0])[4] } - -# The AKS cluster identity has the Contributor role on the AKS second resource group (MC_myResourceGroup_myAKSCluster_eastus) -# However when using a custom VNET, the AKS cluster identity needs the Network Contributor role on the VNET subnets -# used by the system node pool and by any additional node pools. -# https://learn.microsoft.com/en-us/azure/aks/configure-kubenet#prerequisites -# https://learn.microsoft.com/en-us/azure/aks/configure-azure-cni#prerequisites -# https://github.com/Azure/terraform-azurerm-aks/issues/178 -resource "azurerm_role_assignment" "network_contributor" { - for_each = var.create_role_assignment_network_contributor && (var.client_id == "" || var.client_secret == "") ? local.subnet_ids : [] - - principal_id = coalesce(try(data.azurerm_user_assigned_identity.cluster_identity[0].principal_id, azurerm_kubernetes_cluster.main.identity[0].principal_id), var.client_id) - scope = each.value - role_definition_name = "Network Contributor" - - lifecycle { - precondition { - condition = length(var.network_contributor_role_assigned_subnet_ids) == 0 - error_message = "Cannot set both of `var.create_role_assignment_network_contributor` and `var.network_contributor_role_assigned_subnet_ids`." - } - } -} - -resource "azurerm_role_assignment" "network_contributor_on_subnet" { - for_each = var.network_contributor_role_assigned_subnet_ids - - principal_id = coalesce(try(data.azurerm_user_assigned_identity.cluster_identity[0].principal_id, azurerm_kubernetes_cluster.main.identity[0].principal_id), var.client_id) - scope = each.value - role_definition_name = "Network Contributor" - - lifecycle { - precondition { - condition = !var.create_role_assignment_network_contributor - error_message = "Cannot set both of `var.create_role_assignment_network_contributor` and `var.network_contributor_role_assigned_subnet_ids`." - } - } -} diff --git a/role_assignments.tf b/role_assignments.tf new file mode 100644 index 00000000..0d31044f --- /dev/null +++ b/role_assignments.tf @@ -0,0 +1,97 @@ +resource "azurerm_role_assignment" "acr" { + for_each = var.attached_acr_id_map + + principal_id = azurerm_kubernetes_cluster.main.kubelet_identity[0].object_id + scope = each.value + role_definition_name = "AcrPull" + skip_service_principal_aad_check = true +} + +# The AKS cluster identity has the Contributor role on the AKS second resource group (MC_myResourceGroup_myAKSCluster_eastus) +# However when using a custom VNET, the AKS cluster identity needs the Network Contributor role on the VNET subnets +# used by the system node pool and by any additional node pools. +# https://learn.microsoft.com/en-us/azure/aks/configure-kubenet#prerequisites +# https://learn.microsoft.com/en-us/azure/aks/configure-azure-cni#prerequisites +# https://github.com/Azure/terraform-azurerm-aks/issues/178 +resource "azurerm_role_assignment" "network_contributor" { + for_each = var.create_role_assignment_network_contributor && (var.client_id == "" || var.client_secret == "") ? local.subnet_ids : [] + + principal_id = coalesce(try(data.azurerm_user_assigned_identity.cluster_identity[0].principal_id, azurerm_kubernetes_cluster.main.identity[0].principal_id), var.client_id) + scope = each.value + role_definition_name = "Network Contributor" + + lifecycle { + precondition { + condition = length(var.network_contributor_role_assigned_subnet_ids) == 0 + error_message = "Cannot set both of `var.create_role_assignment_network_contributor` and `var.network_contributor_role_assigned_subnet_ids`." + } + } +} + +resource "azurerm_role_assignment" "network_contributor_on_subnet" { + for_each = var.network_contributor_role_assigned_subnet_ids + + principal_id = coalesce(try(data.azurerm_user_assigned_identity.cluster_identity[0].principal_id, azurerm_kubernetes_cluster.main.identity[0].principal_id), var.client_id) + scope = each.value + role_definition_name = "Network Contributor" + + lifecycle { + precondition { + condition = !var.create_role_assignment_network_contributor + error_message = "Cannot set both of `var.create_role_assignment_network_contributor` and `var.network_contributor_role_assigned_subnet_ids`." + } + } +} + +data "azurerm_client_config" "this" {} + +data "azurerm_virtual_network" "application_gateway_vnet" { + count = local.create_role_assignments_for_application_gateway ? 1 : 0 + + name = local.application_gateway_subnet_vnet_name + resource_group_name = local.application_gateway_subnet_resource_group_name +} + +resource "azurerm_role_assignment" "application_gateway_vnet_network_contributor" { + count = local.create_role_assignments_for_application_gateway ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id + scope = data.azurerm_virtual_network.application_gateway_vnet[0].id + role_definition_name = "Network Contributor" + + lifecycle { + precondition { + condition = data.azurerm_client_config.this.subscription_id == local.application_gateway_subnet_subscription_id_for_ingress + error_message = "Application Gateway's subnet must be in the same subscription, or `var.application_gateway_for_ingress.create_role_assignments` must be set to `false`." + } + } +} + +resource "azurerm_role_assignment" "application_gateway_contributor" { + count = local.create_role_assignments_for_application_gateway ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id + scope = var.application_gateway_for_ingress.id + role_definition_name = "Contributor" + + lifecycle { + precondition { + condition = data.azurerm_client_config.this.subscription_id == local.application_gateway_subscription_id_for_ingress + error_message = "Application Gateway must be in the same subscription, or `var.application_gateway_for_ingress.create_role_assignments` must be set to `false`." + } + } +} + +data "azurerm_resource_group" "ingress_appgw" { + count = local.create_role_assignments_for_application_gateway ? 1 : 0 + + name = local.application_gateway_resource_group_for_ingress +} + +resource "azurerm_role_assignment" "application_gateway_resource_group_reader" { + count = local.create_role_assignments_for_application_gateway ? 1 : 0 + + principal_id = azurerm_kubernetes_cluster.main.ingress_application_gateway[0].ingress_application_gateway_identity[0].object_id + scope = data.azurerm_resource_group.ingress_appgw[0].id + role_definition_name = "Reader" +} diff --git a/test/e2e/terraform_aks_test.go b/test/e2e/terraform_aks_test.go index c5a7044e..1f4ec874 100644 --- a/test/e2e/terraform_aks_test.go +++ b/test/e2e/terraform_aks_test.go @@ -1,9 +1,15 @@ package e2e import ( + "io" + "net/http" "os" "regexp" + "strings" "testing" + "time" + + "github.com/stretchr/testify/require" test_helper "github.com/Azure/terraform-module-test-helper" "github.com/gruntwork-io/terratest/modules/terraform" @@ -118,3 +124,40 @@ func TestExamples_differentLocationForLogAnalyticsSolution(t *testing.T) { Vars: vars, }, nil) } + +func TestExamples_applicationGatewayIngress(t *testing.T) { + test_helper.RunE2ETest(t, "../../", "examples/application_gateway_ingress", terraform.Options{ + Upgrade: true, + }, func(t *testing.T, output test_helper.TerraformOutput) { + url, ok := output["ingress_endpoint"].(string) + require.True(t, ok) + var html string + var err error + for i := 0; i < 10; i++ { + html, err = getHTML(url) + require.NoError(t, err) + if strings.Contains(html, "Welcome to .NET") { + return + } + time.Sleep(5 * time.Second) + } + assert.Failf(t, "incorrect response from ingress: %s", html) + }) +} + +func getHTML(url string) (string, error) { + resp, err := http.Get(url) // #nosec G107 + if err != nil { + return "", err + } + defer func() { + _ = resp.Body.Close() + }() + + bytes, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(bytes), nil +} diff --git a/variables.tf b/variables.tf index f31c24b1..a55a7946 100644 --- a/variables.tf +++ b/variables.tf @@ -222,6 +222,20 @@ variable "api_server_subnet_id" { description = "(Optional) The ID of the Subnet where the API server endpoint is delegated to." } +variable "application_gateway_for_ingress" { + type = object({ + id = string + subnet_id = optional(string) + create_role_assignments = optional(bool, true) + }) + default = null + description = <<-EOT + * `id` - (Required) The ID of the Application Gateway that be used as cluster ingress. + * `subnet_id` - (Optional) The ID of the Subnet which the Application Gateway is connected to. Must be set when `create_role_assignments` is `true`. + * `create_role_assignments` - (Optional) Whether to create the corresponding role assignments or not. Defaults to `true`. +EOT +} + variable "attached_acr_id_map" { type = map(string) default = {} @@ -481,12 +495,6 @@ variable "ingress_application_gateway_enabled" { nullable = false } -variable "ingress_application_gateway_id" { - type = string - default = null - description = "The ID of the Application Gateway to integrate with the ingress controller of this Kubernetes Cluster." -} - variable "ingress_application_gateway_name" { type = string default = null