fix: run abe2e tests in custom location (#3672)

Co-authored-by: Cameron Meissner <[email protected]>

e2e/cluster.go

@@ -16,6 +16,8 @@ import (
 
 const (
 	managedClusterResourceType = "Microsoft.ContainerService/managedClusters"
+
+	vmSizeStandardDS2v2 = "Standard_DS2_v2"
 )
 
 type clusterParameters map[string]string
@@ -48,6 +50,22 @@ func (c clusterConfig) needsPreparation() bool {
 	return c.kube == nil || c.parameters == nil || c.subnetId == ""
 }
 
+// This map is used during cluster creation to check what VM size should
+// be used to create the single default agentpool used for running cluster essentials
+// and the jumpbox pods/resources. This is mainly need for regions where we can't use
+// the default Standard_DS2_v2 VM size due to quota/capacity.
+var locationToDefaultClusterAgentPoolVMSize = map[string]string{
+	// TODO: add mapping for southcentralus to perform h100 testing
+}
+
+func getDefaultAgentPoolVMSize(location string) string {
+	defaultAgentPoolVMSize, hasDefaultAgentPoolVMSizeForLocation := locationToDefaultClusterAgentPoolVMSize[location]
+	if !hasDefaultAgentPoolVMSizeForLocation {
+		defaultAgentPoolVMSize = vmSizeStandardDS2v2
+	}
+	return defaultAgentPoolVMSize
+}
+
 func isExistingResourceGroup(ctx context.Context, cloud *azureClient, resourceGroupName string) (bool, error) {
 	rgExistence, err := cloud.resourceGroupClient.CheckExistence(ctx, resourceGroupName, nil)
 	if err != nil {
@@ -57,26 +75,27 @@ func isExistingResourceGroup(ctx context.Context, cloud *azureClient, resourceGr
 	return rgExistence.Success, nil
 }
 
-func ensureResourceGroup(ctx context.Context, cloud *azureClient, resourceGroupName string) error {
-	log.Printf("ensuring resource group %q...", resourceGroupName)
+func ensureResourceGroup(ctx context.Context, cloud *azureClient, suiteConfig *suiteConfig) error {
+	suiteConfig.resourceGroupName = fmt.Sprintf(abe2eResourceGroupNameTemplate, suiteConfig.location)
+	log.Printf("ensuring resource group %q...", suiteConfig.resourceGroupName)
 
-	rgExists, err := isExistingResourceGroup(ctx, cloud, resourceGroupName)
+	rgExists, err := isExistingResourceGroup(ctx, cloud, suiteConfig.resourceGroupName)
 	if err != nil {
 		return err
 	}
 
 	if !rgExists {
 		_, err = cloud.resourceGroupClient.CreateOrUpdate(
 			ctx,
-			resourceGroupName,
+			suiteConfig.resourceGroupName,
 			armresources.ResourceGroup{
-				Location: to.Ptr(e2eTestLocation),
-				Name:     to.Ptr(resourceGroupName),
+				Location: to.Ptr(suiteConfig.location),
+				Name:     to.Ptr(suiteConfig.resourceGroupName),
 			},
 			nil)
 
 		if err != nil {
-			return fmt.Errorf("failed to create RG %q: %w", resourceGroupName, err)
+			return fmt.Errorf("failed to create RG %q: %w", suiteConfig.resourceGroupName, err)
 		}
 	}
 
@@ -408,6 +427,9 @@ func generateClusterName(r *mrand.Rand) string {
 }
 
 func getBaseClusterModel(clusterName, location string) armcontainerservice.ManagedCluster {
+	defaultAgentPoolVMSize := getDefaultAgentPoolVMSize(location)
+	log.Printf("will attempt to use VM size %q for default agentpool of cluster %q", defaultAgentPoolVMSize, clusterName)
+
 	return armcontainerservice.ManagedCluster{
 		Name:     to.Ptr(clusterName),
 		Location: to.Ptr(location),
@@ -417,7 +439,7 @@ func getBaseClusterModel(clusterName, location string) armcontainerservice.Manag
 				{
 					Name:         to.Ptr("nodepool1"),
 					Count:        to.Ptr[int32](2),
-					VMSize:       to.Ptr("Standard_DS2_v2"),
+					VMSize:       to.Ptr(defaultAgentPoolVMSize),
 					MaxPods:      to.Ptr[int32](110),
 					OSType:       to.Ptr(armcontainerservice.OSTypeLinux),
 					Type:         to.Ptr(armcontainerservice.AgentPoolTypeVirtualMachineScaleSets),

e2e/const.go

@@ -1,8 +1,8 @@
 package e2e_test
 
 const (
-	testClusterNameTemplate = "agentbaker-e2e-test-cluster-%s"
-	defaultAzureTokenScope  = "https://management.azure.com/.default"
-	defaultNamespace        = "default"
-	e2eTestLocation         = "eastus"
+	testClusterNameTemplate        = "agentbaker-e2e-test-cluster-%s"
+	defaultAzureTokenScope         = "https://management.azure.com/.default"
+	defaultNamespace               = "default"
+	abe2eResourceGroupNameTemplate = "abe2e-%s"
 )

e2e/e2e-local.sh

@@ -3,16 +3,12 @@
 set -euxo pipefail
 
 : "${SUBSCRIPTION_ID:=8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8}" #Azure Container Service - Test Subscription
-: "${RESOURCE_GROUP_NAME:=agentbaker-e2e-tests}"
 : "${LOCATION:=eastus}"
-: "${CLUSTER_NAME:=agentbaker-e2e-test-cluster}"
 : "${AZURE_TENANT_ID:=72f988bf-86f1-41af-91ab-2d7cd011db47}"
 : "${TIMEOUT:=30m}"
 
 export SUBSCRIPTION_ID
-export RESOURCE_GROUP_NAME
 export LOCATION
-export CLUSTER_NAME
 export AZURE_TENANT_ID
 
 go version

e2e/nodebootstrapping.go

@@ -65,7 +65,7 @@ func getBaseNodeBootstrappingConfiguration(
 	cloud *azureClient,
 	suiteConfig *suiteConfig,
 	clusterParams clusterParameters) (*datamodel.NodeBootstrappingConfiguration, error) {
-	nbc := baseTemplate()
+	nbc := baseTemplate(suiteConfig.location)
 	nbc.ContainerService.Properties.CertificateProfile.CaCertificate = clusterParams["/etc/kubernetes/certs/ca.crt"]
 
 	bootstrapKubeconfig := clusterParams["/var/lib/kubelet/bootstrap-kubeconfig"]

e2e/suite_config.go

@@ -16,9 +16,8 @@ type suiteConfig struct {
 
 func newSuiteConfig() (*suiteConfig, error) {
 	var environment = map[string]string{
-		"SUBSCRIPTION_ID":     "",
-		"LOCATION":            "",
-		"RESOURCE_GROUP_NAME": "",
+		"SUBSCRIPTION_ID": "",
+		"LOCATION":        "",
 	}
 
 	for k := range environment {
@@ -30,11 +29,10 @@ func newSuiteConfig() (*suiteConfig, error) {
 	}
 
 	config := &suiteConfig{
-		subscription:      environment["SUBSCRIPTION_ID"],
-		location:          environment["LOCATION"],
-		resourceGroupName: environment["RESOURCE_GROUP_NAME"],
-		scenariosToRun:    strToBoolMap(os.Getenv("SCENARIOS_TO_RUN")),
-		keepVMSS:          os.Getenv("KEEP_VMSS") == "true",
+		subscription:   environment["SUBSCRIPTION_ID"],
+		location:       environment["LOCATION"],
+		scenariosToRun: strToBoolMap(os.Getenv("SCENARIOS_TO_RUN")),
+		keepVMSS:       os.Getenv("KEEP_VMSS") == "true",
 	}
 
 	include := os.Getenv("SCENARIOS_TO_RUN")

e2e/suite_test.go

@@ -37,7 +37,7 @@ func Test_All(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if err := ensureResourceGroup(ctx, cloud, suiteConfig.resourceGroupName); err != nil {
+	if err := ensureResourceGroup(ctx, cloud, suiteConfig); err != nil {
 		t.Fatal(err)
 	}
 

e2e/template.go

@@ -10,15 +10,15 @@ import (
 // TODO(ace): minimize the actual required defaults.
 // this is what we previously used for bash e2e from e2e/nodebootstrapping_template.json.
 // which itself was extracted from baker_test.go logic, which was inherited from aks-engine.
-func baseTemplate() *datamodel.NodeBootstrappingConfiguration {
+func baseTemplate(location string) *datamodel.NodeBootstrappingConfiguration {
 	var (
 		trueConst  = true
 		falseConst = false
 	)
 	return &datamodel.NodeBootstrappingConfiguration{
 		ContainerService: &datamodel.ContainerService{
 			ID:       "",
-			Location: "eastus",
+			Location: location,
 			Name:     "",
 			Plan:     nil,
 			Tags:     map[string]string(nil),