diff --git a/capz/gmsa/ci-gmsa.sh b/capz/gmsa/ci-gmsa.sh index b93a3fac..3b29d6e7 100755 --- a/capz/gmsa/ci-gmsa.sh +++ b/capz/gmsa/ci-gmsa.sh @@ -34,7 +34,7 @@ SCRIPT_ROOT=$(dirname "${BASH_SOURCE[0]}") CI_RG="${CI_RG:-capz-ci}" GMSA_NODE_RG="${GMSA_NODE_RG:-gmsa-dc}" AZURE_LOCATION="${AZURE_LOCATION:-westus2}" -GMSA_KEYVAULT="${GMSA_KEYVAULT:-$CI_RG-gmsa}" +GMSA_KEYVAULT="${GMSA_KEYVAULT:-$CI_RG-gmsa-community}" # The VM requires setup that needs Role Assignment permissions # This script checks that all that has been configured properly before creating the Azure VM diff --git a/capz/gmsa/domain.init.tmpl b/capz/gmsa/domain.init.tmpl index d6a7f3bf..5fa03cbd 100644 --- a/capz/gmsa/domain.init.tmpl +++ b/capz/gmsa/domain.init.tmpl @@ -16,7 +16,7 @@ write_files: exit } - $keyvaultName="${GMSA_KEYVAULT:-${CI_RG}-gmsa}" + $keyvaultName="${GMSA_KEYVAULT:-${CI_RG}-gmsa-community}" $gmsaAdminName="gmsa-admin-${GMSA_ID}" $gmsaUserName="gmsa-e2e-user-${GMSA_ID}" $resourcegroup="${CI_RG}" diff --git a/capz/gmsa/readme.md b/capz/gmsa/readme.md index 62c023a2..f5d1dd8d 100644 --- a/capz/gmsa/readme.md +++ b/capz/gmsa/readme.md @@ -6,11 +6,11 @@ A general overview of the required to run the gMSA: ![diagram of gmsa architecture with required vms and keyvaults](images/gmsa-design.png) -## Subscrition Setup +## Subscription Setup -The Azure subscription requires a few pre-requisits to be able run the tests. This is becuase the Managed Identities used with the gMSA plugin need Subscription Owner Permissions to create. +The Azure subscription requires a few pre-requisites to be able run the tests. This is because the Managed Identities used with the gMSA plugin need Subscription Owner Permissions to create. -To accomidate this there is a script `setup-gmsa.sh` that does this configuration. +The setup required can be found at the [k8s.io repo](https://github.com/kubernetes/k8s.io/tree/main/infra/azure/terraform/capz). ## Domain VM Creation diff --git a/capz/gmsa/setup-gmsa.sh b/capz/gmsa/setup-gmsa.sh deleted file mode 100755 index be2ada1b..00000000 --- a/capz/gmsa/setup-gmsa.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash - -# Copyright 2021 The Kubernetes Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -############################################################################### - -set -o errexit -set -o nounset -set -o pipefail - -export CAPZ_DIR="${CAPZ_DIR:-"${GOPATH}/src/sigs.k8s.io/cluster-api-provider-azure"}" -: "${CAPZ_DIR:?Environment variable empty or not defined.}" -if [[ ! -d $CAPZ_DIR ]]; then - log "Must have capz repo present" -fi - -# shellcheck disable=SC1091 -source "${CAPZ_DIR}/hack/ensure-azcli.sh" -# shellcheck disable=SC1091 -source "${CAPZ_DIR}/hack/parse-prow-creds.sh" - -CI_RG="${CI_RG:-capz-ci}" -AZURE_LOCATION="${AZURE_LOCATION:-westus2}" -GMSA_KEYVAULT="${GMSA_KEYVAULT:-$CI_RG-gmsa}" - -: "${CI_CLIENT_ID:?Environment variable empty or not defined.}" - -rolejson=$(cat <<-ROLE -{ - "Name": "gMSA", - "Description": "Required permissions for gmsa to read properties of subscriptions and managed identities", - "Actions": [ - "Microsoft.Resources/subscriptions/read", - "Microsoft.ManagedIdentity/userAssignedIdentities/read" - ], - "AssignableScopes": ["/subscriptions/$AZURE_SUBSCRIPTION_ID"] -} -ROLE -) - -main() { - set -x - if [[ "$(az group exists --name "${CI_RG}")" == "false" ]]; then - az group create --name "$CI_RG" --location "$AZURE_LOCATION" - fi - - keyvaultid=$(az keyvault show --name "${GMSA_KEYVAULT}" -g "$CI_RG" --query "id" || true) - if [[ -z $keyvaultid ]]; then - az keyvault create --name "${GMSA_KEYVAULT}" -g "$CI_RG" - fi - - # Give permissions to vms identity to write to keyvault during the domain creation - domainid=$(az identity show --name domain-vm-identity --resource-group "$CI_RG" --query 'principalId' -o tsv || true) - if [[ -z $domainid ]]; then - domainid=$(az identity create -g "$CI_RG" -n domain-vm-identity --query 'principalId' -o tsv) - fi - az keyvault set-policy --name "${GMSA_KEYVAULT}" --object-id "$domainid" --secret-permissions set - - # The identity also needs to be able to read subscription id and managed identities - # This is a custom role to make this least priviliged. - # The creator must have permissions to create roles and assignements. - customSubRole=$(az role definition list --custom-role-only --query [].roleName -o tsv) - if ! [[ $customSubRole =~ "gMSA" ]]; then - echo "If the following fails you need to have someone with permissions create this role" - az role definition create --role-definition "$rolejson" - fi - # on first run this takes ~1-2 mins - until az role assignment create --role "gMSA" --assignee-object-id "$domainid" --assignee-principal-type ServicePrincipal &> /dev/null - do - echo "wait for role propgation" - sleep 10 - done - - # create identity for the worker VMs to use to get keyvault secrets - # this identity needs to be assigned to the the Worker nodes that is labeled during e2e set up. - userId=$(az identity show --name gmsa-user-identity --resource-group "$CI_RG" --query 'principalId' -o tsv || true) - if [[ -z $userId ]]; then - userId=$(az identity create -g "$CI_RG" -n gmsa-user-identity --query 'principalId' -o tsv) - fi - az keyvault set-policy --name "${GMSA_KEYVAULT}" --object-id "$userId" --secret-permissions get - - cloudproviderId=$(az identity show --name cloud-provider-user-identity --resource-group "$CI_RG" --query 'principalId' -o tsv || true) - if [[ -z $cloudproviderId ]]; then - cloudproviderId=$(az identity create -g "$CI_RG" -n cloud-provider-user-identity --query 'principalId' -o tsv) - fi - - # on first run this takes ~1-2 mins - until az role assignment create --role "Contributor" --assignee-object-id "$cloudproviderId" --assignee-principal-type ServicePrincipal &> /dev/null - do - echo "wait for role propgation" - sleep 10 - done - - until az role assignment create --role "AcrPull" --assignee-object-id "$cloudproviderId" --assignee-principal-type ServicePrincipal &> /dev/null - do - echo "wait for role propgation" - sleep 10 - done - - # make sure the service CI principal has read access to set up tests - ciSP=$(az ad sp show --id "$CI_CLIENT_ID" --query id -o tsv) - az keyvault set-policy --name "${GMSA_KEYVAULT}" --object-id "$ciSP" --secret-permissions get delete list purge -} - -main diff --git a/capz/run-capz-e2e.sh b/capz/run-capz-e2e.sh index 388a664f..4019dc2e 100755 --- a/capz/run-capz-e2e.sh +++ b/capz/run-capz-e2e.sh @@ -86,9 +86,9 @@ create_gmsa_domain(){ export CI_RG="${CI_RG:-capz-ci}" export GMSA_ID="${RANDOM}" export GMSA_NODE_RG="gmsa-dc-${GMSA_ID}" - export GMSA_KEYVAULT_URL="https://${GMSA_KEYVAULT:-$CI_RG-gmsa}.vault.azure.net" + export GMSA_KEYVAULT_URL="https://${GMSA_KEYVAULT:-$CI_RG-gmsa-community}.vault.azure.net" - log "setting up domain vm in $GMSA_NODE_RG with keyvault $CI_RG-gmsa" + log "setting up domain vm in $GMSA_NODE_RG with keyvault $CI_RG-gmsa-community" "${SCRIPT_ROOT}/gmsa/ci-gmsa.sh" # export the ip Address so it can be used in e2e test @@ -134,8 +134,8 @@ run_capz_e2e_cleanup() { # clean up GMSA NODE RG if [[ -n ${GMSA:-} ]]; then echo "Cleaning up gMSA resources $GMSA_NODE_RG with keyvault $GMSA_KEYVAULT_URL" - az keyvault secret list --vault-name "${GMSA_KEYVAULT:-$CI_RG-gmsa}" --query "[? contains(name, '${GMSA_ID}')].name" -o tsv | while read -r secret ; do - az keyvault secret delete -n "$secret" --vault-name "${GMSA_KEYVAULT:-$CI_RG-gmsa}" + az keyvault secret list --vault-name "${GMSA_KEYVAULT:-$CI_RG-gmsa-community}" --query "[? contains(name, '${GMSA_ID}')].name" -o tsv | while read -r secret ; do + az keyvault secret delete -n "$secret" --vault-name "${GMSA_KEYVAULT:-$CI_RG-gmsa-community}" done az group delete --name "$GMSA_NODE_RG" --no-wait -y --force-deletion-types=Microsoft.Compute/virtualMachines --force-deletion-types=Microsoft.Compute/virtualMachineScaleSets || true