feat: support auth without GCP Service Account

fixes pfnet-research#87
nazarewk · Nov 13, 2024 · 4f3a084 · 4f3a084
1 parent 047970f
commit 4f3a084
Show file tree

Hide file tree

Showing 10 changed files with 143 additions and 99 deletions.
diff --git a/README.md b/README.md
@@ -52,6 +52,10 @@ Note: GKE or Anthos natively support injecting workload identity for pods.  This
         #           The value must be one of 'gcloud'(default) or 'direct'.
         #           Refer to the next section for 'direct' injection mode
         cloud.google.com/injection-mode: "gcloud"
+
+        # optional: Defaults to value inside `service-account-email`
+        #           
+        cloud.google.com/project: "12345"
     ```
 
 4. All new pods launched using the Kubernetes `ServiceAccount` will be mutated so that they can impersonate the GCP service account. Below is an example pod spec with the environment variables and volume fields mutated by the webhook.
@@ -62,13 +66,13 @@ Note: GKE or Anthos natively support injecting workload identity for pods.  This
     metadata:
       name: app-x-pod
       namespace: service-a
-    annotations:
-      # optional: A comma-separated list of initContainers and container names
-      #   to skip adding volumeMounts and environment variables
-      cloud.google.com/skip-containers: "init-first,sidecar"
-      # optional: Defaults to 86400, or value specified in ServiceAccount
-      #   annotation as shown in previous step, for expirationSeconds if not set
-      cloud.google.com/token-expiration: "86400"
+      annotations:
+        # optional: A comma-separated list of initContainers and container names
+        #   to skip adding volumeMounts and environment variables
+        cloud.google.com/skip-containers: "init-first,sidecar"
+        # optional: Defaults to 86400, or value specified in ServiceAccount
+        #   annotation as shown in previous step, for expirationSeconds if not set
+        cloud.google.com/token-expiration: "86400"
     spec:
       serviceAccountName: app-x
       initContainers:
@@ -167,27 +171,27 @@ To use direct injection mode:
     metadata:
       name: app-x-pod
       namespace: service-a
-    annotations:
-      # optional: A comma-separated list of initContainers and container names
-      #   to skip adding volumeMounts and environment variables
-      cloud.google.com/skip-containers: "init-first,sidecar"
-      #
-      # The Generated External Credentials Json is added as an annotation, and mounted into the container filesystem via the DownwardAPI Volume
-      #
-      cloud.google.com/external-credentials-json: |-
-        {
-          "type": "external_account",
-          "audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/on-prem-kubernetes/providers/this-cluster",
-          "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
-          "token_url": "https://sts.googleapis.com/v1/token",
-          "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/[email protected]:generateAccessToken",
-          "credential_source": {
-            "file": "/var/run/secrets/sts.googleapis.com/serviceaccount/token",
-            "format": {
-              "type": "text"
+      annotations:
+        # optional: A comma-separated list of initContainers and container names
+        #   to skip adding volumeMounts and environment variables
+        cloud.google.com/skip-containers: "init-first,sidecar"
+        #
+        # The Generated External Credentials Json is added as an annotation, and mounted into the container filesystem via the DownwardAPI Volume
+        #
+        cloud.google.com/external-credentials-json: |-
+          {
+            "type": "external_account",
+            "audience": "//iam.googleapis.com/projects/123456789/locations/global/workloadIdentityPools/on-prem-kubernetes/providers/this-cluster",
+            "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+            "token_url": "https://sts.googleapis.com/v1/token",
+            "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/[email protected]:generateAccessToken",
+            "credential_source": {
+              "file": "/var/run/secrets/sts.googleapis.com/serviceaccount/token",
+              "format": {
+                "type": "text"
+              }
             }
           }
-        }
     spec:
       serviceAccountName: app-x
       initContainers:

diff --git a/webhooks/annotations.go b/webhooks/annotations.go
@@ -43,4 +43,10 @@ const (
 	//
 	// Set to 'direct' or 'gcloud' to determine credential injection mode. Defaults to 'gcloud'.
 	InjectionModeAnnotation = "injection-mode"
+
+	//
+	// Annotations for ServiceAccount
+	//
+	// Override GCP Project normally parsed from ServiceAccount.
+	ProjectAnnotation = "project"
 )
diff --git a/webhooks/external_account_config.go b/webhooks/external_account_config.go
@@ -31,7 +31,7 @@ type ExternalAccountCredentials struct {
 	TokenInfoURL string `json:"token_info_url,omitempty"`
 	// ServiceAccountImpersonationURL is the URL for the service account impersonation request. This is only
 	// required for workload identity pools when APIs to be accessed have not integrated with UberMint.
-	ServiceAccountImpersonationURL string `json:"service_account_impersonation_url"`
+	ServiceAccountImpersonationURL string `json:"service_account_impersonation_url,omitempty"`
 	// ServiceAccountImpersonationLifetimeSeconds is the number of seconds the service account impersonation
 	// token will be valid for.
 	ServiceAccountImpersonationLifetimeSeconds int `json:"service_account_impersonation_lifetime_seconds,omitempty"`
@@ -63,9 +63,10 @@ func NewExternalAccountCredentials(aud, gsaEmail string) *ExternalAccountCredent
 			File:   filepath.Join(K8sSATokenMountPath, K8sSATokenName),
 			Format: CredentialFormat{Type: "text"},
 		},
-		ServiceAccountImpersonationURL: fmt.Sprintf("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken", gsaEmail),
 	}
-
+	if gsaEmail != "" {
+		creds.ServiceAccountImpersonationURL = fmt.Sprintf("https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken", gsaEmail)
+	}
 	return creds
 }
 

diff --git a/webhooks/identityconfig.go b/webhooks/identityconfig.go
@@ -16,6 +16,7 @@ var (
 )
 
 type GCPWorkloadIdentityConfig struct {
+	Project                  *string
 	WorkloadIdentityProvider *string
 	ServiceAccountEmail      *string
 	RunAsUser                *int64
@@ -50,6 +51,9 @@ func NewGCPWorkloadIdentityConfig(
 	if v, ok := sa.Annotations[filepath.Join(annotationDomain, AudienceAnnotation)]; ok {
 		cfg.Audience = &v
 	}
+	if v, ok := sa.Annotations[filepath.Join(annotationDomain, ProjectAnnotation)]; ok {
+		cfg.Project = &v
+	}
 
 	if v, ok := sa.Annotations[filepath.Join(annotationDomain, TokenExpirationAnnotation)]; ok {
 		seconds, err := strconv.ParseInt(v, 10, 64)
@@ -84,8 +88,8 @@ func NewGCPWorkloadIdentityConfig(
 		return nil, nil
 	}
 
-	if cfg.WorkloadIdentityProvider == nil || cfg.ServiceAccountEmail == nil {
-		return nil, fmt.Errorf("%s, %s must set at a time", filepath.Join(annotationDomain, WorkloadIdentityProviderAnnotation), filepath.Join(annotationDomain, TokenExpirationAnnotation))
+	if cfg.WorkloadIdentityProvider == nil {
+		return nil, fmt.Errorf("%s must be set", filepath.Join(annotationDomain, WorkloadIdentityProviderAnnotation))
 	}
 
 	if !workloadIdentityProviderRegex.Match([]byte(*cfg.WorkloadIdentityProvider)) {

diff --git a/webhooks/identityconfig_test.go b/webhooks/identityconfig_test.go
@@ -24,7 +24,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						Annotations: map[string]string{},
 					},
 				}
-				idConfig, err := NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err := NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(idConfig).To(BeNil())
 			})
@@ -39,7 +39,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err := NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err := NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(idConfig).To(BeEquivalentTo(&GCPWorkloadIdentityConfig{
 					WorkloadIdentityProvider: &workloadProvider,
@@ -61,7 +61,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err := NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err := NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(idConfig).To(BeEquivalentTo(&GCPWorkloadIdentityConfig{
 					WorkloadIdentityProvider: &workloadProvider,
@@ -84,7 +84,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err := NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err := NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(idConfig).To(BeEquivalentTo(&GCPWorkloadIdentityConfig{
 					WorkloadIdentityProvider: &workloadProvider,
@@ -108,7 +108,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err := NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err := NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(err).NotTo(HaveOccurred())
 				Expect(idConfig).To(BeEquivalentTo(&GCPWorkloadIdentityConfig{
 					WorkloadIdentityProvider: &workloadProvider,
@@ -134,9 +134,9 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err = NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
-				Expect(idConfig).To(BeNil())
-				Expect(err).To(MatchError(ContainSubstring("must set at a time")))
+				idConfig, err = NewGCPWorkloadIdentityConfig(annotationDomain, sa)
+				Expect(idConfig).NotTo(BeNil())
+				Expect(err).To(BeNil())
 
 				By("without workload-identity-provider annotation")
 				sa = corev1.ServiceAccount{
@@ -146,9 +146,9 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err = NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err = NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(idConfig).To(BeNil())
-				Expect(err).To(MatchError(ContainSubstring("must set at a time")))
+				Expect(err).To(MatchError(ContainSubstring("must be set")))
 			})
 		})
 		When("ServiceAccount with malformed workload-identity-provider annotation", func() {
@@ -161,7 +161,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err = NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err = NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(idConfig).To(BeNil())
 				Expect(err).To(MatchError(ContainSubstring("must be form of")))
 			})
@@ -177,7 +177,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err = NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err = NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(idConfig).To(BeNil())
 				Expect(err).To(MatchError(ContainSubstring("must be positive integer string")))
 			})
@@ -193,7 +193,7 @@ var _ = Describe("NewGCPWorkloadIdentityConfig", func() {
 						},
 					},
 				}
-				idConfig, err = NewGCPWorkloadIdentityConfig(annotaitonDomain, sa)
+				idConfig, err = NewGCPWorkloadIdentityConfig(annotationDomain, sa)
 				Expect(idConfig).To(BeNil())
 				Expect(err).To(MatchError(ContainSubstring("mode must be")))
 			})

diff --git a/webhooks/mutatepod.go b/webhooks/mutatepod.go
@@ -57,10 +57,11 @@ func (m *GCPWorkloadIdentityMutator) mutatePod(pod *corev1.Pod, idConfig GCPWork
 	//
 	// calculate project from service account
 	//
-	matches := projectRegex.FindStringSubmatch(*idConfig.ServiceAccountEmail)
 	project := ""
-	if len(matches) >= 2 {
-		project = matches[1] // the group 0 is thw whole match
+	if idConfig.Project != nil {
+		project = *idConfig.Project
+	} else if matches := projectRegex.FindStringSubmatch(*idConfig.ServiceAccountEmail); len(matches) >= 2 {
+		project = matches[1] // the group 0 is the whole match
 	}
 
 	//

diff --git a/webhooks/mutatepod_parts.go b/webhooks/mutatepod_parts.go
@@ -3,6 +3,7 @@ package webhooks
 import (
 	"fmt"
 	"path/filepath"
+	"strings"
 
 	"github.com/MakeNowJust/heredoc"
 	corev1 "k8s.io/api/core/v1"
@@ -100,6 +101,33 @@ func gcloudSetupContainer(
 	if runAsUser != nil {
 		securityContext.RunAsUser = runAsUser
 	}
+	env := []corev1.EnvVar{{
+		Name:  "GCP_WORKLOAD_IDENTITY_PROVIDER",
+		Value: workloadIdProvider,
+	}, {
+		Name:  "CLOUDSDK_CONFIG",
+		Value: GCloudConfigMountPath,
+	}}
+
+	if saEmail != "" {
+		env = append(env, corev1.EnvVar{
+			Name:  "GCP_SERVICE_ACCOUNT",
+			Value: saEmail,
+		})
+	}
+	env = append(env, projectEnvVar(project)...)
+
+	createCredsArgs := []string{
+		"$(GCP_WORKLOAD_IDENTITY_PROVIDER)",
+		fmt.Sprintf("--output-file=$(CLOUDSDK_CONFIG)/%s", ExternalCredConfigFilename),
+		fmt.Sprintf("--credential-source-file=%s", filepath.Join(K8sSATokenMountPath, K8sSATokenName)),
+	}
+	loginArgs := []string{
+		fmt.Sprintf("--cred-file=$(CLOUDSDK_CONFIG)/%s", ExternalCredConfigFilename),
+	}
+	if saEmail != "" {
+		createCredsArgs = append(createCredsArgs, "--service-account=$(GCP_SERVICE_ACCOUNT)")
+	}
 
 	c := corev1.Container{
 		Name:  GCloudSetupInitContainerName,
@@ -108,27 +136,16 @@ func gcloudSetupContainer(
 			"sh", "-c",
 			heredoc.Docf(`
 				gcloud iam workload-identity-pools create-cred-config \
-				  $(GCP_WORKLOAD_IDENTITY_PROVIDER) \
-				  --service-account=$(GCP_SERVICE_ACCOUNT) \
-				  --output-file=$(CLOUDSDK_CONFIG)/%s \
-				  --credential-source-file=%s
-				gcloud auth login --cred-file=$(CLOUDSDK_CONFIG)/%s
-			`, ExternalCredConfigFilename,
-				filepath.Join(K8sSATokenMountPath, K8sSATokenName),
-				ExternalCredConfigFilename,
+				  %s
+				gcloud auth login \
+				  %s
+			`,
+				strings.Join(createCredsArgs, " \\\n  "),
+				strings.Join(loginArgs, " \\\n  "),
 			),
 		},
-		VolumeMounts: volumeMountsToAddOrReplace(GCloudMode),
-		Env: []corev1.EnvVar{{
-			Name:  "GCP_WORKLOAD_IDENTITY_PROVIDER",
-			Value: workloadIdProvider,
-		}, {
-			Name:  "GCP_SERVICE_ACCOUNT",
-			Value: saEmail,
-		}, {
-			Name:  "CLOUDSDK_CONFIG",
-			Value: GCloudConfigMountPath,
-		}, projectEnvVar(project)},
+		VolumeMounts:    volumeMountsToAddOrReplace(GCloudMode),
+		Env:             env,
 		SecurityContext: securityContext,
 	}
 	if resources != nil {
@@ -191,19 +208,25 @@ func envVarsToAddOrReplace(mode InjectionMode) []corev1.EnvVar {
 }
 
 func envVarsToAddIfNotPresent(region, project string) []corev1.EnvVar {
-	return []corev1.EnvVar{cloudSDKComputeRegionEnvVar(region), projectEnvVar(project)}
+	return append(cloudSDKComputeRegionEnvVar(region), projectEnvVar(project)...)
 }
 
-func cloudSDKComputeRegionEnvVar(region string) corev1.EnvVar {
-	return corev1.EnvVar{
-		Name:  "CLOUDSDK_COMPUTE_REGION",
-		Value: region,
+func cloudSDKComputeRegionEnvVar(region string) (ret []corev1.EnvVar) {
+	if region != "" {
+		ret = append(ret, corev1.EnvVar{
+			Name:  "CLOUDSDK_COMPUTE_REGION",
+			Value: region,
+		})
 	}
+	return
 }
 
-func projectEnvVar(project string) corev1.EnvVar {
-	return corev1.EnvVar{
-		Name:  "CLOUDSDK_CORE_PROJECT",
-		Value: project,
+func projectEnvVar(project string) (ret []corev1.EnvVar) {
+	if project != "" {
+		ret = append(ret, corev1.EnvVar{
+			Name:  "CLOUDSDK_CORE_PROJECT",
+			Value: project,
+		})
 	}
+	return
 }
diff --git a/webhooks/mutatepod_parts_test.go b/webhooks/mutatepod_parts_test.go
@@ -24,10 +24,11 @@ func TestGcloudSetupContainer(t *testing.T) {
 			"sh", "-c",
 			`gcloud iam workload-identity-pools create-cred-config \
   $(GCP_WORKLOAD_IDENTITY_PROVIDER) \
-  --service-account=$(GCP_SERVICE_ACCOUNT) \
   --output-file=$(CLOUDSDK_CONFIG)/federation.json \
-  --credential-source-file=/var/run/secrets/sts.googleapis.com/serviceaccount/token
-gcloud auth login --cred-file=$(CLOUDSDK_CONFIG)/federation.json
+  --credential-source-file=/var/run/secrets/sts.googleapis.com/serviceaccount/token \
+  --service-account=$(GCP_SERVICE_ACCOUNT)
+gcloud auth login \
+  --cred-file=$(CLOUDSDK_CONFIG)/federation.json
 `,
 		},
 		VolumeMounts: []corev1.VolumeMount{
@@ -46,14 +47,14 @@ gcloud auth login --cred-file=$(CLOUDSDK_CONFIG)/federation.json
 				Name:  "GCP_WORKLOAD_IDENTITY_PROVIDER",
 				Value: workloadIdProvider,
 			},
-			{
-				Name:  "GCP_SERVICE_ACCOUNT",
-				Value: saEmail,
-			},
 			{
 				Name:  "CLOUDSDK_CONFIG",
 				Value: "/var/run/secrets/gcloud/config",
 			},
+			{
+				Name:  "GCP_SERVICE_ACCOUNT",
+				Value: saEmail,
+			},
 			{
 				Name:  "CLOUDSDK_CORE_PROJECT",
 				Value: project,