csm: modify o11y test to use proxyless bootstrap injector (#141)

As part of CSM Gateway for Mesh GA - we have already implemented a Proxyless Bootstrap Injector in GCP which will allow clusters that are onboarded to CSM Gateway for Mesh to get the following injected into the pod with the right labels: 1. initContainer with TD Bootstrap Injector + volume Mount where the config output is generated 2. CSM O11y related labels and envvars. The test cluster also had to be updated to follow the new onboarding steps. I will create a CL internally to update the cluster creation script Testing: [here](https://source.cloud.google.com/results/invocations/3e3a284a-857d-4a1a-9591-d78efff9008e)
grpc · Dec 4, 2024 · 3e8a89b · 3e8a89b
1 parent 8e62d77
commit 3e8a89b
Show file tree

Hide file tree

Showing 11 changed files with 802 additions and 5 deletions.
diff --git a/.kokoro/psm_interop_kokoro_lib.sh b/.kokoro/psm_interop_kokoro_lib.sh
@@ -318,6 +318,7 @@ psm::csm::get_tests() {
     "gamma.affinity_test"
     "gamma.affinity_session_drain_test"
     "gamma.csm_observability_test"
+    "gamma.csm_observability_test_with_injection"
     "app_net_ssa_test"
     "app_net_csm_observability_test"
   )
@@ -781,7 +782,7 @@ activate_gke_cluster() {
       GKE_CLUSTER_ZONE="us-central1-a"
       ;;
     GKE_CLUSTER_PSM_CSM)
-      GKE_CLUSTER_NAME="psm-interop-csm"
+      GKE_CLUSTER_NAME="psm-interop-csm-gateway"
       GKE_CLUSTER_REGION="us-central1"
       ;;
     GKE_CLUSTER_PSM_GAMMA)

diff --git a/config/grpc-testing.cfg b/config/grpc-testing.cfg
@@ -1,5 +1,6 @@
 --flagfile=config/common.cfg
 --project=grpc-testing
+--project_number=830293263384
 --network=default-vpc
 --gcp_service_account[email protected]
 --private_api_key_secret_name=projects/830293263384/secrets/xds-interop-tests-private-api-access-key

diff --git a/config/local-dev.cfg.example b/config/local-dev.cfg.example
@@ -7,6 +7,7 @@
 
 ## Project settings
 --project=${PROJECT_ID}
+--project_number=${PROJECT_NUMBER}
 --gcp_service_account=${WORKLOAD_SA_EMAIL}
 --private_api_key_secret_name=projects/${PROJECT_NUMBER}/secrets/xds-interop-tests-private-api-access-key
 

diff --git a/framework/infrastructure/k8s.py b/framework/infrastructure/k8s.py
@@ -172,7 +172,7 @@ def grpc_route(self, version: str) -> dynamic_res.Resource:
         api_name = "gateway.networking.k8s.io"
         kind = "GRPCRoute"
         supported_versions = {
-            "v1alpha2",
+            "v1",
         }
         if version not in supported_versions:
             raise NotImplementedError(
@@ -185,7 +185,7 @@ def grpc_route(self, version: str) -> dynamic_res.Resource:
     def http_route(self, version: str) -> dynamic_res.Resource:
         api_name = "gateway.networking.k8s.io"
         kind = "HTTPRoute"
-        supported_versions = {"v1alpha2", "v1beta1"}
+        supported_versions = {"v1", "v1beta1"}
         if version not in supported_versions:
             raise NotImplementedError(
                 f"{kind} {api_name}/{version} not implemented."
@@ -331,7 +331,7 @@ def api_gke_mesh(self) -> dynamic_res.Resource:
     @functools.cache
     def api_grpc_route(self) -> dynamic_res.Resource:
         return self._get_dynamic_api(
-            "gateway.networking.k8s.io/v1alpha2",
+            "gateway.networking.k8s.io/v1",
             "GRPCRoute",
         )
 

diff --git a/framework/xds_flags.py b/framework/xds_flags.py
@@ -22,6 +22,9 @@
 PROJECT = flags.DEFINE_string(
     "project", default=None, help="(required) GCP Project ID."
 )
+PROJECT_NUMBER = flags.DEFINE_string(
+    "project_number", default=None, help="GCP Project Number."
+)
 RESOURCE_PREFIX = flags.DEFINE_string(
     "resource_prefix",
     default=None,

diff --git a/framework/xds_k8s_testcase.py b/framework/xds_k8s_testcase.py
@@ -132,6 +132,7 @@ class XdsKubernetesBaseTestCase(base_testcase.BaseTestCase):
     secondary_k8s_api_manager: Optional[k8s.KubernetesApiManager] = None
     network: str
     project: str
+    project_number: str
     resource_prefix: str
     resource_suffix: str = ""
     # Whether to randomize resources names for each test by appending a
@@ -176,6 +177,7 @@ def setUpClass(cls):
 
         # GCP
         cls.project = xds_flags.PROJECT.value
+        cls.project_number = xds_flags.PROJECT_NUMBER.value
         cls.network = xds_flags.NETWORK.value
         cls.gcp_service_account = xds_k8s_flags.GCP_SERVICE_ACCOUNT.value
         cls.td_bootstrap_image = xds_k8s_flags.TD_BOOTSTRAP_IMAGE.value

diff --git a/kubernetes-manifests/gamma/client.deployment.yaml b/kubernetes-manifests/gamma/client.deployment.yaml
@@ -0,0 +1,77 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${deployment_name}
+  namespace: ${namespace_name}
+  labels:
+    app: ${app_label}
+    deployment_id: ${deployment_id}
+    owner: xds-k8s-interop-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: ${app_label}
+      deployment_id: ${deployment_id}
+  template:
+    metadata:
+      labels:
+        app: ${app_label}
+        deployment_id: ${deployment_id}
+        owner: xds-k8s-interop-test
+    spec:
+      % if service_account_name:
+      serviceAccountName: ${service_account_name}
+      % endif
+      containers:
+        - name: ${deployment_name}
+          image: ${image_name}
+          imagePullPolicy: Always
+          startupProbe:
+            tcpSocket:
+              port: ${stats_port}
+            periodSeconds: 3
+            ## Extend the number of probes well beyond the duration of the test
+            ## driver waiting for the container to start.
+            failureThreshold: 1000
+          args:
+            - "--server=${server_target}"
+            - "--stats_port=${stats_port}"
+            - "--qps=${qps}"
+            - "--rpc=${rpc}"
+            - "--metadata=${metadata}"
+            % if request_payload_size > 0:
+            - "--request_payload_size=${request_payload_size}"
+            % endif
+            % if response_payload_size > 0:
+            - "--response_payload_size=${response_payload_size}"
+            % endif
+            - "--print_response=${print_response}"
+            % if enable_csm_observability:
+            - "--enable_csm_observability=true"
+            % endif
+          ## ####################################################################
+          ## TODO(cl/698639274): Remove the these env once this CL is released.
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: OTEL_RESOURCE_ATTRIBUTES
+              value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME)
+          ## #####################################################################
+          ports:
+            - containerPort: ${stats_port}
+          resources:
+            limits:
+              cpu: 800m
+              memory: 512Mi
+            requests:
+              cpu: 100m
+              memory: 512Mi
+...
diff --git a/kubernetes-manifests/gamma/namespace.yaml b/kubernetes-manifests/gamma/namespace.yaml
@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: ${namespace_name}
+  labels:
+    name: ${namespace_name}
+    owner: xds-k8s-interop-test
+    mesh.cloud.google.com/csm-injection: proxyless
+...
diff --git a/kubernetes-manifests/gamma/route_grpc.yaml b/kubernetes-manifests/gamma/route_grpc.yaml
@@ -1,6 +1,6 @@
 ---
 kind: GRPCRoute
-apiVersion: gateway.networking.k8s.io/v1alpha2
+apiVersion: gateway.networking.k8s.io/v1
 metadata:
   name: ${route_name}
   namespace: ${namespace_name}

diff --git a/kubernetes-manifests/gamma/server.deployment.yaml b/kubernetes-manifests/gamma/server.deployment.yaml
@@ -0,0 +1,75 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${deployment_name}
+  namespace: ${namespace_name}
+  labels:
+    app: ${app_label}
+    deployment_id: ${deployment_id}
+    owner: xds-k8s-interop-test
+spec:
+  replicas: ${replica_count}
+  selector:
+    matchLabels:
+      app: ${app_label}
+      deployment_id: ${deployment_id}
+  template:
+    metadata:
+      labels:
+        app: ${app_label}
+        deployment_id: ${deployment_id}
+        owner: xds-k8s-interop-test
+    spec:
+      % if service_account_name:
+      serviceAccountName: ${service_account_name}
+      % endif
+      % if termination_grace_period_seconds:
+      terminationGracePeriodSeconds: ${termination_grace_period_seconds}
+      % endif
+      containers:
+        - name: ${deployment_name}
+          image: ${image_name}
+          imagePullPolicy: Always
+          ## TODO:(lsafran) test without this when ipv6 HC fw rules are in place
+          % if address_type != "ipv6":
+          startupProbe:
+            tcpSocket:
+              port: ${test_port}
+            periodSeconds: 3
+            ## Extend the number of probes well beyond the duration of the test
+            ## driver waiting for the container to start.
+            failureThreshold: 1000
+          % endif
+          args:
+            - "--port=${test_port}"
+            % if enable_csm_observability:
+            - "--enable_csm_observability=true"
+            % endif
+            % if address_type:
+            - "--address_type=${address_type}"
+            % endif
+          ## ####################################################################
+          ## TODO(cl/698639274): Remove the these env once this CL is released.
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: NAMESPACE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: OTEL_RESOURCE_ATTRIBUTES
+              value: k8s.pod.name=$(POD_NAME),k8s.namespace.name=$(NAMESPACE_NAME)
+          ## #####################################################################
+          ports:
+            - containerPort: ${test_port}
+          resources:
+            limits:
+              cpu: 800m
+              memory: 512Mi
+            requests:
+              cpu: 100m
+              memory: 512Mi
+...