ci: Add Windows CNIv2 to CNI Load Test Pipeline (#2279)

* add: env var to skip tests from make test-load

* add: windows nodepool to existing make calls

* add: Windows CNIv2

* refactor: change cniv2 file name

* style: pipeline yaml layout

* fix: label selector

* ci: increase timeout for restartNode job

* ci: ensure kubeproxy is running on windows

* chore: clean up comments

* ci: capture windows node count

.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml

@@ -46,6 +46,7 @@ stages:
       name: "$(BUILD_POOL_NAME_DEFAULT)"
     dependsOn:
       - create_${{ parameters.name }}
+      - publish
       - setup
     displayName: "Cilium Test - ${{ parameters.name }}"
     jobs:

.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml

@@ -8,6 +8,7 @@ parameters:
 
 jobs:
   - job: CNI_${{ parameters.os }}
+    condition: and( not(canceled()), not(failed()) )
     displayName: CNI k8s E2E ${{ parameters.os }}
     dependsOn: ${{ parameters.dependsOn }}
     pool:

.pipelines/cni/load-test-templates/create-cluster-template.yaml

@@ -5,6 +5,8 @@ parameters:
   vmSize: ""
   vmSizeWin: ""
   osSKU: Ubuntu
+  os: linux
+  nodeCountWin: 2
 
 steps:
   - task: AzureCLI@1
@@ -18,10 +20,10 @@ steps:
         make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }}
         make -C ./hack/aks ${{ parameters.clusterType }} \
         AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \
-        CLUSTER=${{ parameters.clusterName }} NODE_COUNT=${{ parameters.nodeCount }} \
+        CLUSTER=${{ parameters.clusterName }} NODE_COUNT=${{ parameters.nodeCount }} NODE_COUNT_WIN=${{ parameters.nodeCountWin }} \
         VM_SIZE=${{ parameters.vmSize }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} \
         WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} \
-        OS_SKU=${{parameters.osSKU}}
+        OS_SKU=${{parameters.osSKU}} OS=${{parameters.os}}
         echo "Cluster successfully created"
 
     name: "CreateAksCluster"

.pipelines/cni/load-test-templates/restart-cns-template.yaml

@@ -20,17 +20,17 @@ steps:
         echo "Ensure there are pods scheduled on each node"
         cd test/integration/load
         scale=$(( ${{ parameters.scaleup }} * ${{ parameters.nodeCount }} ))
-        REPLICAS=$scale go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load
+        REPLICAS=$scale OS_TYPE=${{ parameters.os }} go test -count 1 -timeout 30m -tags load -run ^TestScaleDeployment$ -tags=load
         cd ../../../
 
         echo "Validate pod IP assignment before CNS restart"
-        make test-validate-state CNI_TYPE=${{ parameters.cni }}
+        make test-validate-state OS_TYPE=${{ parameters.os }} CNI_TYPE=${{ parameters.cni }}
         echo "restart CNS"
         kubectl rollout restart ds azure-cns -n kube-system
         kubectl rollout status ds azure-cns -n kube-system
         kubectl get pod -owide -A
         echo "Validate pod IP assignment after CNS restart"
-        make test-validate-state CNI_TYPE=${{ parameters.cni }}
+        make test-validate-state OS_TYPE=${{ parameters.os }} CNI_TYPE=${{ parameters.cni }}
     name: "restartCNS"
     displayName: "Restart CNS and Validate pods"
     retryCountOnTaskFailure: 3

.pipelines/cni/load-test-templates/restart-node-template.yaml

@@ -12,7 +12,6 @@ steps:
       scriptType: "bash"
       addSpnToEnvironment: true
       inlineScript: |
-        echo "Scale up the pods and immediated restart the nodes"
         clusterName=${{ parameters.clusterName }}
         make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
         make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
@@ -21,12 +20,13 @@ steps:
         for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(LOCATION) --query "[].name" -o tsv); do
           make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(LOCATION) VMSS_NAME=$val
         done
+        kubectl get pods -n kube-system -owide
 
         if ! [ ${{ parameters.cni }} = 'cniv1' ]; then
           echo "Ensure CNS daemonsets are ready and available"
           labels="app=azure-cns app=azure-cns-win"
           for label in $labels; do
-            kubectl get pods -n kube-system -l $label
+            kubectl get pods -n kube-system -l k8s-$label -owide
             kubectl rollout status ds -n kube-system -l $label
           done
         fi
@@ -36,6 +36,20 @@ steps:
     name: "RestartNodes"
     displayName: "Restart Nodes"
 
+  - ${{ if contains(parameters.os, 'windows') }}:
+    - script: |
+        kubectl apply -f test/integration/manifests/load/privileged-daemonset-windows.yaml
+        kubectl rollout status -n kube-system ds privileged-daemonset
+
+        kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows -owide
+        pods=`kubectl get pod -n kube-system -l app=privileged-daemonset,os=windows --no-headers | awk '{print $1}'`
+        for pod in $pods; do
+          kubectl exec -i -n kube-system $pod -- powershell "Restart-service kubeproxy"
+        done
+      name: kubeproxy
+      displayName: Restart Kubeproxy on Windows nodes
+      retryCountOnTaskFailure: 3
+
   - template: ../../templates/log-template.yaml
     parameters:
       clusterName: ${{ parameters.clusterName }}

.pipelines/cni/pipeline.yaml

@@ -142,6 +142,7 @@ stages:
               platforms: linux/amd64 linux/arm64 windows/amd64
             cni:
               name: cni
+              os_versions: ltsc2022
               platforms: linux/amd64 linux/arm64 windows/amd64
             npm:
               name: npm
@@ -154,65 +155,53 @@ stages:
               os_versions: $(os_versions)
               platforms: $(platforms)
 
-  - template: cilium/cilium-overlay-load-test-template.yaml
-    parameters:
-      name: cilium_overlay
-      clusterType: overlay-byocni-nokubeproxy-up
-      clusterName: "cilium-over"
-      nodeCount: ${NODE_COUNT_CILIUM}
-      vmSize: ${VM_SIZE_CILIUM}
-
-  - template: cilium/cilium-overlay-load-test-template.yaml
-    parameters:
-      name: cilium_overlay_hubble
-      clusterType: overlay-byocni-nokubeproxy-up
-      clusterName: "cil-over-hub"
-      hubbleEnabled: true
-      nodeCount: ${NODE_COUNT_CILIUM}
-      vmSize: ${VM_SIZE_CILIUM}
-
-  - template: cilium/cilium-overlay-load-test-template.yaml
+## Windows E2E
+  - template: singletenancy/cniv1-template.yaml
     parameters:
-      name: cilium_overlay_mariner
-      clusterType: overlay-byocni-nokubeproxy-up
-      clusterName: "cil-over-mar"
-      nodeCount: ${NODE_COUNT_CILIUM}
-      vmSize: ${VM_SIZE_CILIUM}
+      name: win22_cniv1
+      clusterType: windows-cniv1-up
+      clusterName: "win22-cniv1"
+      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
+      nodeCountWin: ${NODE_COUNT_WIN}
+      vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
+      vmSizeWin: ${VM_SIZE_WIN}
+      os: windows
+      os_version: 'ltsc2022'
       arch: amd64
-      osSKU: AzureLinux
-
-  - template: cilium/cilium-overlay-load-test-template.yaml
-    parameters:
-      name: cilium_overlay_arm
-      clusterType: overlay-byocni-nokubeproxy-up
-      clusterName: "cil-over-arm"
-      nodeCount: ${NODE_COUNT_CILIUM}
-      vmSize: Standard_D8ps_v5
-      arch: arm64
+      scaleup: ${SCALEUP_WIN}
+      iterations: ${ITERATIONS_WIN}
 
-  - template: cilium/cilium-overlay-load-test-template.yaml
+  - template: singletenancy/cniv2-template.yaml
     parameters:
-      name: cilium_overlay_rdma
-      clusterType: overlay-byocni-nokubeproxy-up
-      clusterName: "cil-over-rdma"
-      nodeCount: 2
-      vmSize: Standard_HC44-16rs
+      name: windows_podsubnet
+      clusterType: swift-byocni-up
+      clusterName: win-podsub
+      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
+      nodeCountWin: ${NODE_COUNT_WIN}
+      vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
+      vmSizeWin: ${VM_SIZE_WIN}
       arch: amd64
+      os: windows
+      os_version: 'ltsc2022'
+      scaleup: ${SCALEUP_WIN}
+      iterations: ${ITERATIONS_WIN}
 
-  - template: singletenancy/cniv1-template.yaml
+  - template: singletenancy/cniv2-template.yaml
     parameters:
-      name: win22_cniv1
-      clusterType: windows-cniv1-up
-      clusterName: "win22-cniv1"
-      nodeCount: ${NODE_COUNT_WIN}
+      name: windows_overlay
+      clusterType: overlay-byocni-up
+      clusterName: win-over
+      nodeCount: ${NODE_COUNT_WINCLUSTER_SYSTEMPOOL}
+      nodeCountWin: ${NODE_COUNT_WIN}
       vmSize: ${VM_SIZE_WINCLUSTER_SYSTEMPOOL}
       vmSizeWin: ${VM_SIZE_WIN}
+      arch: amd64
       os: windows
       os_version: 'ltsc2022'
-      arch: amd64
       scaleup: ${SCALEUP_WIN}
       iterations: ${ITERATIONS_WIN}
 
+## Linux E2E
   - template: singletenancy/cniv1-template.yaml
     parameters:
       name: linux_cniv1
@@ -225,25 +214,29 @@ stages:
       scaleup: ${SCALEUP_LINUX}
       iterations: ${ITERATIONS_LINUX}
 
-  - template: singletenancy/linux-cniv2-template.yaml
+  - template: singletenancy/cniv2-template.yaml
     parameters:
       name: linux_podsubnet
       clusterType: swift-byocni-up
       clusterName: "linux-podsub"
       nodeCount: ${NODE_COUNT_LINUX}
       vmSize: ${VM_SIZE}
       arch: amd64
+      scaleup: ${SCALEUP_LINUX}
+      iterations: ${ITERATIONS_LINUX}
 
-  - template: singletenancy/linux-cniv2-template.yaml
+  - template: singletenancy/cniv2-template.yaml
     parameters:
       name: linux_overlay
       clusterType: overlay-byocni-up
       clusterName: "linux-over"
       nodeCount: ${NODE_COUNT_LINUX}
       vmSize: ${VM_SIZE}
       arch: amd64
+      scaleup: ${SCALEUP_LINUX}
+      iterations: ${ITERATIONS_LINUX}
 
-  - template: singletenancy/linux-cniv2-template.yaml
+  - template: singletenancy/cniv2-template.yaml
     parameters:
       name: mariner_linux_overlay
       clusterType: overlay-byocni-up
@@ -252,24 +245,76 @@ stages:
       vmSize:  ${VM_SIZE}
       arch: amd64
       osSKU: AzureLinux
+      scaleup: ${SCALEUP_LINUX}
+      iterations: ${ITERATIONS_LINUX}
 
-  - template: singletenancy/linux-cniv2-template.yaml
+  - template: singletenancy/cniv2-template.yaml
     parameters:
       name: arm_linux_overlay
       clusterType: overlay-byocni-up
       clusterName: "arm-over"
       nodeCount: ${NODE_COUNT_LINUX}
       vmSize: Standard_D8ps_v5
       arch: arm64
+      scaleup: ${SCALEUP_LINUX}
+      iterations: ${ITERATIONS_LINUX}
 
-  - template: singletenancy/linux-cniv2-template.yaml
+  - template: singletenancy/cniv2-template.yaml
     parameters:
       name: rdma_linux_overlay
       clusterType: overlay-byocni-up
       clusterName: "rdma-over"
       nodeCount: 2
       vmSize: Standard_HC44-16rs
       arch: amd64
+      scaleup: ${SCALEUP_LINUX}
+      iterations: ${ITERATIONS_LINUX}
+
+## Cilium E2E
+  - template: cilium/cilium-overlay-load-test-template.yaml
+    parameters:
+      name: cilium_overlay
+      clusterType: overlay-byocni-nokubeproxy-up
+      clusterName: "cilium-over"
+      nodeCount: ${NODE_COUNT_CILIUM}
+      vmSize: ${VM_SIZE_CILIUM}
+
+  - template: cilium/cilium-overlay-load-test-template.yaml
+    parameters:
+      name: cilium_overlay_hubble
+      clusterType: overlay-byocni-nokubeproxy-up
+      clusterName: "cil-over-hub"
+      hubbleEnabled: true
+      nodeCount: ${NODE_COUNT_CILIUM}
+      vmSize: ${VM_SIZE_CILIUM}
+
+  - template: cilium/cilium-overlay-load-test-template.yaml
+    parameters:
+      name: cilium_overlay_mariner
+      clusterType: overlay-byocni-nokubeproxy-up
+      clusterName: "cil-over-mar"
+      nodeCount: ${NODE_COUNT_CILIUM}
+      vmSize: ${VM_SIZE_CILIUM}
+      arch: amd64
+      osSKU: AzureLinux
+
+  - template: cilium/cilium-overlay-load-test-template.yaml
+    parameters:
+      name: cilium_overlay_arm
+      clusterType: overlay-byocni-nokubeproxy-up
+      clusterName: "cil-over-arm"
+      nodeCount: ${NODE_COUNT_CILIUM}
+      vmSize: Standard_D8ps_v5
+      arch: arm64
+
+  - template: cilium/cilium-overlay-load-test-template.yaml
+    parameters:
+      name: cilium_overlay_rdma
+      clusterType: overlay-byocni-nokubeproxy-up
+      clusterName: "cil-over-rdma"
+      nodeCount: 2
+      vmSize: Standard_HC44-16rs
+      arch: amd64
 
   - stage: delete_resources
     displayName: "Delete Resources"
@@ -289,6 +334,8 @@ stages:
       - mariner_linux_overlay
       - arm_linux_overlay
       - rdma_linux_overlay
+      - windows_podsubnet_HNS
+      - windows_overlay_HNS
       - setup
     variables:
       commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
@@ -335,6 +382,12 @@ stages:
             rdma_linux_overlay:
               name: rdma_linux_overlay
               clusterName: "rdma-over"
+            win-cniv2-podsubnet:
+              name: windows_podsubnet
+              clusterName: win-podsub
+            win-cniv2-overlay:
+              name: windows_overlay
+              clusterName: win-over
         steps:
           - task: AzureCLI@1
             inputs:

.pipelines/cni/singletenancy/cniv1-template.yaml

@@ -4,6 +4,7 @@ parameters:
   clusterType: ""
   clusterName: ""
   nodeCount: ""
+  nodeCountWin: ""
   vmSize: ""
   vmSizeWin: ""
   os: ""
@@ -74,10 +75,12 @@ stages:
                 clusterType: ${{ parameters.clusterType }}
                 clusterName: ${{ parameters.clusterName }}-$(commitID)
                 nodeCount: ${{ parameters.nodeCount }}
+                nodeCountWin: ${{ parameters.nodeCountWin }}
                 vmSize: ${{ parameters.vmSize }}
                 vmSizeWin: ${{ parameters.vmSizeWin }}
                 region: $(LOCATION)
                 osSKU: ${{ parameters.osSKU }}
+                os: ${{ parameters.os }}
 
 # Conditions for below E2E test scenarios confirm that:
 # Pipeline has not been canceled and that the previous job has reports anything other than failure(Succeeded, SuccededWithIssues, Skipped). Previous job is declared by dependsOn:
@@ -131,8 +134,8 @@ stages:
         parameters:
           clusterName: ${{ parameters.clusterName }}-$(commitID)
           os: ${{ parameters.os }}
-          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
           os_version: ${{ parameters.os_version }}
+          sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
           tag: $(npmVersion)
           dependsOn: update_cni
       - job: deploy_pods