test: add scale-test to workflow

Signed-off-by: Alex Castilio dos Santos <[email protected]>

.github/workflows/scale-test-v2.yaml

@@ -68,6 +68,14 @@ permissions:
   contents: read
   id-token: write
 
+env:
+  RESOURCE_GROUP: ${{ inputs.resource_group }}
+  CLUSTER_NAME: ${{ inputs.cluster_name }}
+  TAG: RetinaVersion
+  VM_SIZE: Standard_D4_v3
+  LOCATION: westus2
+  OUTPUT_FILEPATH: ./output.log
+
 jobs:
   setup-cluster:
     if: ${{ github.event.inputs.create_cluster == 'true' }}
@@ -135,14 +143,6 @@ jobs:
         shell: bash
         run: az aks get-credentials --name $CLUSTER_NAME --resource-group $RESOURCE_GROUP --overwrite-existing
 
-      - name: Run Scale Test
-        shell: bash
-        run: |
-          set -euo pipefail
-          # Placeholder for test
-          sleep 300 &
-          echo "TEST_PID=$!" >> $GITHUB_ENV
-
       - name: Clone ClusterLoader2
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -163,18 +163,29 @@ jobs:
           mkdir test
           cp ../../.github/actions/cl2/* ./test -r
           go build ./cmd/clusterloader.go
-          ls -l
 
-      - name: Run CL2
+      - name: Run Scale Test
         shell: bash
+        env:
+          AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION }}
+          AZURE_APP_INSIGHTS_KEY: ${{ secrets.AZURE_APP_INSIGHTS_KEY }}
         run: |
           set -euo pipefail
-          cd perf-tests/clusterloader2
-          ./clusterloader --testconfig=./test/config.yaml --provider=aks --kubeconfig=$HOME/.kube/config --v=2 --report-dir=./report
-
-      - name: Stop test
-        shell: bash
-        run: kill $TEST_PID
+          # Placeholder for test
+          go test ./test/e2e/. -v -tags=scale -timeout 300s -args -image-tag=$(make version) -create-infra=false -delete-infra=false
+
+      # - name: Run CL2
+      #   shell: bash
+      #   run: |
+      #     set -euo pipefail
+      #     cd perf-tests/clusterloader2
+      #     ./clusterloader --testconfig=./test/config.yaml --provider=aks --kubeconfig=$HOME/.kube/config --v=2 --report-dir=./report
+
+      # - name: Stop test
+      #   shell: bash
+      #   run: |
+      #     PID=$(ps aux | grep "go test" | awk '{print $2}')
+      #     kill -s 15 $PID
 
   cleanup:
     name: Cleanup

test/e2e/framework/scaletest/clusterloader2.go

@@ -0,0 +1,34 @@
+package scaletest
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+)
+
+type ClusterLoader2 struct {
+	Args []string
+}
+
+func (d *ClusterLoader2) Prevalidate() error {
+	return nil
+}
+
+func (d *ClusterLoader2) Run() error {
+	cl2Path := "../../perf-tests/clusterloader2/clusterloader"
+	cmd := exec.Command(cl2Path, d.Args...)
+
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("Error executing CL2: %w", err)
+	}
+
+	return nil
+}
+
+func (d *ClusterLoader2) Stop() error {
+	return nil
+}

test/e2e/framework/scaletest/get-publish-metrics.go

@@ -44,6 +44,7 @@ func (g *GetAndPublishMetrics) Run() error {
 	}
 
 	g.stop = make(chan struct{})
+
 	g.wg.Add(1)
 
 	go func() {
@@ -66,7 +67,6 @@ func (g *GetAndPublishMetrics) Run() error {
 
 			}
 		}
-
 	}()
 
 	return nil

test/e2e/framework/scaletest/options.go

@@ -1,40 +1,8 @@
 package scaletest
 
-import "time"
-
 // Options holds parameters for the scale test
 type Options struct {
-	Namespace                     string
-	MaxKwokPodsPerNode            int
-	NumKwokDeployments            int
-	NumKwokReplicas               int
-	MaxRealPodsPerNode            int
-	NumRealDeployments            int
-	RealPodType                   string
-	NumRealReplicas               int
-	NumRealServices               int
-	NumNetworkPolicies            int
-	NumUnapliedNetworkPolicies    int
-	NumUniqueLabelsPerPod         int
-	NumUniqueLabelsPerDeployment  int
-	NumSharedLabelsPerPod         int
-	KubeconfigPath                string
-	RestartNpmPods                bool
-	DebugExitAfterPrintCounts     bool
-	DebugExitAfterGeneration      bool
-	SleepAfterCreation            time.Duration
-	DeleteKwokPods                bool
-	DeleteRealPods                bool
-	DeletePodsInterval            time.Duration
-	DeletePodsTimes               int
-	DeleteLabels                  bool
-	DeleteLabelsInterval          time.Duration
-	DeleteLabelsTimes             int
-	DeleteNetworkPolicies         bool
-	DeleteNetworkPoliciesInterval time.Duration
-	DeleteNetworkPoliciesTimes    int
-	numKwokPods                   int
-	numRealPods                   int
-	LabelsToGetMetrics            map[string]string
-	AdditionalTelemetryProperty   map[string]string
+	KubeconfigPath              string
+	LabelsToGetMetrics          map[string]string
+	AdditionalTelemetryProperty map[string]string
 }

test/e2e/jobs/scale.go

@@ -2,116 +2,44 @@ package retina
 
 import (
 	"os"
-	"time"
 
-	"github.com/microsoft/retina/test/e2e/framework/kubernetes"
 	"github.com/microsoft/retina/test/e2e/framework/scaletest"
 	"github.com/microsoft/retina/test/e2e/framework/types"
 )
 
 func DefaultScaleTestOptions() scaletest.Options {
 	return scaletest.Options{
-		Namespace:                     "scale-test",
-		MaxKwokPodsPerNode:            0,
-		NumKwokDeployments:            0,
-		NumKwokReplicas:               0,
-		MaxRealPodsPerNode:            100,
-		NumRealDeployments:            1000,
-		RealPodType:                   "kapinger",
-		NumRealReplicas:               40,
-		NumRealServices:               1000,
-		NumNetworkPolicies:            10,
-		NumUnapliedNetworkPolicies:    10,
-		NumUniqueLabelsPerPod:         0,
-		NumUniqueLabelsPerDeployment:  1,
-		NumSharedLabelsPerPod:         3,
-		KubeconfigPath:                "",
-		RestartNpmPods:                false,
-		SleepAfterCreation:            0,
-		DeleteKwokPods:                false,
-		DeletePodsInterval:            60 * time.Second,
-		DeleteRealPods:                false,
-		DeletePodsTimes:               1,
-		DeleteLabels:                  false,
-		DeleteLabelsInterval:          60 * time.Second,
-		DeleteLabelsTimes:             1,
-		DeleteNetworkPolicies:         false,
-		DeleteNetworkPoliciesInterval: 60 * time.Second,
-		DeleteNetworkPoliciesTimes:    1,
-		LabelsToGetMetrics:            map[string]string{},
-		AdditionalTelemetryProperty:   map[string]string{},
+		LabelsToGetMetrics:          map[string]string{},
+		AdditionalTelemetryProperty: map[string]string{},
 	}
 }
 
 func ScaleTest(opt *scaletest.Options) *types.Job {
 	job := types.NewJob("Scale Test")
 
-	job.AddStep(&scaletest.ValidateAndPrintOptions{
-		Options: opt,
-	}, nil)
-
-	job.AddStep(&scaletest.ValidateNumOfNodes{
-		KubeConfigFilePath: opt.KubeconfigPath,
-		Label:              map[string]string{"scale-test": "true"},
-		NumNodesRequired: (opt.NumRealDeployments*opt.NumRealReplicas +
-			opt.MaxRealPodsPerNode - 1) / opt.MaxRealPodsPerNode,
-	}, nil)
-
-	job.AddStep(&kubernetes.DeleteNamespace{
-		Namespace: opt.Namespace,
-	}, nil)
-
-	job.AddStep(&kubernetes.CreateNamespace{}, nil)
-
 	job.AddStep(&scaletest.GetAndPublishMetrics{
+		KubeConfigFilePath:          opt.KubeconfigPath,
 		Labels:                      opt.LabelsToGetMetrics,
 		AdditionalTelemetryProperty: opt.AdditionalTelemetryProperty,
-		OutputFilePath:                  os.Getenv("OUTPUT_FILEPATH"),
+		OutputFilePath:              os.Getenv("OUTPUT_FILEPATH"),
 	}, &types.StepOptions{
 		SkipSavingParametersToJob: true,
-		RunInBackgroundWithID:     "get-metrics",
+		RunInBackgroundWithID:     "metrics",
 	})
 
-	job.AddStep(&scaletest.CreateResources{
-		NumKwokDeployments:           opt.NumKwokDeployments,
-		NumKwokReplicas:              opt.NumKwokReplicas,
-		RealPodType:                  opt.RealPodType,
-		NumRealDeployments:           opt.NumRealDeployments,
-		NumRealReplicas:              opt.NumRealReplicas,
-		NumRealServices:              opt.NumRealServices,
-		NumUniqueLabelsPerDeployment: opt.NumUniqueLabelsPerDeployment,
-	}, nil)
-
-	job.AddStep(&scaletest.AddSharedLabelsToAllPods{
-		NumSharedLabelsPerPod: opt.NumSharedLabelsPerPod,
-	}, nil)
-
-	job.AddStep(&scaletest.AddUniqueLabelsToAllPods{
-		NumUniqueLabelsPerPod: opt.NumUniqueLabelsPerPod,
-	}, nil)
-
-	// Apply network policies (applied and unapplied)
-	job.AddStep(&scaletest.CreateNetworkPolicies{
-		NumNetworkPolicies:    opt.NumNetworkPolicies,
-		NumSharedLabelsPerPod: opt.NumSharedLabelsPerPod,
-	}, nil)
-
-	job.AddStep(&kubernetes.WaitPodsReady{
-		LabelSelector: "is-real=true",
-	}, nil)
-
-	job.AddStep(&scaletest.DeleteAndReAddLabels{
-		DeleteLabels:          opt.DeleteLabels,
-		DeleteLabelsInterval:  opt.DeleteLabelsInterval,
-		DeleteLabelsTimes:     opt.DeleteLabelsTimes,
-		NumSharedLabelsPerPod: opt.NumSharedLabelsPerPod,
+	job.AddStep(&scaletest.ClusterLoader2{
+		Args: []string{
+			"--testconfig=../../perf-tests/clusterloader2/test/config.yaml",
+			"--provider=aks",
+			"--kubeconfig=/home/runner/.kube/config",
+			"--v=2",
+			"--report-dir=../../perf-tests/clusterloader2/report",
+		},
 	}, nil)
 
 	job.AddStep(&types.Stop{
-		BackgroundID: "get-metrics",
+		BackgroundID: "metrics",
 	}, nil)
 
-	job.AddStep(&kubernetes.DeleteNamespace{}, nil)
-
 	return job
 }