test(e2e): refactor e2e test case for test coverage (#156)

* build(deps): bump operator-go to latest for fixing podOverride long text validation

* refactor: remove deprecated e2e configuration and test files

* fix: fix e2e test bug

* feat: add e2e test setup

* refactor(e2e): add logging e2e test case

* test(e2e): add oidc e2e test case

* test(e2e): add smoke tests for SparkHistoryServer cluster operations and pod disruption budget, resources

* ci: add product version matrix and update resource limits

* test(e2e): add sleep duration to chainsaw-test.yaml for log assertion

* test(e2e): add pod logs capture for logging vector

.chainsaw.yaml

@@ -3,13 +3,14 @@ kind: Configuration
 metadata:
   name: custom-config
 spec:
+  # namespace: chainsaw
   timeouts:
     apply: 120s
-    assert: 120s
+    assert: 220s
     cleanup: 120s
     delete: 120s
     error: 120s
-    exec: 45s
+    exec: 150s
   # skipDelete: true
   failFast: true
   parallel: 1

.github/workflows/test.yml

@@ -33,6 +33,8 @@ jobs:
     strategy:
       matrix:
         k8s-version: ['1.26.15', '1.27.16']
+        product_version: ['3.5.1'] # TODO: 3.5.2 image is constructing, should added after it is ready
+      max-parallel: 1
     steps:
       - name: Clone the code
         uses: actions/checkout@v4
@@ -61,4 +63,5 @@ jobs:
           KINDTEST_K8S_VERSION: ${{ matrix.k8s-version }}
           KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }}
           KIND_KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }}
+          PRODUCT_VERSION: ${{ matrix.product_version }}
         run: make chainsaw-test

.gitignore

@@ -145,3 +145,4 @@ Temporary Items
 **/__debug_*
 
 kind-kubeconfig*
+*-local*

Makefile

@@ -261,6 +261,7 @@ endif
 # Tool Versions
 KINDTEST_K8S_VERSION ?= 1.26.15
 CHAINSAW_VERSION ?= v0.2.11
+PRODUCT_VERSION ?= 3.5.1
 
 KIND_IMAGE ?= kindest/node:v${KINDTEST_K8S_VERSION}
 KIND_KUBECONFIG ?= ./kind-kubeconfig-$(KINDTEST_K8S_VERSION)
@@ -314,7 +315,7 @@ chainsaw-setup: ## Run the chainsaw setup
 
 .PHONY: chainsaw-test
 chainsaw-test: chainsaw ## Run the chainsaw test
-	KUBECONFIG=$(KIND_KUBECONFIG) $(CHAINSAW) test --cluster cluster-1=$(KIND_KUBECONFIG) --test-dir ./test/e2e/
+	echo "product_version: $(PRODUCT_VERSION)" | KUBECONFIG=$(KIND_KUBECONFIG) $(CHAINSAW) test --cluster cluster-1=$(KIND_KUBECONFIG) --test-dir ./test/e2e/ --values -
 
 .PHONY: chainsaw-cleanup
 chainsaw-cleanup: ## Run the chainsaw cleanup

api/v1alpha1/sparkhistoryserver_types.go

@@ -165,10 +165,6 @@ type RoleSpec struct {
 	RoleConfig *commonsv1alpha1.RoleConfigSpec `json:"roleConfig,omitempty"`
 }
 
-type ConfigOverridesSpec struct {
-	SparkConfig map[string]string `json:"spark-defaults.conf,omitempty"`
-}
-
 type ConfigSpec struct {
 	*commonsv1alpha1.RoleGroupConfigSpec `json:",inline"`
 

config/crd/bases/spark.kubedoop.dev_sparkhistoryservers.yaml

@@ -183,6 +183,8 @@ spec:
                     type: object
                   vectorAggregatorConfigMapName:
                     type: string
+                required:
+                - logFileDirectory
                 type: object
               clusterOperation:
                 description: ClusterOperationSpec defines the desired state of ClusterOperation
@@ -226,7 +228,6 @@ spec:
                     properties:
                       affinity:
                         type: object
-                        x-kubernetes-embedded-resource: true
                         x-kubernetes-preserve-unknown-fields: true
                       cleaner:
                         type: boolean
@@ -369,7 +370,6 @@ spec:
                     type: object
                   podOverrides:
                     type: object
-                    x-kubernetes-embedded-resource: true
                     x-kubernetes-preserve-unknown-fields: true
                   roleConfig:
                     properties:
@@ -406,7 +406,6 @@ spec:
                           properties:
                             affinity:
                               type: object
-                              x-kubernetes-embedded-resource: true
                               x-kubernetes-preserve-unknown-fields: true
                             cleaner:
                               type: boolean
@@ -549,7 +548,6 @@ spec:
                           type: object
                         podOverrides:
                           type: object
-                          x-kubernetes-embedded-resource: true
                           x-kubernetes-preserve-unknown-fields: true
                         replicas:
                           default: 1
@@ -559,6 +557,7 @@ spec:
                     type: object
                 type: object
             required:
+            - clusterConfig
             - node
             type: object
           status:

go.mod

@@ -1,12 +1,12 @@
 module github.com/zncdatadev/spark-k8s-operator
 
-go 1.23.2
+go 1.23.4
 
 require (
-	github.com/zncdatadev/operator-go v0.12.0
-	k8s.io/api v0.31.3
-	k8s.io/apimachinery v0.31.3
-	k8s.io/client-go v0.31.3
+	github.com/zncdatadev/operator-go v0.12.2-0.20241218094036-d210053e6769
+	k8s.io/api v0.31.4
+	k8s.io/apimachinery v0.31.4
+	k8s.io/client-go v0.31.4
 	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/controller-runtime v0.19.3
 )

go.sum

@@ -96,10 +96,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM=
-github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
-github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4=
-github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
+github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg=
+github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo=
+github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw=
+github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -136,8 +136,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/zncdatadev/operator-go v0.12.0 h1:vsqX2m5l4DM/65lhE4F0V8CSJzGOjrQjAtBod364nbQ=
-github.com/zncdatadev/operator-go v0.12.0/go.mod h1:MYXjJAuUC61RJqmFl1+k8gVDNTx38RvZU2F02IoKTYQ=
+github.com/zncdatadev/operator-go v0.12.2-0.20241218094036-d210053e6769 h1:MLDEfIhaizBq8m1ljxm5o6fjq9CO12Hfk3Z82X6rc5U=
+github.com/zncdatadev/operator-go v0.12.2-0.20241218094036-d210053e6769/go.mod h1:Aj+fVsEFVaDYwXG/bnc7VtmTrZmYjK/PpBBFXf/9rWQ=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
 go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo=
@@ -229,16 +229,16 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8=
-k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE=
+k8s.io/api v0.31.4 h1:I2QNzitPVsPeLQvexMEsj945QumYraqv9m74isPDKhM=
+k8s.io/api v0.31.4/go.mod h1:d+7vgXLvmcdT1BCo79VEgJxHHryww3V5np2OYTr6jdw=
 k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk=
 k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk=
-k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4=
-k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
+k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM=
+k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo=
 k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY=
 k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk=
-k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4=
-k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs=
+k8s.io/client-go v0.31.4 h1:t4QEXt4jgHIkKKlx06+W3+1JOwAFU/2OPiOo7H92eRQ=
+k8s.io/client-go v0.31.4/go.mod h1:kvuMro4sFYIa8sulL5Gi5GFqUPvfH2O/dXuKstbaaeg=
 k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs=
 k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo=
 k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk=

internal/controller/historyserver/cluster.go

@@ -12,6 +12,10 @@ import (
 
 var _ reconciler.Reconciler = &ClusterReconciler{}
 
+const (
+	RoleName = "node"
+)
+
 type ClusterReconciler struct {
 	reconciler.BaseCluster[*shsv1alpha1.SparkHistoryServerSpec]
 	ClusterConfig *shsv1alpha1.ClusterConfigSpec
@@ -55,7 +59,7 @@ func (r *ClusterReconciler) GetImage() *util.Image {
 func (r *ClusterReconciler) RegisterResource(ctx context.Context) error {
 	roleInfo := reconciler.RoleInfo{
 		ClusterInfo: r.ClusterInfo,
-		RoleName:    "node",
+		RoleName:    RoleName,
 	}
 
 	node := NewNodeRoleReconciler(

internal/controller/historyserver/configmap.go

@@ -7,6 +7,7 @@ import (
 	"slices"
 	"strings"
 
+	loggingv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/commons/v1alpha1"
 	"github.com/zncdatadev/operator-go/pkg/builder"
 	"github.com/zncdatadev/operator-go/pkg/client"
 	"github.com/zncdatadev/operator-go/pkg/productlogging"
@@ -24,10 +25,6 @@ type ConfigMapBuilder struct {
 
 	ClusteerConfig  *sparkv1alpha1.ClusterConfigSpec
 	RoleGroupConfig *sparkv1alpha1.ConfigSpec
-
-	ClusterName   string
-	RoleName      string
-	RoleGroupName string
 }
 
 func NewSparkConfigMapBuilder(
@@ -76,7 +73,7 @@ func (b *ConfigMapBuilder) Build(ctx context.Context) (ctrlclient.Object, error)
 	if vectorConfig, err := b.getVectorConfig(ctx); err != nil {
 		return nil, err
 	} else if vectorConfig != "" {
-		b.AddItem(builder.VectorConfigFile, vectorConfig)
+		b.AddItem(builder.VectorConfigFileName, vectorConfig)
 	}
 
 	return b.GetObject(), nil
@@ -141,32 +138,30 @@ func (b *ConfigMapBuilder) isCleaner() (bool, error) {
 }
 
 func (b *ConfigMapBuilder) getLog4j() (string, error) {
+	var loggingConfig loggingv1alpha1.LoggingConfigSpec
 	if b.RoleGroupConfig != nil && b.RoleGroupConfig.RoleGroupConfigSpec != nil && b.RoleGroupConfig.Logging != nil && len(b.RoleGroupConfig.Logging.Containers) > 0 {
-
-		loggingConfig, ok := b.RoleGroupConfig.Logging.Containers[SparkHistoryContainerName]
+		var ok bool
+		loggingConfig, ok = b.RoleGroupConfig.Logging.Containers[SparkHistoryContainerName]
 		if !ok {
 			return "", nil
 		}
+	}
 
-		logGenerator, err := productlogging.NewConfigGenerator(
-			&loggingConfig,
-			SparkHistoryContainerName,
-			"spark.log4j2.xml",
-			productlogging.LogTypeLog4j2,
-			func(cgo *productlogging.ConfigGeneratorOption) {
-				cgo.ConsoleHandlerFormatter = ptr.To("%d{ISO8601} %p [%t] %c - %m%n")
-			},
-		)
-
-		if err != nil {
-			return "", err
-		}
+	logGenerator, err := productlogging.NewConfigGenerator(
+		&loggingConfig,
+		SparkHistoryContainerName,
+		"spark.log4j2.xml",
+		productlogging.LogTypeLog4j2,
+		func(cgo *productlogging.ConfigGeneratorOption) {
+			cgo.ConsoleHandlerFormatter = ptr.To("%d{ISO8601} %p [%t] %c - %m%n")
+		},
+	)
 
-		return logGenerator.Content()
+	if err != nil {
+		return "", err
 	}
 
-	return "", nil
-
+	return logGenerator.Content()
 }
 
 func (b *ConfigMapBuilder) getSparkDefaules(s3Logconfig *S3Logconfig) string {