diff --git a/.chainsaw.yaml b/.chainsaw.yaml index cce0d36..7b33cf5 100644 --- a/.chainsaw.yaml +++ b/.chainsaw.yaml @@ -3,13 +3,14 @@ kind: Configuration metadata: name: custom-config spec: + # namespace: chainsaw timeouts: apply: 120s - assert: 120s + assert: 220s cleanup: 120s delete: 120s error: 120s - exec: 45s + exec: 150s # skipDelete: true failFast: true parallel: 1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index aef8e9b..0862356 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,8 @@ jobs: strategy: matrix: k8s-version: ['1.26.15', '1.27.16'] + product_version: ['3.5.1'] # TODO: 3.5.2 image is constructing, should added after it is ready + max-parallel: 1 steps: - name: Clone the code uses: actions/checkout@v4 @@ -61,4 +63,5 @@ jobs: KINDTEST_K8S_VERSION: ${{ matrix.k8s-version }} KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} KIND_KUBECONFIG: kind-kubeconfig-${{ matrix.k8s-version }} + PRODUCT_VERSION: ${{ matrix.product_version }} run: make chainsaw-test diff --git a/.gitignore b/.gitignore index 5c9bb50..cf1a683 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,4 @@ Temporary Items **/__debug_* kind-kubeconfig* +*-local* diff --git a/Makefile b/Makefile index f28a806..cc8e625 100644 --- a/Makefile +++ b/Makefile @@ -261,6 +261,7 @@ endif # Tool Versions KINDTEST_K8S_VERSION ?= 1.26.15 CHAINSAW_VERSION ?= v0.2.11 +PRODUCT_VERSION ?= 3.5.1 KIND_IMAGE ?= kindest/node:v${KINDTEST_K8S_VERSION} KIND_KUBECONFIG ?= ./kind-kubeconfig-$(KINDTEST_K8S_VERSION) @@ -314,7 +315,7 @@ chainsaw-setup: ## Run the chainsaw setup .PHONY: chainsaw-test chainsaw-test: chainsaw ## Run the chainsaw test - KUBECONFIG=$(KIND_KUBECONFIG) $(CHAINSAW) test --cluster cluster-1=$(KIND_KUBECONFIG) --test-dir ./test/e2e/ + echo "product_version: $(PRODUCT_VERSION)" | KUBECONFIG=$(KIND_KUBECONFIG) $(CHAINSAW) test --cluster cluster-1=$(KIND_KUBECONFIG) --test-dir ./test/e2e/ --values - .PHONY: chainsaw-cleanup chainsaw-cleanup: ## Run the chainsaw cleanup diff --git a/api/v1alpha1/sparkhistoryserver_types.go b/api/v1alpha1/sparkhistoryserver_types.go index c529fbf..6864cec 100644 --- a/api/v1alpha1/sparkhistoryserver_types.go +++ b/api/v1alpha1/sparkhistoryserver_types.go @@ -165,10 +165,6 @@ type RoleSpec struct { RoleConfig *commonsv1alpha1.RoleConfigSpec `json:"roleConfig,omitempty"` } -type ConfigOverridesSpec struct { - SparkConfig map[string]string `json:"spark-defaults.conf,omitempty"` -} - type ConfigSpec struct { *commonsv1alpha1.RoleGroupConfigSpec `json:",inline"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 0dde389..ced7c8a 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -91,28 +91,6 @@ func (in *ClusterConfigSpec) DeepCopy() *ClusterConfigSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ConfigOverridesSpec) DeepCopyInto(out *ConfigOverridesSpec) { - *out = *in - if in.SparkConfig != nil { - in, out := &in.SparkConfig, &out.SparkConfig - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConfigOverridesSpec. -func (in *ConfigOverridesSpec) DeepCopy() *ConfigOverridesSpec { - if in == nil { - return nil - } - out := new(ConfigOverridesSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConfigSpec) DeepCopyInto(out *ConfigSpec) { *out = *in @@ -216,6 +194,11 @@ func (in *RoleGroupSpec) DeepCopyInto(out *RoleGroupSpec) { *out = new(commonsv1alpha1.OverridesSpec) (*in).DeepCopyInto(*out) } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } if in.Config != nil { in, out := &in.Config, &out.Config *out = new(ConfigSpec) diff --git a/config/crd/bases/spark.kubedoop.dev_sparkhistoryservers.yaml b/config/crd/bases/spark.kubedoop.dev_sparkhistoryservers.yaml index 28f56dc..9e34e3b 100644 --- a/config/crd/bases/spark.kubedoop.dev_sparkhistoryservers.yaml +++ b/config/crd/bases/spark.kubedoop.dev_sparkhistoryservers.yaml @@ -183,6 +183,8 @@ spec: type: object vectorAggregatorConfigMapName: type: string + required: + - logFileDirectory type: object clusterOperation: description: ClusterOperationSpec defines the desired state of ClusterOperation @@ -226,7 +228,6 @@ spec: properties: affinity: type: object - x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true cleaner: type: boolean @@ -369,7 +370,6 @@ spec: type: object podOverrides: type: object - x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true roleConfig: properties: @@ -406,7 +406,6 @@ spec: properties: affinity: type: object - x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true cleaner: type: boolean @@ -549,7 +548,6 @@ spec: type: object podOverrides: type: object - x-kubernetes-embedded-resource: true x-kubernetes-preserve-unknown-fields: true replicas: default: 1 @@ -559,6 +557,7 @@ spec: type: object type: object required: + - clusterConfig - node type: object status: diff --git a/go.mod b/go.mod index 5a0f031..2ae904f 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,12 @@ module github.com/zncdatadev/spark-k8s-operator -go 1.23.2 +go 1.23.4 require ( - github.com/zncdatadev/operator-go v0.12.0 - k8s.io/api v0.31.3 - k8s.io/apimachinery v0.31.3 - k8s.io/client-go v0.31.3 + github.com/zncdatadev/operator-go v0.12.2-0.20241218094036-d210053e6769 + k8s.io/api v0.31.4 + k8s.io/apimachinery v0.31.4 + k8s.io/client-go v0.31.4 k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 sigs.k8s.io/controller-runtime v0.19.3 ) diff --git a/go.sum b/go.sum index 5a9ae9d..8160816 100644 --- a/go.sum +++ b/go.sum @@ -96,10 +96,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/ginkgo/v2 v2.22.0 h1:Yed107/8DjTr0lKCNt7Dn8yQ6ybuDRQoMGrNFKzMfHg= +github.com/onsi/ginkgo/v2 v2.22.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= +github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= +github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -136,8 +136,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/zncdatadev/operator-go v0.12.0 h1:vsqX2m5l4DM/65lhE4F0V8CSJzGOjrQjAtBod364nbQ= -github.com/zncdatadev/operator-go v0.12.0/go.mod h1:MYXjJAuUC61RJqmFl1+k8gVDNTx38RvZU2F02IoKTYQ= +github.com/zncdatadev/operator-go v0.12.2-0.20241218094036-d210053e6769 h1:MLDEfIhaizBq8m1ljxm5o6fjq9CO12Hfk3Z82X6rc5U= +github.com/zncdatadev/operator-go v0.12.2-0.20241218094036-d210053e6769/go.mod h1:Aj+fVsEFVaDYwXG/bnc7VtmTrZmYjK/PpBBFXf/9rWQ= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= @@ -229,16 +229,16 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= -k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= +k8s.io/api v0.31.4 h1:I2QNzitPVsPeLQvexMEsj945QumYraqv9m74isPDKhM= +k8s.io/api v0.31.4/go.mod h1:d+7vgXLvmcdT1BCo79VEgJxHHryww3V5np2OYTr6jdw= k8s.io/apiextensions-apiserver v0.31.0 h1:fZgCVhGwsclj3qCw1buVXCV6khjRzKC5eCFt24kyLSk= k8s.io/apiextensions-apiserver v0.31.0/go.mod h1:b9aMDEYaEe5sdK+1T0KU78ApR/5ZVp4i56VacZYEHxk= -k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= -k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM= +k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= k8s.io/apiserver v0.31.0 h1:p+2dgJjy+bk+B1Csz+mc2wl5gHwvNkC9QJV+w55LVrY= k8s.io/apiserver v0.31.0/go.mod h1:KI9ox5Yu902iBnnyMmy7ajonhKnkeZYJhTZ/YI+WEMk= -k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= -k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= +k8s.io/client-go v0.31.4 h1:t4QEXt4jgHIkKKlx06+W3+1JOwAFU/2OPiOo7H92eRQ= +k8s.io/client-go v0.31.4/go.mod h1:kvuMro4sFYIa8sulL5Gi5GFqUPvfH2O/dXuKstbaaeg= k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs= k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= diff --git a/internal/controller/historyserver/cluster.go b/internal/controller/historyserver/cluster.go index 9acf17f..1c7a8bc 100644 --- a/internal/controller/historyserver/cluster.go +++ b/internal/controller/historyserver/cluster.go @@ -12,6 +12,10 @@ import ( var _ reconciler.Reconciler = &ClusterReconciler{} +const ( + RoleName = "node" +) + type ClusterReconciler struct { reconciler.BaseCluster[*shsv1alpha1.SparkHistoryServerSpec] ClusterConfig *shsv1alpha1.ClusterConfigSpec @@ -55,7 +59,7 @@ func (r *ClusterReconciler) GetImage() *util.Image { func (r *ClusterReconciler) RegisterResource(ctx context.Context) error { roleInfo := reconciler.RoleInfo{ ClusterInfo: r.ClusterInfo, - RoleName: "node", + RoleName: RoleName, } node := NewNodeRoleReconciler( diff --git a/internal/controller/historyserver/configmap.go b/internal/controller/historyserver/configmap.go index e25d892..38447bd 100644 --- a/internal/controller/historyserver/configmap.go +++ b/internal/controller/historyserver/configmap.go @@ -7,6 +7,7 @@ import ( "slices" "strings" + loggingv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/commons/v1alpha1" "github.com/zncdatadev/operator-go/pkg/builder" "github.com/zncdatadev/operator-go/pkg/client" "github.com/zncdatadev/operator-go/pkg/productlogging" @@ -24,10 +25,6 @@ type ConfigMapBuilder struct { ClusteerConfig *sparkv1alpha1.ClusterConfigSpec RoleGroupConfig *sparkv1alpha1.ConfigSpec - - ClusterName string - RoleName string - RoleGroupName string } func NewSparkConfigMapBuilder( @@ -76,7 +73,7 @@ func (b *ConfigMapBuilder) Build(ctx context.Context) (ctrlclient.Object, error) if vectorConfig, err := b.getVectorConfig(ctx); err != nil { return nil, err } else if vectorConfig != "" { - b.AddItem(builder.VectorConfigFile, vectorConfig) + b.AddItem(builder.VectorConfigFileName, vectorConfig) } return b.GetObject(), nil @@ -141,32 +138,30 @@ func (b *ConfigMapBuilder) isCleaner() (bool, error) { } func (b *ConfigMapBuilder) getLog4j() (string, error) { + var loggingConfig loggingv1alpha1.LoggingConfigSpec if b.RoleGroupConfig != nil && b.RoleGroupConfig.RoleGroupConfigSpec != nil && b.RoleGroupConfig.Logging != nil && len(b.RoleGroupConfig.Logging.Containers) > 0 { - - loggingConfig, ok := b.RoleGroupConfig.Logging.Containers[SparkHistoryContainerName] + var ok bool + loggingConfig, ok = b.RoleGroupConfig.Logging.Containers[SparkHistoryContainerName] if !ok { return "", nil } + } - logGenerator, err := productlogging.NewConfigGenerator( - &loggingConfig, - SparkHistoryContainerName, - "spark.log4j2.xml", - productlogging.LogTypeLog4j2, - func(cgo *productlogging.ConfigGeneratorOption) { - cgo.ConsoleHandlerFormatter = ptr.To("%d{ISO8601} %p [%t] %c - %m%n") - }, - ) - - if err != nil { - return "", err - } + logGenerator, err := productlogging.NewConfigGenerator( + &loggingConfig, + SparkHistoryContainerName, + "spark.log4j2.xml", + productlogging.LogTypeLog4j2, + func(cgo *productlogging.ConfigGeneratorOption) { + cgo.ConsoleHandlerFormatter = ptr.To("%d{ISO8601} %p [%t] %c - %m%n") + }, + ) - return logGenerator.Content() + if err != nil { + return "", err } - return "", nil - + return logGenerator.Content() } func (b *ConfigMapBuilder) getSparkDefaules(s3Logconfig *S3Logconfig) string { diff --git a/internal/controller/historyserver/deployment.go b/internal/controller/historyserver/deployment.go index d4c4334..0efa45d 100644 --- a/internal/controller/historyserver/deployment.go +++ b/internal/controller/historyserver/deployment.go @@ -15,10 +15,12 @@ import ( "github.com/zncdatadev/operator-go/pkg/builder" resourceClient "github.com/zncdatadev/operator-go/pkg/client" "github.com/zncdatadev/operator-go/pkg/constants" + "github.com/zncdatadev/operator-go/pkg/productlogging" "github.com/zncdatadev/operator-go/pkg/reconciler" "github.com/zncdatadev/operator-go/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/intstr" ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" shsv1alpha1 "github.com/zncdatadev/spark-k8s-operator/api/v1alpha1" @@ -26,11 +28,12 @@ import ( const ( SparkConfigDefauleFileName = "spark-defaults.conf" - SparkHistoryContainerName = "spark-history" - - LogVolumeName = "log" + SparkHistoryContainerName = RoleName + LogVolumeName = builder.LogDataVolumeName ConfigVolumeName = "config" + + MaxLogFileSize = "10Mi" ) var _ builder.DeploymentBuilder = &DeploymentBuilder{} @@ -92,9 +95,6 @@ func (b *DeploymentBuilder) getMainContainerCmdArgs(s3LogConfig *S3Logconfig) st } args := ` -# TODO: remove this when refactor spark dockerfile -# fix jar conflict in container image -rm -rf jars/slf4j-api-1.7.36.jar mkdir -p ` + constants.KubedoopConfigDir + ` cp ` + path.Join(constants.KubedoopConfigDirMount, `*`) + " " + constants.KubedoopConfigDir + ` @@ -116,6 +116,10 @@ func (b *DeploymentBuilder) getMainContainerEnvVars() []corev1.EnvVar { Name: "SPARK_NO_DAEMONIZE", Value: "true", }, + { + Name: "SPARK_DAEMON_CLASSPATH", + Value: "/kubedoop/spark/extra-jars/*", + }, { Name: "SPARK_HISTORY_OPTS", Value: strings.Join(jvmOpts, " "), @@ -132,6 +136,21 @@ func (b *DeploymentBuilder) getMainContainer(s3LogConfig *S3Logconfig) *builder. containerBuilder.AddPorts(b.Ports) containerBuilder.AddEnvVars(b.getMainContainerEnvVars()) containerBuilder.SetSecurityContext(0, 0, false) + + probe := &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + TCPSocket: &corev1.TCPSocketAction{ + Port: intstr.FromInt(18080), + }, + }, + InitialDelaySeconds: 10, + TimeoutSeconds: 5, + PeriodSeconds: 10, + SuccessThreshold: 1, + } + containerBuilder.SetReadinessProbe(probe) + containerBuilder.SetLivenessProbe(probe) + return containerBuilder } @@ -169,6 +188,29 @@ func (b *DeploymentBuilder) addS3CrenditialVolume(containerBuilder *builder.Cont containerBuilder.AddVolumeMount(volumeMount) } +// add log volume to container +func (b *DeploymentBuilder) addLogVolume(containerBuilder *builder.Container) { + volume := &corev1.Volume{ + Name: LogVolumeName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{ + SizeLimit: func() *resource.Quantity { + q := resource.MustParse(MaxLogFileSize) + size := productlogging.CalculateLogVolumeSizeLimit([]resource.Quantity{q}) + return &size + }(), + }, + }, + } + b.AddVolume(volume) + + volumeMount := &corev1.VolumeMount{ + Name: LogVolumeName, + MountPath: constants.KubedoopLogDir, + } + containerBuilder.AddVolumeMount(volumeMount) +} + func (b *DeploymentBuilder) getOidcContainer(ctx context.Context) (*corev1.Container, error) { authClass := &authv1alpha1.AuthenticationClass{} if err := b.Client.GetWithOwnerNamespace(ctx, b.ClusteerConfig.Authentication.AuthenticationClass, authClass); err != nil { @@ -272,8 +314,12 @@ func (b *DeploymentBuilder) getOidcContainer(ctx context.Context) (*corev1.Conta Value: "0.0.0.0:4180", }, { - Name: "OAUTH2_PROXY_REDIRECT_URL", - Value: "http://localhost:4180/oauth2/callback", + Name: "OAUTH2_PROXY_COOKIE_SECURE", // https://github.com/oauth2-proxy/oauth2-proxy/blob/c64ec1251b8366b48c6c445bbeb307b18fcb314f/oauthproxy.go#L1091 + Value: "false", + }, + { + Name: "OAUTH2_PROXY_WHITELIST_DOMAINS", + Value: "*", }, { Name: "OAUTH2_PROXY_CODE_CHALLENGE_METHOD", @@ -286,7 +332,7 @@ func (b *DeploymentBuilder) getOidcContainer(ctx context.Context) (*corev1.Conta }, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1000m"), + corev1.ResourceCPU: resource.MustParse("600m"), corev1.ResourceMemory: resource.MustParse("512Mi"), }, }, @@ -304,6 +350,7 @@ func (b *DeploymentBuilder) Build(ctx context.Context) (ctrlclient.Object, error mainContainer := b.getMainContainer(s3LogConfig) b.addS3CrenditialVolume(mainContainer, s3LogConfig) + b.addLogVolume(mainContainer) b.addSparkDefaultConfigVolume(mainContainer) b.AddContainer(mainContainer.Build()) @@ -316,21 +363,21 @@ func (b *DeploymentBuilder) Build(ctx context.Context) (ctrlclient.Object, error b.AddContainer(oidcContainer) } - obj, err := b.GetObject() - if err != nil { - return nil, err - } - if b.ClusteerConfig != nil && b.ClusteerConfig.VectorAggregatorConfigMapName != "" { - builder.NewVectorDecorator( - obj, - b.GetImage(), - LogVolumeName, + vectorBuilder := builder.NewVector( ConfigVolumeName, - b.ClusteerConfig.VectorAggregatorConfigMapName, + LogVolumeName, + b.GetImage(), ) + + b.AddContainer(vectorBuilder.GetContainer()) + b.AddVolumes(vectorBuilder.GetVolumes()) } + obj, err := b.GetObject() + if err != nil { + return nil, err + } return obj, nil } diff --git a/internal/controller/historyserver/node.go b/internal/controller/historyserver/node.go index 91296b6..04ac7a2 100644 --- a/internal/controller/historyserver/node.go +++ b/internal/controller/historyserver/node.go @@ -6,6 +6,7 @@ import ( commonsv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/commons/v1alpha1" "github.com/zncdatadev/operator-go/pkg/builder" resourceClient "github.com/zncdatadev/operator-go/pkg/client" + "github.com/zncdatadev/operator-go/pkg/constants" "github.com/zncdatadev/operator-go/pkg/reconciler" "github.com/zncdatadev/operator-go/pkg/util" corev1 "k8s.io/api/core/v1" @@ -136,6 +137,14 @@ func (r *NodeRoleReconciler) GetImageResourceWithRoleGroup( r.Client, info.GetFullName(), append(SparkHistoryPorts, OidcPorts...), + func(o *builder.ServiceBuilderOptions) { + o.ListenerClass = constants.ListenerClass(r.ClusterConfig.ListenerClass) + o.ClusterName = info.GetClusterName() + o.RoleName = info.GetRoleName() + o.RoleGroupName = info.GetGroupName() + o.Labels = info.GetLabels() + o.Annotations = info.GetAnnotations() + }, ) return []reconciler.Reconciler{cm, deployment, svc}, nil } diff --git a/internal/controller/historyserver/s3.go b/internal/controller/historyserver/s3.go index 5f033c1..d7ab832 100644 --- a/internal/controller/historyserver/s3.go +++ b/internal/controller/historyserver/s3.go @@ -114,7 +114,7 @@ func NewS3Logconfig( } func (s *S3Logconfig) GetMountPath() string { - return path.Join(constants.KubedoopSecretDir, "s3-credentials") + return path.Join(constants.KubedoopSecretDir, S3VolumeName) } func (s *S3Logconfig) GetVolumeName() string { diff --git a/test/e2e/default/chainsaw-test.yaml b/test/e2e/default/chainsaw-test.yaml deleted file mode 100644 index 2f2a2c3..0000000 --- a/test/e2e/default/chainsaw-test.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: chainsaw.kyverno.io/v1alpha1 -kind: Test -metadata: - name: default -spec: - bindings: - - name: MINIO_USER - value: miniouser - - name: MINIO_PASSWORD - value: miniouserpassword - - name: MINIO_BUCKET - value: spark-history - steps: - - try: - - apply: - file: ../setup/minio.yaml - - assert: - file: ../setup/minio-assert.yaml - - try: - - apply: - file: ../setup/spark-s3-credentials.yaml - - assert: - file: ../setup/spark-s3-credentials-assert.yaml - - apply: - file: ../setup/s3bucket.yaml - - assert: - file: ../setup/s3bucket.yaml - cleanup: - - try: - - apply: - file: sparkhistoryserver.yaml - - assert: - file: sparkhistoryserver-assert.yaml - cleanup: - - sleep: - duration: 5s diff --git a/test/e2e/logging/chainsaw-test.yaml b/test/e2e/logging/chainsaw-test.yaml new file mode 100644 index 0000000..413403b --- /dev/null +++ b/test/e2e/logging/chainsaw-test.yaml @@ -0,0 +1,91 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: logging +spec: + bindings: + - name: minio_user + value: spark + - name: minio_password + value: sparkspark + - name: minio_bucket + value: spark-history + steps: + - name: install minio + try: + - script: + env: + - name: NAMESPACE + value: ($namespace) + - name: MINIO_USER + value: ($minio_user) + - name: MINIO_PASSWORD + value: ($minio_password) + content: | + envsubst < ../setup/minio-secret.yaml | kubectl apply -f - + # - apply: + # file: ../setup/minio-secret.yaml + - script: + content: | + bash ../setup/minio-install.sh + # - assert: + # file: minio-assert.yaml + - name: install vector-aggregator + try: + - script: + content: >- + helm upgrade --install vector-aggregator vector + --namespace $NAMESPACE + --version 0.36.1 + --repo https://helm.vector.dev + --values vector-aggregator-values.yaml + - apply: + file: vector-aggregator.yaml + - assert: + file: vector-aggregator-assert.yaml + - name: install sparkhistoryserver cluster + try: + - apply: + file: ../setup/minio-s3-connection.yaml + - apply: + file: sparkhistoryserver.yaml + - assert: + file: sparkhistoryserver-assert.yaml + - name: assert sparkhistoryserver logs + try: + - sleep: + duration: 50s + - script: + env: + - name: NAMESPACE + value: ($namespace) + content: | + #!/bin/bash + # Get logs from vector-aggregator-0 and check for specific log pattern + kubectl -n $NAMESPACE logs statefulset/vector-aggregator -c vector | \ + grep -q '"cluster":"sparkhistory","container":"node","errors":\[\],"file":"spark.log4j2.xml"' + exit_code=$? + + if [ $exit_code -eq 0 ]; then + echo "Found expected log pattern" + exit 0 + else + echo "Did not find expected log pattern" + exit 1 + fi + cleanup: + - sleep: + duration: 50s + catch: + - script: + env: + - name: NAMESPACE + value: ($namespace) + content: | + kubectl -n $NAMESPACE describe pods + - podLogs: + selector: app.kubernetes.io/instance=vector-aggregator + tail: -1 + - podLogs: + selector: app.kubernetes.io/instance=sparkhistory + tail: -1 diff --git a/test/e2e/default/sparkhistoryserver-assert.yaml b/test/e2e/logging/sparkhistoryserver-assert.yaml similarity index 100% rename from test/e2e/default/sparkhistoryserver-assert.yaml rename to test/e2e/logging/sparkhistoryserver-assert.yaml diff --git a/test/e2e/default/sparkhistoryserver.yaml b/test/e2e/logging/sparkhistoryserver.yaml similarity index 62% rename from test/e2e/default/sparkhistoryserver.yaml rename to test/e2e/logging/sparkhistoryserver.yaml index 7fd03c7..0ecc018 100644 --- a/test/e2e/default/sparkhistoryserver.yaml +++ b/test/e2e/logging/sparkhistoryserver.yaml @@ -3,6 +3,8 @@ kind: SparkHistoryServer metadata: name: sparkhistory spec: + image: + productVersion: ($values.product_version) clusterConfig: listenerClass: cluster-internal logFileDirectory: @@ -10,7 +12,12 @@ spec: prefix: events bucket: reference: spark-history + vectorAggregatorConfigMapName: vector-aggregator-discovery + node: roleGroups: default: replicas: 1 + config: + logging: + enableVectorAgent: true diff --git a/test/e2e/logging/vector-aggregator-assert.yaml b/test/e2e/logging/vector-aggregator-assert.yaml new file mode 100644 index 0000000..903ff12 --- /dev/null +++ b/test/e2e/logging/vector-aggregator-assert.yaml @@ -0,0 +1,15 @@ + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: vector-aggregator +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +data: + ADDRESS: "vector-aggregator:6000" +kind: ConfigMap +metadata: + name: vector-aggregator-discovery diff --git a/test/e2e/logging/vector-aggregator-values.yaml b/test/e2e/logging/vector-aggregator-values.yaml new file mode 100644 index 0000000..a496633 --- /dev/null +++ b/test/e2e/logging/vector-aggregator-values.yaml @@ -0,0 +1,16 @@ +api: + enabled: true +log_schema: + host_key: "pod" +sinks: + debug_console: + type: "console" + inputs: + - vector + encoding: + codec: "json" +sources: + vector: + address: 0.0.0.0:6000 + type: vector + version: "2" diff --git a/test/e2e/logging/vector-aggregator.yaml b/test/e2e/logging/vector-aggregator.yaml new file mode 100644 index 0000000..e2b7066 --- /dev/null +++ b/test/e2e/logging/vector-aggregator.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: "vector-aggregator:6000" diff --git a/test/e2e/oidc/chainsaw-test.yaml b/test/e2e/oidc/chainsaw-test.yaml index 1dc6548..96641da 100644 --- a/test/e2e/oidc/chainsaw-test.yaml +++ b/test/e2e/oidc/chainsaw-test.yaml @@ -4,38 +4,40 @@ metadata: name: oidc spec: bindings: - - name: MINIO_USER - value: miniouser - - name: MINIO_PASSWORD - value: miniouserpassword - - name: MINIO_BUCKET + - name: minio_user + value: spark + - name: minio_password + value: sparkspark + - name: minio_bucket value: spark-history - - name: KEYCLOAK_REALM + - name: keycloak_realm value: kubedoop - - name: KEYCLOAK_CLIENT_ID + - name: keycloak_client_id value: auth2-proxy - - name: KEYCLOAK_CLIENT_SECRET + - name: keycloak_client_secret value: auth2-proxy - - name: KEYCLOAK_USER + - name: keycloak_user value: user - - name: KEYCLOAK_PASSWORD + - name: keycloak_password value: password steps: - - try: - - apply: - file: ../setup/minio.yaml - - assert: - file: ../setup/minio-assert.yaml - - try: - - apply: - file: ../setup/spark-s3-credentials.yaml - - assert: - file: ../setup/spark-s3-credentials-assert.yaml - - apply: - file: ../setup/s3bucket.yaml - - assert: - file: ../setup/s3bucket.yaml - - try: + - name: install minio + try: + - script: + env: + - name: NAMESPACE + value: ($namespace) + - name: MINIO_USER + value: ($minio_user) + - name: MINIO_PASSWORD + value: ($minio_password) + content: | + envsubst < ../setup/minio-secret.yaml | kubectl apply -f - + - script: + content: | + bash ../setup/minio-install.sh + - name: install keycloak + try: - apply: file: keycloak.yaml - script: @@ -43,27 +45,31 @@ spec: - name: NAMESPACE value: ($namespace) - name: KEYCLOAK_REALM - value: ($KEYCLOAK_REALM) + value: ($keycloak_realm) - name: KEYCLOAK_USER - value: ($KEYCLOAK_USER) + value: ($keycloak_user) - name: KEYCLOAK_PASSWORD - value: ($KEYCLOAK_PASSWORD) + value: ($keycloak_password) - name: KEYCLOAK_CLIENT_ID - value: ($KEYCLOAK_CLIENT_ID) + value: ($keycloak_client_id) - name: KEYCLOAK_CLIENT_SECRET - value: ($KEYCLOAK_CLIENT_SECRET) + value: ($keycloak_client_secret) content: | cat keycloak-config.yaml | envsubst | kubectl apply -n $NAMESPACE -f - - assert: file: keycloak-assert.yaml - - try: + - name: install sparkhistoryserver cluster + try: + - apply: + file: ../setup/minio-s3-connection.yaml - apply: file: oidc-credentials.yaml - apply: - file: authenticationclass.yaml - - try: + file: oidc-authenticationclass.yaml - apply: file: sparkhistoryserver.yaml + - sleep: + duration: 100s - assert: file: sparkhistoryserver-assert.yaml cleanup: @@ -82,7 +88,33 @@ spec: - describe: apiVersion: v1 kind: Pod - selector: app.kubernetes.io/instance=sparkhistory + # selector: app.kubernetes.io/instance=sparkhistory - podLogs: selector: app.kubernetes.io/instance=sparkhistory tail: -1 + - name: test OIDC authentication + try: + - apply: + file: ../setup/testing-tools-install.yaml + - assert: + file: ../setup/testing-tools-assert.yaml + - script: + env: + - name: NAMESPACE + value: ($namespace) + content: | + echo "env subst from oidc-login-assert.py" + envsubst < oidc-login-assert.py | kubectl exec -n $NAMESPACE -i testing-tools-0 -- tee /tmp/oidc-login-assert.py > /dev/null + + echo "asserting the OIDC login" + kubectl exec -n $NAMESPACE testing-tools-0 -- python /tmp/oidc-login-assert.py + catch: + - script: + env: + - name: NAMESPACE + value: ($namespace) + content: | + set -ex + kubectl -n $NAMESPACE get pods + kubectl -n $NAMESPACE describe pods + kubectl -n $NAMESPACE logs testing-tools-0 diff --git a/test/e2e/oidc/keycloak-config.yaml b/test/e2e/oidc/keycloak-config.yaml index a7b3208..9323457 100644 --- a/test/e2e/oidc/keycloak-config.yaml +++ b/test/e2e/oidc/keycloak-config.yaml @@ -17,6 +17,7 @@ data: "firstName": "user", "lastName": "user", "email": "user@example.com", + "emailVerified": true, "credentials": [ { "type": "password", diff --git a/test/e2e/oidc/keycloak.yaml b/test/e2e/oidc/keycloak.yaml index 187fba4..7c736a4 100644 --- a/test/e2e/oidc/keycloak.yaml +++ b/test/e2e/oidc/keycloak.yaml @@ -29,14 +29,14 @@ spec: resources: limits: memory: "1Gi" - cpu: "1000m" + cpu: "850m" ports: - containerPort: 8080 name: http readinessProbe: httpGet: scheme: HTTP - path: (join('', ['/realms/', ($KEYCLOAK_REALM)])) + path: (join('', ['/realms/', ($keycloak_realm)])) port: 8080 volumeMounts: - name: keycloak-config @@ -54,5 +54,5 @@ spec: selector: app: keycloak ports: - - port: 80 + - port: 8080 targetPort: http diff --git a/test/e2e/oidc/authenticationclass.yaml b/test/e2e/oidc/oidc-authenticationclass.yaml similarity index 81% rename from test/e2e/oidc/authenticationclass.yaml rename to test/e2e/oidc/oidc-authenticationclass.yaml index 9690883..05a0403 100644 --- a/test/e2e/oidc/authenticationclass.yaml +++ b/test/e2e/oidc/oidc-authenticationclass.yaml @@ -6,8 +6,8 @@ spec: provider: oidc: hostname: (join('', ['keycloak.', ($namespace), '.svc.cluster.local'])) - port: 80 - rootPath: (join('', ['/realms/', ($KEYCLOAK_REALM)])) + port: 8080 + rootPath: (join('', ['/realms/', ($keycloak_realm)])) providerHint: keycloak scopes: - openid diff --git a/test/e2e/oidc/oidc-credentials.yaml b/test/e2e/oidc/oidc-credentials.yaml index 09f020d..e0ccc5e 100644 --- a/test/e2e/oidc/oidc-credentials.yaml +++ b/test/e2e/oidc/oidc-credentials.yaml @@ -4,5 +4,5 @@ metadata: name: oidc-credentials type: Opaque stringData: - CLIENT_ID: ($KEYCLOAK_CLIENT_ID) - CLIENT_SECRET: ($KEYCLOAK_CLIENT_SECRET) + CLIENT_ID: ($keycloak_client_id) + CLIENT_SECRET: ($keycloak_client_secret) diff --git a/test/e2e/oidc/oidc-login-assert.py b/test/e2e/oidc/oidc-login-assert.py new file mode 100644 index 0000000..db00395 --- /dev/null +++ b/test/e2e/oidc/oidc-login-assert.py @@ -0,0 +1,37 @@ +# $NAMESPACE will be replaced with the namespace of the test case. + +import json +import logging +import requests +import sys +from bs4 import BeautifulSoup + +logging.basicConfig( + level='DEBUG', + format="%(asctime)s %(levelname)s: %(message)s", + stream=sys.stdout) + +session = requests.Session() + +# Click on "Sign In with keycloak" in Spark-history +logging.info("Opening the Spark-history login page") +login_page = session.get("http://sparkhistory-node-default:4180/oauth2/start?rd=%2F") + +assert login_page.ok, "Redirection from Spark-history to Keycloak failed" +assert login_page.url.startswith("http://keycloak.$NAMESPACE.svc.cluster.local:8080/realms/kubedoop/protocol/openid-connect/auth?approval_prompt=force&client_id=auth2-proxy"), \ + f"Redirection to the Keycloak login page expected, actual URL: {login_page.url}" + +# Enter username and password into the Keycloak login page and click on "Sign In" +logging.info("Logging by oauth2-proxy") +login_page_html = BeautifulSoup(login_page.text, 'html.parser') +authenticate_url = login_page_html.form['action'] +welcome_page = session.post(authenticate_url, data={ + 'username': "user", + 'password': "password", +}) + +assert welcome_page.ok, "Login failed" +assert welcome_page.url.rstrip("/") == "http://sparkhistory-node-default:4180", \ + f"Redirection to the Spark-history welcome page expected, actual URL: {welcome_page.url}" + +# TODO: sign out from the oauth2-proxy, but sign out from the Keycloak is not possible diff --git a/test/e2e/oidc/sparkhistoryserver.yaml b/test/e2e/oidc/sparkhistoryserver.yaml index 350e76e..55f50dc 100644 --- a/test/e2e/oidc/sparkhistoryserver.yaml +++ b/test/e2e/oidc/sparkhistoryserver.yaml @@ -3,6 +3,8 @@ kind: SparkHistoryServer metadata: name: sparkhistory spec: + image: + productVersion: ($values.product_version) clusterConfig: listenerClass: cluster-internal authentication: @@ -18,3 +20,10 @@ spec: roleGroups: default: replicas: 1 + # config: + # resources: + # cpu: + # max: 650m + # min: 300m + # memory: + # limit: 800Mi diff --git a/test/e2e/pdb/chainsaw-test.yaml b/test/e2e/pdb/chainsaw-test.yaml deleted file mode 100644 index e0fed46..0000000 --- a/test/e2e/pdb/chainsaw-test.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: chainsaw.kyverno.io/v1alpha1 -kind: Test -metadata: - name: pdb -spec: - bindings: - - name: MINIO_USER - value: miniouser - - name: MINIO_PASSWORD - value: miniouserpassword - - name: MINIO_BUCKET - value: spark-history - steps: - - try: - - apply: - file: ../setup/minio.yaml - - assert: - file: ../setup/minio-assert.yaml - - try: - - apply: - file: ../setup/spark-s3-credentials.yaml - - assert: - file: ../setup/spark-s3-credentials-assert.yaml - - apply: - file: ../setup/s3bucket.yaml - - assert: - file: ../setup/s3bucket.yaml - cleanup: - - try: - - apply: - file: sparkhistoryserver.yaml - - assert: - file: sparkhistoryserver-assert.yaml - - assert: - file: pdb-assert.yaml - cleanup: - - sleep: - duration: 5s diff --git a/test/e2e/pdb/pdb-assert.yaml b/test/e2e/pdb/pdb-assert.yaml deleted file mode 100644 index aef7ef3..0000000 --- a/test/e2e/pdb/pdb-assert.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: policy/v1 -kind: PodDisruptionBudget -metadata: - name: sparkhistory-node -spec: - maxUnavailable: 2 diff --git a/test/e2e/pdb/sparkhistoryserver-assert.yaml b/test/e2e/pdb/sparkhistoryserver-assert.yaml deleted file mode 100644 index 7546d27..0000000 --- a/test/e2e/pdb/sparkhistoryserver-assert.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sparkhistory-node-default -status: - availableReplicas: 1 - replicas: 1 - readyReplicas: 1 ---- -apiVersion: v1 -kind: Service -metadata: - name: sparkhistory-node-default -spec: - type: ClusterIP diff --git a/test/e2e/setup/helm-bitnami-minio-values.yaml b/test/e2e/setup/helm-bitnami-minio-values.yaml new file mode 100644 index 0000000..9ca718f --- /dev/null +++ b/test/e2e/setup/helm-bitnami-minio-values.yaml @@ -0,0 +1,54 @@ +--- +mode: standalone +disableWebUI: false +extraEnvVars: + # - name: BITNAMI_DEBUG + # value: "true" + - name: MINIO_LOG_LEVEL + value: DEBUG + +defaultBuckets: spark-history/events + +provisioning: + enabled: true + buckets: + - name: spark-history + usersExistingSecrets: + - centralized-minio-users + resources: + requests: + memory: 512Mi + cpu: "200m" + limits: + memory: "600Mi" + cpu: "300m" + podSecurityContext: + enabled: false + containerSecurityContext: + enabled: false + +volumePermissions: + enabled: false + +podSecurityContext: + enabled: false + +containerSecurityContext: + enabled: false + +persistence: + enabled: false + +resources: + requests: + memory: 512Mi + cpu: "200m" + limits: + memory: "600Mi" + cpu: "300m" + +auth: + existingSecret: minio-credentials + +service: + type: NodePort diff --git a/test/e2e/setup/minio-install.sh b/test/e2e/setup/minio-install.sh new file mode 100644 index 0000000..7af36da --- /dev/null +++ b/test/e2e/setup/minio-install.sh @@ -0,0 +1,7 @@ +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +helm upgrade --install minio \ + --namespace $NAMESPACE \ + --version 12.6.4 \ + -f "${SCRIPT_DIR}/helm-bitnami-minio-values.yaml" \ + --repo https://charts.bitnami.com/bitnami minio diff --git a/test/e2e/setup/minio-s3-connection.yaml b/test/e2e/setup/minio-s3-connection.yaml new file mode 100644 index 0000000..a253a92 --- /dev/null +++ b/test/e2e/setup/minio-s3-connection.yaml @@ -0,0 +1,41 @@ +apiVersion: s3.kubedoop.dev/v1alpha1 +kind: S3Bucket +metadata: + name: spark-history +spec: + bucketName: spark-history + connection: + reference: minio +--- +apiVersion: s3.kubedoop.dev/v1alpha1 +kind: S3Connection +metadata: + name: minio + namespace: ($namespace) +spec: + host: minio + port: 9000 + credentials: + secretClass: s3-credentials +--- +apiVersion: secrets.kubedoop.dev/v1alpha1 +kind: SecretClass +metadata: + name: s3-credentials + namespace: ($namespace) +spec: + backend: + k8sSearch: + searchNamespace: + pod: {} +--- +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials + namespace: ($namespace) + labels: + secrets.kubedoop.dev/class: s3-credentials +stringData: + ACCESS_KEY: ($minio_user) + SECRET_KEY: ($minio_password) diff --git a/test/e2e/setup/minio-secret.yaml b/test/e2e/setup/minio-secret.yaml new file mode 100644 index 0000000..9977e91 --- /dev/null +++ b/test/e2e/setup/minio-secret.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Secret +metadata: + name: minio-credentials + namespace: ${NAMESPACE} + labels: + secrets.kubedoop.dev/class: s3-credentials-class # important!!! +stringData: + accessKey: minioAccessKey + secretKey: minioSecretKey + # The following two entries are used by the Bitnami chart for MinIO to + # set up credentials for accessing buckets managed by the MinIO tenant. + root-user: minioAccessKey + root-password: minioSecretKey +--- +apiVersion: v1 +kind: Secret +metadata: + name: centralized-minio-users + namespace: ${NAMESPACE} +type: Opaque +stringData: + username1: | + username=${MINIO_USER} + password=${MINIO_PASSWORD} + disabled=false + policies=readwrite,consoleAdmin,diagnostics + setPolicies=false diff --git a/test/e2e/setup/minio.yaml b/test/e2e/setup/minio.yaml deleted file mode 100644 index a03c18a..0000000 --- a/test/e2e/setup/minio.yaml +++ /dev/null @@ -1,166 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: minio-admin -type: Opaque -stringData: - MINIO_ROOT_USER: minioadmin - MINIO_ROOT_PASSWORD: minioadmin - MINIO_USER: ($MINIO_USER) - MINIO_PASSWORD: ($MINIO_PASSWORD) - MINIO_BUCKET: ($MINIO_BUCKET) ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: minio-script -data: - init.sh: | - #!/bin/sh - echo "init minio" - ## config mc - mc alias set minio http://localhost:9000 ${MINIO_ROOT_USER} ${MINIO_ROOT_USER} - mc admin info minio - - ## add user - mc admin user add minio ${MINIO_USER} ${MINIO_PASSWORD} - mc admin user list minio - - ## add bucket - mc mb minio/${MINIO_BUCKET}/events - mc ls minio - - ## set policy - cat < /tmp/policy.json - { - "Version": "2012-10-17", - "Id": "LakeHouseBuckeyPolicy", - "Statement": [ - { - "Sid": "Stment01", - "Effect": "Allow", - "Action": [ - "s3:GetBucketLocation", - "s3:ListBucket", - "s3:ListBucketMultipartUploads", - "s3:ListBucketVersions", - "s3:GetObject", - "s3:PutObject", - "s3:DeleteObject", - "s3:ListMultipartUploadParts", - "s3:AbortMultipartUpload" - ], - "Resource": [ - "arn:aws:s3:::${MINIO_BUCKET}/*", - "arn:aws:s3:::${MINIO_BUCKET}" - ] - } - ] - } - EOF - mc admin policy create minio ${MINIO_BUCKET} /tmp/policy.json - mc admin policy list minio - - ## attach policy - policy_count=$(mc admin policy entities --user foo minio | wc -l) - if [ $policy_count -eq 1 ]; then - mc admin policy attach minio ${MINIO_BUCKET} --user ${MINIO_USER} - fi - echo "init minio done" ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - labels: - app: minio -spec: - replicas: 1 - selector: - matchLabels: - app: minio - template: - metadata: - labels: - app: minio - spec: - containers: - - name: minio - image: quay.io/minio/minio:latest - command: - - /bin/bash - - -c - args: - - | - function delay_init() { - sleep 5 - exec /init.sh - } - - delay_init & - minio server /data --console-address "0.0.0.0:9001" --address "0.0.0.0:9000" - envFrom: - - secretRef: - name: minio-admin - ports: - - containerPort: 9000 - name: http - protocol: TCP - - containerPort: 9001 - name: console - protocol: TCP - resources: - requests: - cpu: 128m - memory: 512Mi - limits: - cpu: 512m - memory: 1Gi - volumeMounts: - - name: minio-storage - mountPath: /data - - name: minio-script - mountPath: /init.sh - subPath: init.sh - readinessProbe: - exec: - command: - - sh - - -c - - | - set -e - # check policy count > 1 - policy_count=$(mc admin policy entities --user ${MINIO_USER} minio | wc -l) - if [ $policy_count -ge 1 ]; then - exit 0 - else - echo "policy count: $policy_count" - exit 1 - fi - initialDelaySeconds: 5 - periodSeconds: 5 - timeoutSeconds: 3 - volumes: - - name: minio-storage - emptyDir: {} - - name: minio-script - configMap: - name: minio-script - defaultMode: 0755 ---- -apiVersion: v1 -kind: Service -metadata: - name: minio -spec: - type: ClusterIP - ports: - - port: 9000 - name: http - targetPort: 9000 - - port: 9001 - name: console - targetPort: 9001 - selector: - app: minio diff --git a/test/e2e/setup/s3bucket.yaml b/test/e2e/setup/s3bucket.yaml deleted file mode 100644 index 3ae5748..0000000 --- a/test/e2e/setup/s3bucket.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: s3.kubedoop.dev/v1alpha1 -kind: S3Bucket -metadata: - name: spark-history -spec: - bucketName: spark-history - connection: - inline: - credentials: - secretClass: spark-s3-credentials - host: (join('', ['minio.', ($namespace), '.svc.cluster.local'])) - port: 9000 - pathStyle: true diff --git a/test/e2e/setup/spark-s3-credentials-assert.yaml b/test/e2e/setup/spark-s3-credentials-assert.yaml deleted file mode 100644 index 0b023ba..0000000 --- a/test/e2e/setup/spark-s3-credentials-assert.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: spark-s3-credentials - labels: - secrets.kubedoop.dev/class: spark-s3-credentials ---- -apiVersion: secrets.kubedoop.dev/v1alpha1 -kind: SecretClass -metadata: - name: spark-s3-credentials -spec: - backend: - k8sSearch: - searchNamespace: - pod: {} diff --git a/test/e2e/setup/spark-s3-credentials.yaml b/test/e2e/setup/spark-s3-credentials.yaml deleted file mode 100644 index 83f8331..0000000 --- a/test/e2e/setup/spark-s3-credentials.yaml +++ /dev/null @@ -1,21 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: spark-s3-credentials - labels: - secrets.kubedoop.dev/class: spark-s3-credentials -type: Opaque -stringData: - ACCESS_KEY: ($MINIO_USER) - SECRET_KEY: ($MINIO_PASSWORD) ---- -apiVersion: secrets.kubedoop.dev/v1alpha1 -kind: SecretClass -metadata: - name: spark-s3-credentials -spec: - backend: - k8sSearch: - searchNamespace: - pod: {} diff --git a/test/e2e/setup/minio-assert.yaml b/test/e2e/setup/testing-tools-assert.yaml similarity index 56% rename from test/e2e/setup/minio-assert.yaml rename to test/e2e/setup/testing-tools-assert.yaml index 6dd6c4b..60eeb0c 100644 --- a/test/e2e/setup/minio-assert.yaml +++ b/test/e2e/setup/testing-tools-assert.yaml @@ -1,8 +1,7 @@ apiVersion: apps/v1 -kind: Deployment +kind: StatefulSet metadata: - name: minio + name: testing-tools status: readyReplicas: 1 - availableReplicas: 1 replicas: 1 diff --git a/test/e2e/setup/testing-tools-install.yaml b/test/e2e/setup/testing-tools-install.yaml new file mode 100644 index 0000000..0235591 --- /dev/null +++ b/test/e2e/setup/testing-tools-install.yaml @@ -0,0 +1,30 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: testing-tools + labels: + app: testing-tools +spec: + replicas: 1 + selector: + matchLabels: + app: testing-tools + template: + metadata: + labels: + app: testing-tools + spec: + # serviceAccount: integration-tests-sa + securityContext: + fsGroup: 1000 + containers: + - name: testing-tools + image: quay.io/zncdatadev/testing-tools:0.1.0-kubedoop0.0.0-dev + command: ["sleep", "infinity"] + securityContext: + runAsUser: 0 + runAsGroup: 0 + # resources: + # limits: + # memory: "1Gi" + # cpu: "200m" diff --git a/test/e2e/smoke/cluster-operation/chainsaw-test.yaml b/test/e2e/smoke/cluster-operation/chainsaw-test.yaml new file mode 100644 index 0000000..6c31113 --- /dev/null +++ b/test/e2e/smoke/cluster-operation/chainsaw-test.yaml @@ -0,0 +1,93 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: smoke-cluster-operation +spec: + bindings: + - name: minio_user + value: spark + - name: minio_password + value: sparkspark + - name: minio_bucket + value: spark-history + - name: cluster_stopped + value: false + - name: cluster_paused + value: false + steps: + - name: install minio + try: + - script: + env: + - name: NAMESPACE + value: ($namespace) + - name: MINIO_USER + value: ($minio_user) + - name: MINIO_PASSWORD + value: ($minio_password) + content: | + envsubst < ../../setup/minio-secret.yaml | kubectl apply -f - + - script: + content: | + bash ../../setup/minio-install.sh + - name: install sparkhistoryserver + try: + # install sparkhistoryserver sparkhistoryserver, clusterOperation.stopped: false, clusterOperation.reconciliationPaused: false + - apply: + file: ../../setup/minio-s3-connection.yaml + - apply: + file: sparkhistoryserver.yaml + - assert: + bindings: + - name: available_replicas + value: 1 + file: sparkhistoryserver-assert.yaml + - name: stop sparkhistoryserver-sparkhistoryserver cluster + try: + - apply: + bindings: + - name: cluster_stopped + value: true + file: sparkhistoryserver.yaml + - assert: + resource: # TODO: refactor the sparkhistoryserver kind to statefulset + apiVersion: apps/v1 + kind: Deployment + metadata: + name: test-sparkhistoryserver-node-default + status: + conditions: + - reason: MinimumReplicasAvailable + type: Available + status: "True" + - reason: NewReplicaSetAvailable + type: Progressing + status: "True" + observedGeneration: 2 + # pause sparkhistoryserver-sparkhistoryserver cluster, clusterOperation.reconciliationPaused: true + - name: pause sparkhistoryserver-sparkhistoryserver cluster + try: + - apply: + bindings: + - name: cluster_paused + value: true + - name: cluster_stopped + value: false + file: sparkhistoryserver.yaml + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + # restart sparkhistoryserver-sparkhistoryserver cluster, clusterOperation.stopped: false, clusterOperation.reconciliationPaused: false + - name: restart sparkhistoryserver-sparkhistoryserver cluster + try: + - apply: + file: sparkhistoryserver.yaml + - assert: + bindings: + - name: available_replicas + value: 1 + file: sparkhistoryserver-assert.yaml + cleanup: + - sleep: + duration: 30s diff --git a/test/e2e/smoke/cluster-operation/sparkhistoryserver-assert.yaml b/test/e2e/smoke/cluster-operation/sparkhistoryserver-assert.yaml new file mode 100644 index 0000000..64d3798 --- /dev/null +++ b/test/e2e/smoke/cluster-operation/sparkhistoryserver-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: test-sparkhistoryserver-node-default +status: + availableReplicas: ($available_replicas) + replicas: ($available_replicas) diff --git a/test/e2e/pdb/sparkhistoryserver.yaml b/test/e2e/smoke/cluster-operation/sparkhistoryserver.yaml similarity index 62% rename from test/e2e/pdb/sparkhistoryserver.yaml rename to test/e2e/smoke/cluster-operation/sparkhistoryserver.yaml index ca395f2..3ecbbf5 100644 --- a/test/e2e/pdb/sparkhistoryserver.yaml +++ b/test/e2e/smoke/cluster-operation/sparkhistoryserver.yaml @@ -1,8 +1,13 @@ apiVersion: spark.kubedoop.dev/v1alpha1 kind: SparkHistoryServer metadata: - name: sparkhistory + name: test-sparkhistoryserver spec: + image: + productVersion: ($values.product_version) + clusterOperation: + reconciliationPaused: ($cluster_paused) + stopped: ($cluster_stopped) clusterConfig: listenerClass: cluster-internal logFileDirectory: @@ -11,9 +16,6 @@ spec: bucket: reference: spark-history node: - roleConfig: - podDisruptionBudget: - maxUnavailable: 2 roleGroups: default: replicas: 1 diff --git a/test/e2e/smoke/override-pdb/chainsaw-test.yaml b/test/e2e/smoke/override-pdb/chainsaw-test.yaml new file mode 100644 index 0000000..36ce539 --- /dev/null +++ b/test/e2e/smoke/override-pdb/chainsaw-test.yaml @@ -0,0 +1,158 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: smoke-override-pdb +spec: + bindings: + - name: minio_user + value: spark + - name: minio_password + value: sparkspark + - name: minio_bucket + value: spark-history + - name: cluster_stopped + value: false + - name: cluster_paused + value: false + steps: + - name: install minio + try: + - script: + env: + - name: NAMESPACE + value: ($namespace) + - name: MINIO_USER + value: ($minio_user) + - name: MINIO_PASSWORD + value: ($minio_password) + content: | + envsubst < ../../setup/minio-secret.yaml | kubectl apply -f - + - script: + content: | + bash ../../setup/minio-install.sh + - name: install sparkhistoryserver + try: + # install sparkhistoryserver sparkhistoryserver, clusterOperation.stopped: false, clusterOperation.reconciliationPaused: false + - apply: + file: ../../setup/minio-s3-connection.yaml + - apply: + file: sparkhistoryserver.yaml + - assert: + bindings: + - name: available_replicas + value: 1 + file: sparkhistoryserver-assert.yaml + - name: assert pdb + try: + - assert: + timeout: 240s + resource: + kind: PodDisruptionBudget + apiVersion: policy/v1 + metadata: + name: test-sparkhistoryserver-node + namespace: ($namespace) + spec: + maxUnavailable: 1 + status: + expectedPods: 2 + currentHealthy: 2 + disruptionsAllowed: 1 + - name: assert podOverride + try: + - assert: + timeout: 240s + resource: + kind: Deployment + apiVersion: apps/v1 + metadata: + name: test-sparkhistoryserver-node-default + namespace: ($namespace) + spec: + template: + spec: + containers: + - name: node + resources: + requests: + cpu: 300m + memory: 526Mi + limits: + cpu: 350m + memory: 1300Mi + status: + replicas: 2 + readyReplicas: 2 + availableReplicas: 2 + - name: test env overrides + try: + - script: + bindings: + - name: NAMESPACE + value: ($namespace) + content: | + #!/bin/bash + kubectl -n $NAMESPACE get deployment test-sparkhistoryserver-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "node") | .env[] | select (.name == "COMMON_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get deployment test-sparkhistoryserver-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "node") | .env[] | select (.name == "GROUP_VAR" and .value == "group-value")' + kubectl -n $NAMESPACE get deployment test-sparkhistoryserver-node-default -o yaml | yq -e '.spec.template.spec.containers[] | select (.name == "node") | .env[] | select (.name == "ROLE_VAR" and .value == "role-value")' + - name: run spark example + try: + - sleep: + duration: 240s + - script: + bindings: + - name: NAMESPACE + value: ($namespace) + content: | + #!/bin/bash + set -x + POD=$(kubectl -n $NAMESPACE get pods -l app.kubernetes.io/name=sparkhistoryserver --field-selector status.phase=Running -o jsonpath='{.items[0].metadata.name}') + + + script=$(cat <