test(e2e): refactor e2e

.chainsaw.yaml

@@ -5,11 +5,11 @@ metadata:
 spec:
   timeouts:
     apply: 120s
-    assert: 180s
+    assert: 200s
     cleanup: 120s
     delete: 120s
     error: 10s
     exec: 45s
+  forceTerminationGracePeriod: 10s
   # skipDelete: true
   failFast: true
-  parallel: 1 # use 1 concurrent to test, to voide multiple csi driver conflict

Makefile

@@ -339,9 +339,8 @@ $(CHAINSAW): $(LOCALBIN)
 
 .PHONY: chainsaw-setup
 chainsaw-setup: ## Run the chainsaw setup
-	make docker-build
 	make csi-docker-build
-	$(KIND) --name $(KIND_CLUSTER_NAME) load docker-image $(IMG) $(CSIDRIVER_IMG)
+	$(KIND) --name $(KIND_CLUSTER_NAME) load docker-image $(CSIDRIVER_IMG)
 	KUBECONFIG=$(KIND_KUBECONFIG) make helm-install-depends
 	KUBECONFIG=$(KIND_KUBECONFIG) make deploy
 

cmd/csi_driver/main.go

@@ -29,6 +29,7 @@ import (
 	// to ensure that exec-entrypoint and run can make use of them.
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
+	listenerv1alpha1 "github.com/zncdatadev/operator-go/pkg/apis/listeners/v1alpha1"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -55,6 +56,7 @@ var (
 
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
+	utilruntime.Must(listenerv1alpha1.AddToScheme(scheme))
 
 	utilruntime.Must(secretv1alpha1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme

config/csi/daemonset.yaml

@@ -27,19 +27,19 @@ spec:
       securityContext:
         {}
       containers:
-        - name: secret-operator
+        - name: secret-csi-driver
           securityContext:
             privileged: true
             runAsUser: 0
           image: "csi-driver:latest"
           imagePullPolicy: IfNotPresent
           resources:
             limits:
-              cpu: 100m
+              cpu: 512m
               memory: 128Mi
             requests:
-              cpu: 100m
-              memory: 128Mi
+              cpu: 50m
+              memory: 50Mi
           env:
             - name: ADDRESS
               value: unix:///csi/csi.sock

config/rbac/role.yaml

@@ -38,6 +38,14 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - listeners.kubedoop.dev
+  resources:
+  - listeners
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - secrets.kubedoop.dev
   resources:

internal/controller/secretclass_controller.go

@@ -44,6 +44,7 @@ type SecretClassReconciler struct {
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch
 // +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;create;update;patch;delete
+// +kubebuilder:rbac:groups=listeners.kubedoop.dev,resources=listeners,verbs=get;list;watch
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

internal/csi/backend/ca/ca_manager.go

@@ -95,7 +95,7 @@ func (c *CertificateManager) secretCreateIfDoesNotExist(ctx context.Context) err
 		return err
 	}
 
-	logger.V(1).Info("Created a new secret", "name", c.name, "namespace", c.namespace, "auto", c.auto)
+	logger.V(1).Info("created a new secret", "name", c.name, "namespace", c.namespace, "auto", c.auto)
 	return nil
 
 }
@@ -115,7 +115,7 @@ func (c CertificateManager) getPEMKeyPairsFromSecret(ctx context.Context) ([]PEM
 		}
 	}
 
-	logger.V(0).Info("Get certificate authorities PEM key pairs from secret", "name", c.name, "namespace", c.namespace, "len", len(keyPairs))
+	logger.V(0).Info("got certificate authorities PEM key pairs from secret", "name", c.name, "namespace", c.namespace, "len", len(keyPairs))
 	return keyPairs, nil
 }
 
@@ -171,7 +171,7 @@ func (c *CertificateManager) getCertificateAuthorities(pemKeyPairs []PEMkeyPair)
 			return nil, err
 		}
 		if ca.Certificate.NotAfter.Before(time.Now()) {
-			logger.V(0).Info("Certificate authority is expired, skip it.", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
+			logger.V(0).Info("certificate authority is expired, skip it.", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
 			continue
 		}
 		cas = append(cas, ca)
@@ -187,7 +187,7 @@ func (c *CertificateManager) getCertificateAuthorities(pemKeyPairs []PEMkeyPair)
 			)
 		}
 
-		logger.V(0).Info("Could not find any certificate authorities, created a new self-signed certificate authority", "name", c.name, "namespace", c.namespace, "auto", c.auto)
+		logger.V(0).Info("could not find any certificate authorities, created a new self-signed certificate authority", "name", c.name, "namespace", c.namespace, "auto", c.auto)
 		ca, err := c.createSelfSignedCertificateAuthority()
 		if err != nil {
 			return nil, err
@@ -213,7 +213,7 @@ func (c *CertificateManager) createSelfSignedCertificateAuthority() (*Certificat
 	if err != nil {
 		return nil, err
 	}
-	logger.V(0).Info("Created new self-signed certificate authority", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
+	logger.V(0).Info("created new self-signed certificate authority", "serialNumber", ca.SerialNumber(), "notAfter", ca.Certificate.NotAfter)
 	return ca, nil
 }
 
@@ -248,19 +248,19 @@ func (c *CertificateManager) rotateCertificateAuthority(cas []*CertificateAuthor
 			if err != nil {
 				return nil, err
 			}
-			logger.V(0).Info("Rotated certificate authority, because the old ca is about to expire",
+			logger.V(0).Info("rotated certificate authority, because the old ca is about to expire",
 				"serialNumber", newestCA.SerialNumber(),
 				"notAfter", newCA.Certificate.NotAfter,
 			)
 			cas = append(cas, newCA)
 		} else {
-			logger.V(0).Info("Certificate authority is about to expire, but auto-generate is disabled, please rotate manually.",
+			logger.V(0).Info("certificate authority is about to expire, but auto-generate is disabled, please rotate manually.",
 				"serialNumber", newestCA.SerialNumber(),
 				"notAfter", newestCA.Certificate.NotAfter,
 			)
 		}
 	} else {
-		logger.V(0).Info("Certificate authority is still valid, no need to rotate",
+		logger.V(0).Info("certificate authority is still valid, no need to rotate",
 			"serialNumber", newestCA.SerialNumber(),
 			"notAfter", newestCA.Certificate.NotAfter,
 		)
@@ -283,7 +283,7 @@ func (c *CertificateManager) getAliveCertificateAuthority(atAfter time.Time, cas
 		}
 		return 0
 	})
-	logger.V(0).Info("Get alive certificate authority", "serialNumber", oldestCA.SerialNumber(), "notAfter", oldestCA.Certificate.NotAfter)
+	logger.V(0).Info("got alive certificate authority", "serialNumber", oldestCA.SerialNumber(), "notAfter", oldestCA.Certificate.NotAfter)
 
 	return oldestCA
 }

internal/csi/controller.go

@@ -56,7 +56,7 @@ func (c *ControllerServer) CreateVolume(ctx context.Context, request *csi.Create
 	requiredCap := request.CapacityRange.GetRequiredBytes()
 
 	if request.Parameters["secretFinalizer"] == "true" {
-		logger.V(1).Info("Finalizer is true")
+		logger.V(1).Info("finalizer is true")
 	}
 
 	// requests.parameters is StorageClass.Parameters, which is set by user when creating PVC.
@@ -179,7 +179,7 @@ func (c *ControllerServer) DeleteVolume(ctx context.Context, request *csi.Delete
 	}
 
 	if !dynamic {
-		logger.V(5).Info("Volume is not dynamic, skip delete volume")
+		logger.V(5).Info("volume is not dynamic, skip delete volume")
 		return &csi.DeleteVolumeResponse{}, nil
 	}
 

internal/csi/driver.go

@@ -47,7 +47,7 @@ func NewDriver(
 
 func (d *Driver) Run(ctx context.Context, testMode bool) error {
 
-	logger.V(1).Info("Driver information", "versionInfo", version.GetVersion(d.name))
+	logger.V(1).Info("driver information", "versionInfo", version.GetVersion(d.name))
 
 	// check node id
 	if d.nodeID == "" {
@@ -56,7 +56,7 @@ func (d *Driver) Run(ctx context.Context, testMode bool) error {
 
 	ns := NewNodeServer(
 		d.nodeID,
-		mount.New(""),
+		mount.New("secret-csi"),
 		d.client,
 	)
 
@@ -72,7 +72,7 @@ func (d *Driver) Run(ctx context.Context, testMode bool) error {
 	}()
 
 	d.server.Wait()
-	logger.Info("Server stopped")
+	logger.Info("csi driver stopped")
 	return nil
 }
 

internal/csi/node.go

@@ -144,7 +144,7 @@ func (n *NodeServer) updatePod(ctx context.Context, pod *corev1.Pod, volumeID st
 	}
 	patch := client.MergeFrom(pod.DeepCopy())
 	if expiresTime == nil {
-		logger.V(5).Info("Expiration time is nil, skip update pod annotation", "pod", pod.Name)
+		logger.V(5).Info("expiration time is nil, skip update pod annotation", "pod", pod.Name)
 		return nil
 	}
 
@@ -157,14 +157,14 @@ func (n *NodeServer) updatePod(ctx context.Context, pod *corev1.Pod, volumeID st
 
 	annotationExpiresName := constants.PrefixLabelRestarterExpiresAt + hex.EncodeToString(volumeTag)
 	expiresTimeStr := expiresTime.Format(time.RFC3339)
-	logger.V(5).Info("Update pod annotation", "pod", pod.Name, "key", annotationExpiresName, "value", expiresTimeStr)
+	logger.V(5).Info("update pod annotation", "pod", pod.Name, "key", annotationExpiresName, "value", expiresTimeStr)
 
 	pod.Annotations[annotationExpiresName] = expiresTimeStr
 
 	if err := n.client.Patch(ctx, pod, patch); err != nil {
 		return err
 	}
-	logger.V(5).Info("Pod patched", "pod", pod.Name)
+	logger.V(5).Info("pod patched", "pod", pod.Name)
 	return nil
 }
 
@@ -177,9 +177,9 @@ func (n *NodeServer) writeData(targetPath string, data map[string]string) error
 		if err := os.WriteFile(fileName, []byte(content), fs.FileMode(0644)); err != nil {
 			return err
 		}
-		logger.V(5).Info("File written", "file", fileName)
+		logger.V(5).Info("file written", "file", fileName)
 	}
-	logger.V(5).Info("Data written", "target", targetPath)
+	logger.V(5).Info("data written", "target", targetPath)
 	return nil
 }
 
@@ -218,7 +218,7 @@ func (n *NodeServer) mount(targetPath string) error {
 	if err := n.mounter.Mount("tmpfs", targetPath, "tmpfs", opts); err != nil {
 		return status.Error(codes.Internal, err.Error())
 	}
-	logger.V(1).Info("Volume mounted", "source", "tmpfs", "target", targetPath, "fsType", "tmpfs", "options", opts)
+	logger.V(1).Info("volume mounted", "source", "tmpfs", "target", targetPath, "fsType", "tmpfs", "options", opts)
 	return nil
 }
 
@@ -239,7 +239,7 @@ func (n *NodeServer) NodeUnpublishVolume(ctx context.Context, request *csi.NodeU
 	if err := n.mounter.Unmount(targetPath); err != nil {
 		// FIXME: use status.Error to return error
 		// return nil, status.Error(codes.Internal, err.Error())
-		logger.V(0).Info("Volume not found, skip delete volume")
+		logger.V(0).Info("volume not found, skip delete volume")
 	}
 
 	// remove the target path

internal/csi/server.go

@@ -72,7 +72,7 @@ func (s *nonBlockingServer) serveGrpc(endpoint string, ids csi.IdentityServer, c
 	if proto == "unix" {
 		addr = "/" + addr
 		if err := os.Remove(addr); err != nil && !os.IsNotExist(err) {
-			logger.V(0).Info("Failed to remove", "addr", addr, "error", err.Error())
+			logger.V(0).Info("failed to remove", "addr", addr, "error", err.Error())
 		}
 	}
 

pkg/kerberos/kadmin.go

@@ -118,7 +118,7 @@ func (k *Kadmin) Query(query string) (result string, err error) {
 		kadminLogger.Error(err, "Failed to execute kadmin query", "cmd", cmd.String(), "output", result)
 		return "", err
 	}
-	kadminLogger.Info("Executed kadmin query", "cmd", cmd.String(), "output", result)
+	kadminLogger.Info("executed kadmin query", "cmd", cmd.String(), "output", result)
 
 	return result, nil
 
@@ -149,7 +149,7 @@ func (k *Kadmin) Ktadd(principals ...string) ([]byte, error) {
 		return nil, err
 	}
 
-	kadminLogger.Info("Saved keytab", "principal", principals, "keytab", keytab, "output", output)
+	kadminLogger.Info("saved keytab", "principal", principals, "keytab", keytab, "output", output)
 
 	return os.ReadFile(keytab)
 }
@@ -186,7 +186,7 @@ func (k *Kadmin) AddPrincipal(principal string) error {
 		return err
 	}
 
-	kadminLogger.Info("Added principal", "principal", principal, "output", output)
+	kadminLogger.Info("created a new principal", "principal", principal, "output", output)
 
 	return nil
 }

pkg/pod_info/pod_info.go

@@ -76,7 +76,7 @@ func (p *PodInfo) getNodeAddresses(ctx context.Context) ([]Address, error) {
 		}
 	}
 
-	logger.V(1).Info("get node ip filter by internal and external", "pod", p.getPodName(),
+	logger.V(1).Info("got node ip filter by internal and external", "pod", p.getPodName(),
 		"namespace", p.getPodNamespace(), "addresses", addresses)
 	return addresses, nil
 }
@@ -333,6 +333,12 @@ func (p *PodInfo) getListenerAddresses(ctx context.Context) ([]Address, error) {
 		if err != nil {
 			return nil, err
 		}
+
+		// check listener status
+		if len(listener.Status.IngressAddresses) == 0 {
+			return nil, fmt.Errorf("listener %s/%s status not ready", listener.Namespace, listener.Name)
+		}
+
 		for _, ingressAddress := range listener.Status.IngressAddresses {
 			if ingressAddress.AddressType == operatorlistenersv1alpha1.AddressTypeHostname {
 				addresses = append(addresses, Address{

pkg/util/log.go

@@ -14,30 +14,32 @@ var (
 	log = ctrl.Log.WithName("csi-grpc")
 )
 
-func GetLogLevel(method string) int {
+func getLogLevel(method string) int {
+	v := ctrl.Log.GetV()
+
 	if method == "/csi.v1.Identity/Probe" ||
 		method == "/csi.v1.Node/NodeGetCapabilities" ||
 		method == "/csi.v1.Node/NodeGetVolumeStats" {
 		return 8
 	}
-	return 2
+	return v
 }
 
 func LogGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo,
 	handler grpc.UnaryHandler) (interface{}, error) {
-	level := GetLogLevel(info.FullMethod)
-	log.V(level).Info("GRPC calling", "method", info.FullMethod, "request", protosanitizer.StripSecrets(req))
+	level := getLogLevel(info.FullMethod)
+	log.V(level).Info("gRPC calling", "method", info.FullMethod, "request", protosanitizer.StripSecrets(req))
 
 	resp, err := handler(ctx, req)
 	if err != nil {
-		log.Error(err, "GRPC called error", "method", info.FullMethod)
+		log.Error(err, "RPC called error", "method", info.FullMethod)
 		if level >= 5 {
 			stack := debug.Stack()
 			errStack := fmt.Errorf("\n%s", stack)
 			log.Error(err, "GRPC called error", errStack.Error())
 		}
 	} else {
-		log.V(level).Info("GRPC called", "method", info.FullMethod, "response", protosanitizer.StripSecrets(resp))
+		log.V(level).Info("gRPC called", "method", info.FullMethod, "response", protosanitizer.StripSecrets(resp))
 	}
 	return resp, err
 }

pkg/volume/volume.go

@@ -188,7 +188,7 @@ func NewvolumeContextFromMap(parameters map[string]string) (*SecretVolumeContext
 		case VolumeKubernetesStorageProvisioner:
 			v.Provisioner = value
 		case DeprecatedVolumeKubernetesStorageProvisioner:
-			logger.V(0).Info("Deprecated key since v1.23, please use new key",
+			logger.V(0).Info("deprecated key since v1.23, please use new key",
 				"key", key,
 				"value", value,
 				"new key", VolumeKubernetesStorageProvisioner,

test/e2e/krb5/chainsaw-test.yaml

@@ -89,6 +89,14 @@ spec:
         file: node-scope.yaml
     - assert:
         file: node-scope-assert.yaml
+    catch:
+      - script:
+          env:
+            - name: NAMESPACE
+              value: ($namespace)
+          content: |
+            kubectl -n "$NAMESPACE" get pod
+            kubectl -n "$NAMESPACE" describe pod
   - try:
     - apply:
         file: service-scope.yaml