From 0c2a6d54bc193d9fd88f715ecbc3358b35547d84 Mon Sep 17 00:00:00 2001 From: Tasko Olevski Date: Tue, 29 Oct 2024 08:07:38 -0400 Subject: [PATCH] fix: properly deal with saved storage secrets (#37) --- pkg/rclone/controllerserver.go | 31 ++++++++++---- pkg/rclone/nodeserver.go | 14 +++++++ test/sanity_test.go | 74 +++++++++++++++++++++++++++++----- 3 files changed, 101 insertions(+), 18 deletions(-) diff --git a/pkg/rclone/controllerserver.go b/pkg/rclone/controllerserver.go index 331979f..6b5e4fa 100644 --- a/pkg/rclone/controllerserver.go +++ b/pkg/rclone/controllerserver.go @@ -15,6 +15,8 @@ import ( csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common" ) +const secretAnnotationName = "csi-rclone.dev/secretName" + type controllerServer struct { *csicommon.DefaultControllerServer RcloneOps Operations @@ -80,21 +82,34 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol // See https://github.com/kubernetes-csi/external-provisioner/blob/v5.1.0/pkg/controller/controller.go#L75 // on how parameters from the persistent volume are parsed - // We have to pass these into the context so that the node server can use them - secretName, nameFound := req.Parameters["csi.storage.k8s.io/provisioner-secret-name"] - secretNs, nsFound := req.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"] + // We have to pass the secret name and namespace into the context so that the node server can use them + // The external provisioner uses the secret name and namespace but it does not pass them into the request, + // so we read the PVC here to extract them ourselves because we may need them in the node server for decoding secrets. + pvcName, pvcNameFound := req.Parameters["csi.storage.k8s.io/pvc/name"] + pvcNamespace, pvcNamespaceFound := req.Parameters["csi.storage.k8s.io/pvc/namespace"] + if !pvcNameFound || !pvcNamespaceFound { + return nil, status.Error(codes.FailedPrecondition, "The PVC name and/or namespace are not present in the create volume request parameters.") + } volumeContext := map[string]string{} - if nameFound && nsFound { + if len(req.GetSecrets()) > 0 { + pvc, err := getPVC(ctx, pvcNamespace, pvcName) + if err != nil { + return nil, err + } + secretName, secretNameFound := pvc.Annotations[secretAnnotationName] + if !secretNameFound { + return nil, status.Error(codes.FailedPrecondition, "The secret name is not present in the PVC annotations.") + } volumeContext["secretName"] = secretName - volumeContext["secretNamespace"] = secretNs + volumeContext["secretNamespace"] = pvcNamespace } else { // This is here for compatibility reasons before this update the secret name was equal to the PVC - volumeContext["secretName"] = req.Parameters["csi.storage.k8s.io/pvc/name"] - volumeContext["secretNamespace"] = req.Parameters["csi.storage.k8s.io/pvc/namespace"] + volumeContext["secretName"] = pvcName + volumeContext["secretNamespace"] = pvcNamespace } return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ - VolumeId: volumeName, + VolumeId: volumeName, VolumeContext: volumeContext, }, }, nil diff --git a/pkg/rclone/nodeserver.go b/pkg/rclone/nodeserver.go index f7ee950..9929dde 100644 --- a/pkg/rclone/nodeserver.go +++ b/pkg/rclone/nodeserver.go @@ -162,6 +162,20 @@ func getSecret(ctx context.Context, namespace, name string) (*v1.Secret, error) return cs.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{}) } +func getPVC(ctx context.Context, namespace, name string) (*v1.PersistentVolumeClaim, error) { + cs, err := kube.GetK8sClient() + if err != nil { + return nil, err + } + if namespace == "" { + return nil, fmt.Errorf("Failed to read PVC with K8s client because namespace is blank") + } + if name == "" { + return nil, fmt.Errorf("Failed to read PVC with K8s client because name is blank") + } + return cs.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, name, metav1.GetOptions{}) +} + func validatePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error { if req.GetVolumeId() == "" { return status.Error(codes.InvalidArgument, "empty volume id") diff --git a/test/sanity_test.go b/test/sanity_test.go index 7ccdbf0..6b91469 100644 --- a/test/sanity_test.go +++ b/test/sanity_test.go @@ -16,6 +16,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -170,14 +171,42 @@ provider=AWS`}, cfg.SecretsFile = "testdata/secrets.yaml" cfg.TargetPath = mntDir cfg.StagingPath = stageDir + kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-secret", Namespace: "csi-rclone"}, + StringData: map[string]string{ + "remote": "my-s3", + "remotePath": "giab/", + "secretKey": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4=", + "configData": `[my-s3] +type= +provider=AWS`}, + Type: "Opaque", + }, metav1.CreateOptions{}) + className := "csi-rclone-secret-annotation" + kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Create(context.Background(), &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-pvc", Namespace: "csi-rclone", + Annotations: map[string]string{"csi-rclone.dev/secretName": "pvc-secret"}, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{"storage": resource.MustParse("1Gi")}, + }, + StorageClassName: &className, + }, + }, metav1.CreateOptions{}) cfg.TestVolumeParameters = map[string]string{ - "csi.storage.k8s.io/pvc/namespace": "csi-rclone", - "csi.storage.k8s.io/pvc/name": "some-pvc", - "csi.storage.k8s.io/node-publish-secret-namespace": "csi-rclone", - "csi.storage.k8s.io/node-publish-secret-name": "test-pvc", + "csi.storage.k8s.io/pvc/namespace": "csi-rclone", + "csi.storage.k8s.io/pvc/name": "some-pvc", } }) + AfterEach(func() { + kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "pvc-secret", metav1.DeleteOptions{}) + kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Delete(context.Background(), "some-pvc", metav1.DeleteOptions{}) + }) + AfterAll(func() { testCtx.Finalize() }) @@ -194,10 +223,35 @@ provider=AWS`}, BeforeEach(func() { mntDir, stageDir := getMountDirs() kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "test-pvc-secrets", Namespace: "csi-rclone"}, + ObjectMeta: metav1.ObjectMeta{Name: "pvc-secret", Namespace: "csi-rclone"}, + StringData: map[string]string{ + "remote": "my-s3", + "remotePath": "giab/", + "secretKey": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4=", + "configData": `[my-s3] +type= +provider=AWS`}, + Type: "Opaque", + }, metav1.CreateOptions{}) + kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "pvc-secret-secrets", Namespace: "csi-rclone"}, StringData: map[string]string{"type": "gAAAAABK-fBwYcjuQgctfZknI2ko2uLqj6DRzRa7kFTKnWm_nkjwGWGTai5eyhNXlp6_6QjeTC7B8IWvhBsvG1Q6Zk2eDYDVQg=="}, Type: "Opaque", }, metav1.CreateOptions{}) + className := "csi-rclone-secret-annotation" + kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Create(context.Background(), &v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-pvc", Namespace: "csi-rclone", + Annotations: map[string]string{"csi-rclone.dev/secretName": "pvc-secret"}, + }, + Spec: v1.PersistentVolumeClaimSpec{ + AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce}, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{"storage": resource.MustParse("1Gi")}, + }, + StorageClassName: &className, + }, + }, metav1.CreateOptions{}) *cfg = sanity.NewTestConfig() cfg.Address = endpoint @@ -205,15 +259,15 @@ provider=AWS`}, cfg.TargetPath = mntDir cfg.StagingPath = stageDir cfg.TestVolumeParameters = map[string]string{ - "csi.storage.k8s.io/pvc/namespace": "csi-rclone", - "csi.storage.k8s.io/pvc/name": "some-pvc", - "csi.storage.k8s.io/node-publish-secret-namespace": "csi-rclone", - "csi.storage.k8s.io/node-publish-secret-name": "test-pvc", + "csi.storage.k8s.io/pvc/namespace": "csi-rclone", + "csi.storage.k8s.io/pvc/name": "some-pvc", } }) AfterEach(func() { - kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "test-pvc-secrets", metav1.DeleteOptions{}) + kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "pvc-secret", metav1.DeleteOptions{}) + kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "pvc-secret-secrets", metav1.DeleteOptions{}) + kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Delete(context.Background(), "some-pvc", metav1.DeleteOptions{}) }) AfterAll(func() {