Skip to content

Commit

Permalink
fix: properly deal with saved storage secrets (#37)
Browse files Browse the repository at this point in the history
  • Loading branch information
@olevski
olevski authored Oct 29, 2024
1 parent 14d554c commit 0c2a6d5
Show file tree
Hide file tree
Showing 3 changed files with 101 additions and 18 deletions.
31 changes: 23 additions & 8 deletions pkg/rclone/controllerserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import (
csicommon "github.com/kubernetes-csi/drivers/pkg/csi-common"
)

const secretAnnotationName = "csi-rclone.dev/secretName"

type controllerServer struct {
*csicommon.DefaultControllerServer
RcloneOps Operations
Expand Down Expand Up @@ -80,21 +82,34 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol

// See https://github.com/kubernetes-csi/external-provisioner/blob/v5.1.0/pkg/controller/controller.go#L75
// on how parameters from the persistent volume are parsed
// We have to pass these into the context so that the node server can use them
secretName, nameFound := req.Parameters["csi.storage.k8s.io/provisioner-secret-name"]
secretNs, nsFound := req.Parameters["csi.storage.k8s.io/provisioner-secret-namespace"]
// We have to pass the secret name and namespace into the context so that the node server can use them
// The external provisioner uses the secret name and namespace but it does not pass them into the request,
// so we read the PVC here to extract them ourselves because we may need them in the node server for decoding secrets.
pvcName, pvcNameFound := req.Parameters["csi.storage.k8s.io/pvc/name"]
pvcNamespace, pvcNamespaceFound := req.Parameters["csi.storage.k8s.io/pvc/namespace"]
if !pvcNameFound || !pvcNamespaceFound {
return nil, status.Error(codes.FailedPrecondition, "The PVC name and/or namespace are not present in the create volume request parameters.")
}
volumeContext := map[string]string{}
if nameFound && nsFound {
if len(req.GetSecrets()) > 0 {
pvc, err := getPVC(ctx, pvcNamespace, pvcName)
if err != nil {
return nil, err
}
secretName, secretNameFound := pvc.Annotations[secretAnnotationName]
if !secretNameFound {
return nil, status.Error(codes.FailedPrecondition, "The secret name is not present in the PVC annotations.")
}
volumeContext["secretName"] = secretName
volumeContext["secretNamespace"] = secretNs
volumeContext["secretNamespace"] = pvcNamespace
} else {
// This is here for compatibility reasons before this update the secret name was equal to the PVC
volumeContext["secretName"] = req.Parameters["csi.storage.k8s.io/pvc/name"]
volumeContext["secretNamespace"] = req.Parameters["csi.storage.k8s.io/pvc/namespace"]
volumeContext["secretName"] = pvcName
volumeContext["secretNamespace"] = pvcNamespace
}
return &csi.CreateVolumeResponse{
Volume: &csi.Volume{
VolumeId: volumeName,
VolumeId: volumeName,
VolumeContext: volumeContext,
},
}, nil
Expand Down
14 changes: 14 additions & 0 deletions pkg/rclone/nodeserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,20 @@ func getSecret(ctx context.Context, namespace, name string) (*v1.Secret, error)
return cs.CoreV1().Secrets(namespace).Get(ctx, name, metav1.GetOptions{})
}

func getPVC(ctx context.Context, namespace, name string) (*v1.PersistentVolumeClaim, error) {
cs, err := kube.GetK8sClient()
if err != nil {
return nil, err
}
if namespace == "" {
return nil, fmt.Errorf("Failed to read PVC with K8s client because namespace is blank")
}
if name == "" {
return nil, fmt.Errorf("Failed to read PVC with K8s client because name is blank")
}
return cs.CoreV1().PersistentVolumeClaims(namespace).Get(ctx, name, metav1.GetOptions{})
}

func validatePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error {
if req.GetVolumeId() == "" {
return status.Error(codes.InvalidArgument, "empty volume id")
Expand Down
74 changes: 64 additions & 10 deletions test/sanity_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
)
Expand Down Expand Up @@ -170,14 +171,42 @@ provider=AWS`},
cfg.SecretsFile = "testdata/secrets.yaml"
cfg.TargetPath = mntDir
cfg.StagingPath = stageDir
kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "pvc-secret", Namespace: "csi-rclone"},
StringData: map[string]string{
"remote": "my-s3",
"remotePath": "giab/",
"secretKey": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4=",
"configData": `[my-s3]
type=<sensitive>
provider=AWS`},
Type: "Opaque",
}, metav1.CreateOptions{})
className := "csi-rclone-secret-annotation"
kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Create(context.Background(), &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "some-pvc", Namespace: "csi-rclone",
Annotations: map[string]string{"csi-rclone.dev/secretName": "pvc-secret"},
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{"storage": resource.MustParse("1Gi")},
},
StorageClassName: &className,
},
}, metav1.CreateOptions{})
cfg.TestVolumeParameters = map[string]string{
"csi.storage.k8s.io/pvc/namespace": "csi-rclone",
"csi.storage.k8s.io/pvc/name": "some-pvc",
"csi.storage.k8s.io/node-publish-secret-namespace": "csi-rclone",
"csi.storage.k8s.io/node-publish-secret-name": "test-pvc",
"csi.storage.k8s.io/pvc/namespace": "csi-rclone",
"csi.storage.k8s.io/pvc/name": "some-pvc",
}
})

AfterEach(func() {
kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "pvc-secret", metav1.DeleteOptions{})
kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Delete(context.Background(), "some-pvc", metav1.DeleteOptions{})
})

AfterAll(func() {
testCtx.Finalize()
})
Expand All @@ -194,26 +223,51 @@ provider=AWS`},
BeforeEach(func() {
mntDir, stageDir := getMountDirs()
kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "test-pvc-secrets", Namespace: "csi-rclone"},
ObjectMeta: metav1.ObjectMeta{Name: "pvc-secret", Namespace: "csi-rclone"},
StringData: map[string]string{
"remote": "my-s3",
"remotePath": "giab/",
"secretKey": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4=",
"configData": `[my-s3]
type=<sensitive>
provider=AWS`},
Type: "Opaque",
}, metav1.CreateOptions{})
kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{
ObjectMeta: metav1.ObjectMeta{Name: "pvc-secret-secrets", Namespace: "csi-rclone"},
StringData: map[string]string{"type": "gAAAAABK-fBwYcjuQgctfZknI2ko2uLqj6DRzRa7kFTKnWm_nkjwGWGTai5eyhNXlp6_6QjeTC7B8IWvhBsvG1Q6Zk2eDYDVQg=="},
Type: "Opaque",
}, metav1.CreateOptions{})
className := "csi-rclone-secret-annotation"
kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Create(context.Background(), &v1.PersistentVolumeClaim{
ObjectMeta: metav1.ObjectMeta{
Name: "some-pvc", Namespace: "csi-rclone",
Annotations: map[string]string{"csi-rclone.dev/secretName": "pvc-secret"},
},
Spec: v1.PersistentVolumeClaimSpec{
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
Resources: v1.ResourceRequirements{
Requests: v1.ResourceList{"storage": resource.MustParse("1Gi")},
},
StorageClassName: &className,
},
}, metav1.CreateOptions{})

*cfg = sanity.NewTestConfig()
cfg.Address = endpoint
cfg.SecretsFile = "testdata/secrets.yaml"
cfg.TargetPath = mntDir
cfg.StagingPath = stageDir
cfg.TestVolumeParameters = map[string]string{
"csi.storage.k8s.io/pvc/namespace": "csi-rclone",
"csi.storage.k8s.io/pvc/name": "some-pvc",
"csi.storage.k8s.io/node-publish-secret-namespace": "csi-rclone",
"csi.storage.k8s.io/node-publish-secret-name": "test-pvc",
"csi.storage.k8s.io/pvc/namespace": "csi-rclone",
"csi.storage.k8s.io/pvc/name": "some-pvc",
}
})

AfterEach(func() {
kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "test-pvc-secrets", metav1.DeleteOptions{})
kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "pvc-secret", metav1.DeleteOptions{})
kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "pvc-secret-secrets", metav1.DeleteOptions{})
kubeClient.CoreV1().PersistentVolumeClaims("csi-rclone").Delete(context.Background(), "some-pvc", metav1.DeleteOptions{})
})

AfterAll(func() {
Expand Down

0 comments on commit 0c2a6d5

Please sign in to comment.