diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5cd6fe1..3af9c5c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -36,7 +36,38 @@ jobs: init-kind-cluster local-deploy get-kind-kubeconfig - go test -v ./... + go test -v test/sanity_test.go + - name: Print rclone log + if: ${{ failure() }} + run: cat /tmp/rclone.log + tests-with-decryption: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install fuse + run: | + sudo apt-get update + sudo apt-get install -y fuse3 + sudo bash -c 'echo "user_allow_other" >> /etc/fuse.conf' + - uses: actions/setup-go@v4 + with: + go-version: '1.20' + - uses: cachix/install-nix-action@v22 + with: + nix_path: nixpkgs=channel:nixos-unstable + - name: Flake check + run: nix flake check + - name: Helm check + run: helm lint deploy/csi-rclone + - name: Run tests with secret decryption + uses: workflow/nix-shell-action@v3.3.0 + with: + flakes-from-devshell: true + script: | + init-kind-cluster + local-deploy + get-kind-kubeconfig + go test -v test/sanity_with_decrypt_test.go - name: Print rclone log if: ${{ failure() }} run: cat /tmp/rclone.log diff --git a/devenv/nix/goModule.nix b/devenv/nix/goModule.nix index ea48220..b8a6bbd 100644 --- a/devenv/nix/goModule.nix +++ b/devenv/nix/goModule.nix @@ -5,7 +5,7 @@ let pname = "csi-rclone-pvc-1"; version = "0.1.7"; src = ../../.; - vendorHash = "sha256-XY0XgDky2g7DQ210VsT+KKjyYL1EJPCNGP0F5GhY2gM="; + vendorHash = "sha256-q1tfnO5B6U9c+Ve+kpOfnWGvbdShgkPXvR7axsA7O5Y="; # CGO = 0; # preBuild = '' # whoami diff --git a/go.mod b/go.mod index b0f6cd6..0f79ef0 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/gogo/protobuf v1.3.1 // indirect diff --git a/go.sum b/go.sum index 6501778..29426f9 100644 --- a/go.sum +++ b/go.sum @@ -151,6 +151,8 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611 h1:JwYtKJ/DVEoIA5dH45OEU7uoryZY/gjd/BQiwwAOImM= +github.com/fernet/fernet-go v0.0.0-20240119011108-303da6aec611/go.mod h1:zHMNeYgqrTpKyjawjitDg0Osd1P/FmeA0SZLYK3RfLQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= diff --git a/pkg/rclone/controllerserver.go b/pkg/rclone/controllerserver.go index b213bc3..2a2464e 100644 --- a/pkg/rclone/controllerserver.go +++ b/pkg/rclone/controllerserver.go @@ -88,7 +88,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol if err != nil { return nil, err } - remote, remotePath, _, _, err := extractFlags(req.GetParameters(), req.GetSecrets(), pvcSecret) + remote, remotePath, _, _, err := extractFlags(req.GetParameters(), req.GetSecrets(), pvcSecret, nil) if err != nil { return nil, status.Errorf(codes.InvalidArgument, "CreateVolume: %v", err) } diff --git a/pkg/rclone/nodeserver.go b/pkg/rclone/nodeserver.go index 9819259..8723d82 100644 --- a/pkg/rclone/nodeserver.go +++ b/pkg/rclone/nodeserver.go @@ -6,17 +6,21 @@ package rclone // Follow lifecycle import ( + "bytes" "errors" + "fmt" "os" "strings" "time" + "gopkg.in/ini.v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" "github.com/SwissDataScienceCenter/csi-rclone/pkg/kube" "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/fernet/fernet-go" "golang.org/x/net/context" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -59,7 +63,15 @@ func (ns *nodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublis if err != nil { return nil, err } - remote, remotePath, configData, flags, e := extractFlags(req.GetVolumeContext(), req.GetSecrets(), pvcSecret) + + savedSecretName := secretName + "-secrets" + + savedPvcSecret, err := GetPvcSecret(ctx, namespace, savedSecretName) + if err != nil { + klog.Warningf("Cannot find saved secrets %s: %s", savedSecretName, err) + } + + remote, remotePath, configData, flags, e := extractFlags(req.GetVolumeContext(), req.GetSecrets(), pvcSecret, savedPvcSecret) delete(flags, "secretName") delete(flags, "namespace") if e != nil { @@ -142,7 +154,7 @@ func validatePublishVolumeRequest(req *csi.NodePublishVolumeRequest) error { return nil } -func extractFlags(volumeContext map[string]string, secret map[string]string, pvcSecret *v1.Secret) (string, string, string, map[string]string, error) { +func extractFlags(volumeContext map[string]string, secret map[string]string, pvcSecret *v1.Secret, savedPvcSecret *v1.Secret) (string, string, string, map[string]string, error) { // Empty argument list flags := make(map[string]string) @@ -187,9 +199,59 @@ func extractFlags(volumeContext map[string]string, secret map[string]string, pvc configData, flags := extractConfigData(flags) + if savedPvcSecret != nil { + if savedSecrets, err := decryptSecrets(flags, savedPvcSecret); err != nil { + klog.Errorf("cannot decode saved storage secrets: %s", err) + } else { + if modifiedConfigData, err := updateConfigData(remote, configData, savedSecrets); err == nil { + configData = modifiedConfigData + } else { + klog.Errorf("cannot update config data: %s", err) + } + } + } + return remote, remotePath, configData, flags, nil } +func decryptSecrets(flags map[string]string, savedPvcSecret *v1.Secret) (map[string]string, error) { + savedSecrets := make(map[string]string) + + userSecretKey, ok := flags["secretKey"] + if !ok { + return savedSecrets, status.Error(codes.InvalidArgument, "missing user secret key") + } + fernetKey, err := fernet.DecodeKey(userSecretKey) + if err != nil { + return savedSecrets, status.Errorf(codes.InvalidArgument, "cannot decode user secret key: %s", err) + } + + if len(savedPvcSecret.Data) > 0 { + for k, v := range savedPvcSecret.Data { + savedSecrets[k] = string(fernet.VerifyAndDecrypt([]byte(v), 0, []*fernet.Key{fernetKey})) + } + } + + return savedSecrets, nil +} + +func updateConfigData(remote string, configData string, savedSecrets map[string]string) (string, error) { + iniData, err := ini.Load([]byte(configData)) + if err != nil { + return "", fmt.Errorf("cannot load ini config data: %s", err) + } + + section := iniData.Section(remote) + for k, v := range savedSecrets { + section.Key(k).SetValue(v) + } + + buf := new(bytes.Buffer) + iniData.WriteTo(buf) + + return buf.String(), nil +} + func validateFlags(flags map[string]string) error { if _, ok := flags["remote"]; !ok { return status.Errorf(codes.InvalidArgument, "missing volume context value: remote") diff --git a/pkg/rclone/rclone.go b/pkg/rclone/rclone.go index 6cf6dad..52105fb 100644 --- a/pkg/rclone/rclone.go +++ b/pkg/rclone/rclone.go @@ -281,7 +281,7 @@ func (r Rclone) GetVolumeById(ctx context.Context, volumeId string) (*RcloneVolu if err != nil { return nil, err } - remote, path, _, _, err = extractFlags(pv.Spec.CSI.VolumeAttributes, secrets, pvcSecret) + remote, path, _, _, err = extractFlags(pv.Spec.CSI.VolumeAttributes, secrets, pvcSecret, nil) if err != nil { return nil, err } diff --git a/test/sanity_test.go b/test/sanity_test.go index ffefab8..5999528 100644 --- a/test/sanity_test.go +++ b/test/sanity_test.go @@ -56,6 +56,7 @@ func TestMyDriver(t *testing.T) { StringData: map[string]string{ "remote": "my-s3", "remotePath": "giab/", + "secretKey": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4=", "configData": `[my-s3] type=s3 provider=AWS`}, diff --git a/test/sanity_with_decrypt_test.go b/test/sanity_with_decrypt_test.go new file mode 100644 index 0000000..6200299 --- /dev/null +++ b/test/sanity_with_decrypt_test.go @@ -0,0 +1,91 @@ +package test + +import ( + "context" + "os" + "testing" + + "github.com/SwissDataScienceCenter/csi-rclone/pkg/kube" + "github.com/SwissDataScienceCenter/csi-rclone/pkg/rclone" + "github.com/kubernetes-csi/csi-test/v5/pkg/sanity" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestMyDriverWithDecryption(t *testing.T) { + // Setup the full driver and its environment + endpoint := "unix:///tmp/plugin/csi.sock" + kubeClient, err := kube.GetK8sClient() + if err != nil { + panic(err) + } + os.Setenv("DRIVER_NAME", "csi-rclone") + driver := rclone.NewDriver("hostname", endpoint, kubeClient) + go driver.Run() + err = os.MkdirAll("/tmp/sanity/mount/", 0700) + if err != nil { + t.Fatal(err) + } + err = os.MkdirAll("/tmp/sanity/stage/", 0700) + if err != nil { + t.Fatal(err) + } + err = os.MkdirAll("/tmp/plugin/", 0700) + if err != nil { + t.Fatal(err) + } + + mntDir, err := os.MkdirTemp("/tmp/sanity/mount/", "mount") + if err != nil { + t.Fatal(err) + } + os.RemoveAll(mntDir) + //defer os.RemoveAll(mntDir) + + mntStageDir, err := os.MkdirTemp("/tmp/sanity/stage/", "stage") + if err != nil { + t.Fatal(err) + } + os.Getwd() + os.RemoveAll(mntStageDir) + //defer os.RemoveAll(mntStageDir) + + // create secret containing storage config for use in the test + kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pvc", Namespace: "csi-rclone"}, + StringData: map[string]string{ + "remote": "my-s3", + "remotePath": "giab/", + "secretKey": "cw_0x689RpI-jtRR7oE8h_eQsKImvJapLeSbXpwF4e4=", + "configData": `[my-s3] +type= +provider=AWS`}, + Type: "Opaque", + }, metav1.CreateOptions{}) + + // create secret containing saved storage secrets. `type` which is `s3` is encrypted like a secret + // if decryption fails, then the storage cannot be mounted + kubeClient.CoreV1().Secrets("csi-rclone").Create(context.Background(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "test-pvc-secrets", Namespace: "csi-rclone"}, + StringData: map[string]string{"type": "gAAAAABK-fBwYcjuQgctfZknI2ko2uLqj6DRzRa7kFTKnWm_nkjwGWGTai5eyhNXlp6_6QjeTC7B8IWvhBsvG1Q6Zk2eDYDVQg=="}, + Type: "Opaque", + }, metav1.CreateOptions{}) + + cfg := sanity.NewTestConfig() + cfg.Address = endpoint + + cfg.TargetPath = mntDir + cfg.StagingPath = mntStageDir + cfg.Address = endpoint + // cfg.SecretsFile = "testdata/secrets.yaml" + cfg.TestVolumeParameters = map[string]string{ + "csi.storage.k8s.io/pvc/namespace": "csi-rclone", + "csi.storage.k8s.io/pvc/name": "test-pvc", + } + sanity.Test(t, cfg) + + // sanity just completely kills the driver, leaking the rclone daemon, so we cleanup manually + driver.RcloneOps.Cleanup() + + kubeClient.CoreV1().Secrets("csi-rclone").Delete(context.Background(), "test-pvc", metav1.DeleteOptions{}) +}