Swift uploader (#10)

* add swift-uploader based on Lucretius#19

README.md

@@ -383,10 +383,15 @@ uploaders:
     accountKey: <key>
     container: <container>
     cloudDomain: <domain>
-  google:
+  gcp:
     bucket: <bucket>
   local:
     path: <path>
+  swift:
+    container: <container>
+    username: <username>
+    apiKey: <api-key>
+    authUrl: <auth-url>
 ```
 
 Note that if you specify more than one storage option, *all* options will be written to.  For example, specifying `local` and `aws` will write to both locations. Each options can be specified exactly once - thus is is currently not possible to e.g. upload to multiple aws regions by specifying multiple `aws`-entries.
@@ -420,13 +425,24 @@ uploaders:
 - `cloudDomain` *(default: blob.core.windows.net) - domain of the cloud-service to use
 
 
-#### Google Storage
+#### Google Cloud Storage
 `bucket` **(required)** - the Google Storage Bucket to write to.  Auth is expected to be default machine credentials.
 
 
 #### Local Storage
 `path` **(required)** - fully qualified path, not including file name, for where the snapshot should be written.  i.e. `/raft/snapshots`
 
+#### Openstack Swift Storage
+- `container` **(required)** - the name of the container to write to
+- `username` **(required)** - the username used for authentication
+- `apiKey` **(required)** - the api-key used for authentication
+- `authUrl` **(required)** - the auth-url to authenicate against
+- `region` - optional region to use eg "LON", "ORD"
+- `domain` - optional user's domain name
+- `tenantId` - optional id of the tenant
+- `timeout` *(default: 60s)** - timeout for snapshot-uploads
+
+
 
 ## License
 - Source code is licensed under MIT
@@ -435,4 +451,5 @@ uploaders:
 - Vault Raft Snapshot Agent was originally developed by [@Lucretius](https://github.com/Lucretius/vault_raft_snapshot_agent/)
 - contains improvements done by [@Boostport](https://github.com/Boostport/vault_raft_snapshot_agent/)
 - enhancements for azure-uploader by [@vikramhansawat](https://github.com/Lucretius/vault_raft_snapshot_agent/pull/26)
-- support for additional authentication methods based on code from [@alexeiser](https://github.com/Lucretius/vault_raft_snapshot_agent/pull/25)
+- support for additional authentication methods based on code from [@alexeiser](https://github.com/Lucretius/vault_raft_snapshot_agent/pull/25)
+- support for Openstack Swift Storage based on code from [@Pyjou](https://github.com/Lucretius/vault_raft_snapshot_agent/pull/19)

cmd/vault-raft-snapshot-agent/main.go

@@ -105,15 +105,15 @@ Options:
 }
 
 func startSnapshotter(configFile cli.Path) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	snapshotterOptions.ConfigFilePath = configFile
-	snapshotter, err := internal.CreateSnapshotter(snapshotterOptions)
+	snapshotter, err := internal.CreateSnapshotter(ctx, snapshotterOptions)
 	if err != nil {
 		log.Fatalf("Cannot create snapshotter: %s\n", err)
 	}
 
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 	go func() {

go.mod

@@ -31,6 +31,9 @@ require (
 	google.golang.org/api v0.140.0
 )
 
+// Swift-Uploader
+require github.com/ncw/swift/v2 v2.0.2
+
 // Vault
 require (
 	github.com/hashicorp/vault/api v1.10.0

go.sum

@@ -392,6 +392,8 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk=
+github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
 github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
 github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
 github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 h1:Qj1ukM4GlMWXNdMBuXcXfz/Kw9s1qm0CLY32QxuSImI=

internal/app/vault_raft_snapshot_agent/snapshotter.go

@@ -54,23 +54,23 @@ type snapshotterVaultAPI interface {
 	TakeSnapshot(ctx context.Context, writer io.Writer) error
 }
 
-func CreateSnapshotter(options SnapshotterOptions) (*Snapshotter, error) {
-	c := SnapshotterConfig{}
+func CreateSnapshotter(ctx context.Context, options SnapshotterOptions) (*Snapshotter, error) {
+	data := SnapshotterConfig{}
 	parser := config.NewParser[*SnapshotterConfig](options.ConfigFileName, options.EnvPrefix, options.ConfigFileSearchPaths...)
 
-	if err := parser.ReadConfig(&c, options.ConfigFilePath); err != nil {
+	if err := parser.ReadConfig(&data, options.ConfigFilePath); err != nil {
 		return nil, err
 	}
 
-	snapshotter, err := createSnapshotter(c)
+	snapshotter, err := createSnapshotter(ctx, data)
 	if err != nil {
 		return nil, err
 	}
 
 	parser.OnConfigChange(
 		&SnapshotterConfig{},
 		func(config *SnapshotterConfig) error {
-			if err := snapshotter.reconfigure(*config); err != nil {
+			if err := snapshotter.reconfigure(ctx, *config); err != nil {
 				log.Printf("could not reconfigure snapshotter: %s\n", err)
 				return err
 			}
@@ -81,20 +81,20 @@ func CreateSnapshotter(options SnapshotterOptions) (*Snapshotter, error) {
 	return snapshotter, nil
 }
 
-func createSnapshotter(config SnapshotterConfig) (*Snapshotter, error) {
+func createSnapshotter(ctx context.Context, config SnapshotterConfig) (*Snapshotter, error) {
 	snapshotter := &Snapshotter{}
 
-	err := snapshotter.reconfigure(config)
+	err := snapshotter.reconfigure(ctx, config)
 	return snapshotter, err
 }
 
-func (s *Snapshotter) reconfigure(config SnapshotterConfig) error {
+func (s *Snapshotter) reconfigure(ctx context.Context, config SnapshotterConfig) error {
 	client, err := vault.CreateVaultClient(config.Vault)
 	if err != nil {
 		return err
 	}
 
-	uploaders, err := upload.CreateUploaders(config.Uploaders)
+	uploaders, err := upload.CreateUploaders(ctx, config.Uploaders)
 	if err != nil {
 		return err
 	}

internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go

@@ -101,30 +101,40 @@ func TestReadCompleteConfig(t *testing.T) {
 			TimestampFormat: "2006-01-02",
 		},
 		Uploaders: upload.UploadersConfig{
-			AWS: upload.AWSConfig{
+			AWS: upload.AWSUploaderConfig{
 				Endpoint:                "test-endpoint",
 				Region:                  "test-region",
 				Bucket:                  "test-bucket",
 				KeyPrefix:               "test-prefix",
 				UseServerSideEncryption: true,
 				ForcePathStyle:          true,
-				Credentials: upload.AWSCredentialsConfig{
+				Credentials: upload.AWSUploaderCredentialsConfig{
 					Key:    "test-key",
 					Secret: "test-secret",
 				},
 			},
-			Azure: upload.AzureConfig{
+			Azure: upload.AzureUploaderConfig{
 				AccountName:   "test-account",
 				AccountKey:    "test-key",
 				ContainerName: "test-container",
 				CloudDomain:   "blob.core.chinacloudapi.cn",
 			},
-			GCP: upload.GCPConfig{
+			GCP: upload.GCPUploaderConfig{
 				Bucket: "test-bucket",
 			},
-			Local: upload.LocalConfig{
+			Local: upload.LocalUploaderConfig{
 				Path: ".",
 			},
+			Swift: upload.SwiftUploaderConfig{
+				Container: "test-container",
+				UserName:  "test-username",
+				ApiKey:    "test-api-key",
+				AuthUrl:   "http://auth.com",
+				Domain:    "http://user.com",
+				Region:    "test-region",
+				TenantId:  "test-tenant",
+				Timeout:   180 * time.Second,
+			},
 		},
 	}
 
@@ -186,18 +196,22 @@ func TestReadConfigSetsDefaultValues(t *testing.T) {
 			TimestampFormat: "2006-01-02T15-04-05Z-0700",
 		},
 		Uploaders: upload.UploadersConfig{
-			AWS: upload.AWSConfig{
-				Credentials: upload.AWSCredentialsConfig{Empty: true},
+			AWS: upload.AWSUploaderConfig{
+				Credentials: upload.AWSUploaderCredentialsConfig{Empty: true},
 				Empty:       true,
 			},
-			Azure: upload.AzureConfig{
+			Azure: upload.AzureUploaderConfig{
 				CloudDomain: "blob.core.windows.net",
 				Empty:       true,
 			},
-			GCP: upload.GCPConfig{Empty: true},
-			Local: upload.LocalConfig{
+			GCP: upload.GCPUploaderConfig{Empty: true},
+			Local: upload.LocalUploaderConfig{
 				Path: ".",
 			},
+			Swift: upload.SwiftUploaderConfig{
+				Timeout: time.Minute,
+				Empty:       true,
+			},
 		},
 	}
 

internal/app/vault_raft_snapshot_agent/upload/aws.go

@@ -14,18 +14,18 @@ import (
 	s3Types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 )
 
-type AWSConfig struct {
-	Credentials             AWSCredentialsConfig `default:"{\"Empty\": true}"`
-	Bucket                  string               `validate:"required_if=Empty false"`
-	KeyPrefix               string               `mapstructure:",omitifempty"`
-	Endpoint                string               `mapstructure:",omitifempty"`
+type AWSUploaderConfig struct {
+	Credentials             AWSUploaderCredentialsConfig `default:"{\"Empty\": true}"`
+	Bucket                  string                       `validate:"required_if=Empty false"`
+	KeyPrefix               string                       `mapstructure:",omitifempty"`
+	Endpoint                string                       `mapstructure:",omitifempty"`
 	Region                  string
 	UseServerSideEncryption bool
 	ForcePathStyle          bool
 	Empty                   bool
 }
 
-type AWSCredentialsConfig struct {
+type AWSUploaderCredentialsConfig struct {
 	Key    string `validate:"required_if=Empty false"`
 	Secret string `validate:"required_if=Empty false"`
 	Empty  bool
@@ -39,8 +39,8 @@ type awsUploaderImpl struct {
 	sse       bool
 }
 
-func createAWSUploader(config AWSConfig) (*uploader[s3Types.Object], error) {
-	clientConfig, err := awsConfig.LoadDefaultConfig(context.Background(), awsConfig.WithRegion(config.Region))
+func createAWSUploader(ctx context.Context, config AWSUploaderConfig) (*uploader[s3Types.Object], error) {
+	clientConfig, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithRegion(config.Region))
 
 	if err != nil {
 		return nil, fmt.Errorf("failed to load default aws config: %w", err)

internal/app/vault_raft_snapshot_agent/upload/azure.go

@@ -9,7 +9,7 @@ import (
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container"
 )
 
-type AzureConfig struct {
+type AzureUploaderConfig struct {
 	AccountName   string `validate:"required_if=Empty false"`
 	AccountKey    string `validate:"required_if=Empty false"`
 	ContainerName string `mapstructure:"container" validate:"required_if=Empty false"`
@@ -22,7 +22,7 @@ type azureUploaderImpl struct {
 	container string
 }
 
-func createAzureUploader(config AzureConfig) (*uploader[*container.BlobItem], error) {
+func createAzureUploader(ctx context.Context, config AzureUploaderConfig) (*uploader[*container.BlobItem], error) {
 	credential, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey)
 	if err != nil {
 		return nil, fmt.Errorf("invalid credentials for azure: %w", err)

internal/app/vault_raft_snapshot_agent/upload/gcp.go

@@ -9,7 +9,7 @@ import (
 	"google.golang.org/api/iterator"
 )
 
-type GCPConfig struct {
+type GCPUploaderConfig struct {
 	Bucket string `validate:"required_if=Empty false"`
 	Empty  bool
 }
@@ -19,8 +19,7 @@ type gcpUploaderImpl struct {
 	bucket      *storage.BucketHandle
 }
 
-func createGCPUploader(config GCPConfig) (*uploader[storage.ObjectAttrs], error) {
-	ctx := context.Background()
+func createGCPUploader(ctx context.Context, config GCPUploaderConfig) (*uploader[storage.ObjectAttrs], error) {
 	client, err := storage.NewClient(ctx)
 	if err != nil {
 		return nil, err
@@ -42,7 +41,7 @@ func (u gcpUploaderImpl) Destination() string {
 // implements interface uploaderImpl
 func (u gcpUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error {
 	obj := u.bucket.Object(name)
-	w := obj.NewWriter(context.Background())
+	w := obj.NewWriter(ctx)
 
 	if _, err := io.Copy(w, data); err != nil {
 		return err

internal/app/vault_raft_snapshot_agent/upload/local.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-type LocalConfig struct {
+type LocalUploaderConfig struct {
 	Path  string `validate:"required_if=Empty false,omitempty,dir"`
 	Empty bool
 }
@@ -17,7 +17,7 @@ type localUploaderImpl struct {
 	path string
 }
 
-func createLocalUploader(config LocalConfig) (uploader[os.FileInfo], error) {
+func createLocalUploader(ctx context.Context, config LocalUploaderConfig) (uploader[os.FileInfo], error) {
 	return uploader[os.FileInfo]{
 		localUploaderImpl{
 			path: config.Path,