Added Openstack Swift support

config/config.go

@@ -18,6 +18,7 @@ type Configuration struct {
 	Local           LocalConfig `json:"local_storage"`
 	GCP             GCPConfig   `json:"google_storage"`
 	Azure           AzureConfig `json:"azure_storage"`
+	Swift           SwiftConfig `json:"swift_storage"`
 	RoleID          string      `json:"role_id"`
 	SecretID        string      `json:"secret_id"`
 	Approle         string      `json:"approle"`
@@ -57,6 +58,17 @@ type S3Config struct {
 	S3ForcePathStyle   bool   `json:"s3_force_path_style"`
 }
 
+// Openstack Swift configuration
+type SwiftConfig struct {
+	UserName  string `json:"username"`
+	ApiKey    string `json:"api_key"`
+	AuthUrl   string `json:"auth_url"`
+	Region    string `json:"region"`
+	Container string `json:"container"`
+	TenantId  string `json:"tenant_id"`
+	Domain    string `json:"domain"`
+}
+
 // ReadConfig reads the configuration file
 func ReadConfig() (*Configuration, error) {
 	file := "/etc/vault.d/snapshot.json"

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
 	github.com/hashicorp/raft v1.1.2
 	github.com/hashicorp/vault/api v1.0.4
+	github.com/ncw/swift/v2 v2.0.1 // indirect
 	go.opencensus.io v0.22.3 // indirect
 	google.golang.org/api v0.22.0
 )

go.sum

@@ -139,6 +139,8 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/ncw/swift/v2 v2.0.1 h1:q1IN8hNViXEv8Zvg3Xdis4a3c4IlIGezkYz09zQL5J0=
+github.com/ncw/swift/v2 v2.0.1/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg=
 github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
 github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=

main.go

@@ -84,6 +84,10 @@ func main() {
 				snapshotPath, err := snapshotter.CreateAzureSnapshot(&snapshot, c, now)
 				logSnapshotError("azure", snapshotPath, err)
 			}
+			if c.Swift.Container != "" {
+				snapshotPath, err := snapshotter.CreateSwiftSnapshot(&snapshot, c, now)
+				logSnapshotError("swift", snapshotPath, err)
+			}
 		}
 		select {
 		case <-time.After(frequency):

snapshot_agent/agent.go

@@ -19,6 +19,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	vaultApi "github.com/hashicorp/vault/api"
+	"github.com/ncw/swift/v2"
 )
 
 type Snapshotter struct {
@@ -27,6 +28,7 @@ type Snapshotter struct {
 	S3Client        *s3.S3
 	GCPBucket       *storage.BucketHandle
 	AzureUploader   azblob.ContainerURL
+	SwiftConnection swift.Connection
 	TokenExpiration time.Time
 }
 
@@ -54,6 +56,12 @@ func NewSnapshotter(config *config.Configuration) (*Snapshotter, error) {
 			return nil, err
 		}
 	}
+	if config.Swift.Container != "" {
+		err = snapshotter.ConfigureSwift(config)
+		if err != nil {
+			return nil, err
+		}
+	}
 	return snapshotter, nil
 }
 
@@ -186,3 +194,29 @@ func (s *Snapshotter) ConfigureAzure(config *config.Configuration) error {
 	s.AzureUploader = azblob.NewContainerURL(*URL, p)
 	return nil
 }
+
+func (s *Snapshotter) ConfigureSwift(config *config.Configuration) error {
+	ctx := context.Background()
+
+	c := swift.Connection{
+		UserName: config.Swift.UserName,
+		ApiKey:   config.Swift.ApiKey,
+		AuthUrl:  config.Swift.AuthUrl,
+		Region:   config.Swift.Region,
+		TenantId: config.Swift.TenantId,
+		Domain:   config.Swift.Domain,
+	}
+
+	err := c.Authenticate(ctx)
+	if err != nil {
+		log.Fatal("Invalid credentials with error: " + err.Error())
+	}
+
+	_, _, err = c.Container(ctx, config.Swift.Container)
+	if err != nil {
+		log.Fatal("Invalid container name: " + err.Error())
+	}
+
+	s.SwiftConnection = c
+	return nil
+}

snapshot_agent/swift.go

@@ -0,0 +1,90 @@
+package snapshot_agent
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"log"
+	"sort"
+
+	"github.com/Lucretius/vault_raft_snapshot_agent/config"
+	"github.com/ncw/swift/v2"
+)
+
+func (s *Snapshotter) CreateSwiftSnapshot(b *bytes.Buffer, config *config.Configuration, currentTs int64) (string, error) {
+	ctx := context.Background()
+	fileName := fmt.Sprintf("raft_snapshot-%d.snap", currentTs)
+
+	_, header, _ := s.SwiftConnection.Container(ctx, config.Swift.Container)
+
+	object, err := s.SwiftConnection.ObjectCreate(ctx, config.Swift.Container, fileName, false, "", "", header)
+	if err != nil {
+		log.Fatal("Can't create Object in Swift: " + err.Error())
+	}
+
+	if _, err := object.Write(b.Bytes()); err != nil {
+		return "", err
+	}
+
+	if err := object.Close(); err != nil {
+		return "", err
+	}
+
+	if config.Retain > 0 {
+		deleteCtx := context.Background()
+		opts := &swift.ObjectsOpts{Prefix: "raft_snapshot-"}
+
+		objects, err := s.SwiftConnection.ObjectsAll(deleteCtx, config.Swift.Container, opts)
+		if err != nil {
+			return "", err
+		}
+
+		timestamp := func(obj1, obj2 *swift.Object) bool {
+			return obj1.LastModified.Before(obj2.LastModified)
+		}
+
+		SwiftBy(timestamp).Sort(objects)
+		if len(objects)-int(config.Retain) <= 0 {
+			return fileName, nil
+		}
+
+		objectsToDelete := objects[0 : len(objects)-int(config.Retain)]
+		for _, objToDelete := range objectsToDelete {
+			log.Printf("Delete snapshot: " + objToDelete.Name)
+			err := s.SwiftConnection.ObjectDelete(deleteCtx, config.Swift.Container, objToDelete.Name)
+			if err != nil {
+				log.Println("Cannot delete snapshot")
+				return fileName, err
+			}
+		}
+	}
+
+	return fileName, nil
+}
+
+type SwiftBy func(f1, f2 *swift.Object) bool
+
+func (by SwiftBy) Sort(objects []swift.Object) {
+	fs := &swiftObjectSorter{
+		objects: objects,
+		by:      by,
+	}
+	sort.Sort(fs)
+}
+
+type swiftObjectSorter struct {
+	objects []swift.Object
+	by      func(f1, f2 *swift.Object) bool
+}
+
+func (s *swiftObjectSorter) Len() int {
+	return len(s.objects)
+}
+
+func (s *swiftObjectSorter) Less(i, j int) bool {
+	return s.by(&s.objects[i], &s.objects[j])
+}
+
+func (s *swiftObjectSorter) Swap(i, j int) {
+	s.objects[i], s.objects[j] = s.objects[j], s.objects[i]
+}