Refactor configuration, authentication and uploaders to react dynamic…

…ally to credentials in env-vars changing (again) and allow external configuration via file-sources (as proposed by @rdeanmcdonald in Lucretius#12)

.gitignore

@@ -16,4 +16,6 @@
 local/*
 main
 out/
-.build/
+.build/
+.idea/
+*.iml

cmd/vault-raft-snapshot-agent/main.go

@@ -44,7 +44,7 @@ import (
 var Version = "development"
 var Platform = "linux/amd64"
 
-var snapshotterOptions internal.SnapshotterOptions = internal.SnapshotterOptions{
+var snapshotterOptions = internal.SnapshotterOptions{
 	ConfigFileName:        "snapshots",
 	ConfigFileSearchPaths: []string{"/etc/vault.d/", "."},
 	EnvPrefix:             "VRSA",
@@ -105,15 +105,15 @@ Options:
 }
 
 func startSnapshotter(configFile cli.Path) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
 	snapshotterOptions.ConfigFilePath = configFile
-	snapshotter, err := internal.CreateSnapshotter(ctx, snapshotterOptions)
+	snapshotter, err := internal.CreateSnapshotter(snapshotterOptions)
 	if err != nil {
 		log.Fatalf("Cannot create snapshotter: %s\n", err)
 	}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	sigs := make(chan os.Signal, 1)
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 	go func() {

internal/app/vault_raft_snapshot_agent/config/config.go

@@ -20,11 +20,7 @@ func NewParser[T Configuration](envPrefix string, configFilename string, configS
 // ReadConfig reads the configuration file
 func (p Parser[T]) ReadConfig(config T, file string) error {
 	err := p.delegate.BindAllEnv(
-		map[string]string{
-			"vault.url":                        "VAULT_ADDR",
-			"uploaders.aws.credentials.key":    "AWS_ACCESS_KEY_ID",
-			"uploaders.aws.credentials.secret": "AWS_SECRET_ACCESS_KEY",
-		},
+		map[string]string{"vault.url": "VAULT_ADDR"},
 	)
 	if err != nil {
 		return fmt.Errorf("could not bind environment-variables: %s", err)

internal/app/vault_raft_snapshot_agent/config/config_test.go

@@ -34,18 +34,13 @@ func TestReadConfigBindsEnvVariables(t *testing.T) {
 	parser := NewParser[*configDataStub]("TEST", "")
 
 	t.Setenv("VAULT_ADDR", "http://from.env:8200")
-	t.Setenv("AWS_ACCESS_KEY_ID", "env-key")
-	t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret")
 	t.Setenv("TEST_VAULT_TEST", "test")
 
-
 	data := configDataStub{hasUploaders: true}
 	err := parser.ReadConfig(&data, "")
 	assert.NoError(t, err, "ReadConfig failed unexpectedly")
 
 	assert.Equal(t, os.Getenv("VAULT_ADDR"), data.Vault.Url, "ReadConfig did not bind env-var VAULT_ADDR")
-	assert.Equal(t, os.Getenv("AWS_ACCESS_KEY_ID"), data.Uploaders.AWS.Credentials.Key, "ReadConfig did not bind env-var AWS_ACCESS_KEY_ID")
-	assert.Equal(t, os.Getenv("AWS_SECRET_ACCESS_KEY"), data.Uploaders.AWS.Credentials.Secret, "ReadConfig did not bind env-var SECRET_ACCESS_KEY")
 	assert.Equal(t, os.Getenv("TEST_VAULT_TEST"), data.Vault.Test, "ReadConfig did not bind env-var TEST_VAULT_TEST")
 }
 

internal/app/vault_raft_snapshot_agent/config/rattlesnake.go

@@ -1,8 +1,11 @@
 package config
 
 import (
+	"errors"
 	"fmt"
+	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret"
 	"path/filepath"
+	"reflect"
 	"strings"
 
 	"github.com/creasty/defaults"
@@ -77,12 +80,12 @@ func (r rattlesnake) Unmarshal(config interface{}) error {
 		return fmt.Errorf("could not set configuration's default-values: %s", err)
 	}
 
-	pathResolver := newPathResolver(filepath.Dir(r.ConfigFileUsed()))
-	if err := pathResolver.Resolve(config); err != nil {
+	if err := secret.ResolveFilePaths(config, filepath.Dir(r.ConfigFileUsed())); err != nil {
 		return fmt.Errorf("could not resolve relative paths in configuration: %s", err)
 	}
 
 	validate := validator.New()
+	validate.RegisterCustomTypeFunc(validateSecret, secret.Zero)
 	if err := validate.Struct(config); err != nil {
 		return err
 	}
@@ -98,10 +101,24 @@ func (r rattlesnake) OnConfigChange(run func()) {
 }
 
 func (r rattlesnake) IsConfigurationNotFoundError(err error) bool {
-	_, notfound := err.(viper.ConfigFileNotFoundError)
+	var configFileNotFoundError viper.ConfigFileNotFoundError
+	notfound := errors.As(err, &configFileNotFoundError)
 	return notfound
 }
 
+func validateSecret(field reflect.Value) interface{} {
+	s, ok := field.Interface().(secret.Secret)
+	if !ok {
+		return nil
+	}
+
+	v, err := s.Resolve(false)
+	if err != nil {
+		v = ""
+	}
+	return v
+}
+
 // implements automatic unmarshalling from environment variables
 // see https://github.com/spf13/viper/pull/1429
 // can be removed if that pr is merged

internal/app/vault_raft_snapshot_agent/config/rattlesnake_test.go

@@ -2,7 +2,7 @@ package config
 
 import (
 	"fmt"
-	"os"
+	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret"
 	"path/filepath"
 	"testing"
 
@@ -11,68 +11,90 @@ import (
 	"github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/test"
 )
 
-type rattlesnakeConfigStub struct {
-	Path string `default:"/test/file" resolve-path:""`
-	Url	 string `validate:"omitempty,http_url"`
-}
-
-func TestUnmarshalResolvesRelativePaths(t *testing.T) {
+func TestUnmarshalResolvesRelativePathsInSecrets(t *testing.T) {
 	rattlesnake := newRattlesnake("test", "TEST")
 
-	wd, err := os.Getwd()
-	assert.NoError(t, err, "Getwd failed unexpectedly")
+	config := struct {
+		File secret.Secret
+	}{
+		File: "file://./file.ext",
+	}
 
-	err = rattlesnake.SetConfigFile(fmt.Sprintf("%s/config.yml", wd))
+	baseDir := t.TempDir()
+	err := rattlesnake.SetConfigFile(fmt.Sprintf("%s/config.yml", baseDir))
 	assert.NoError(t, err, "SetConfigFile failed unexpectedly")
 
-	t.Setenv("TEST_PATH", "./file.ext")
-	config := rattlesnakeConfigStub{}
 	err = rattlesnake.Unmarshal(&config)
-
 	assert.NoError(t, err, "Unmarshal failed unexpectedly")
-	assert.Equal(t, filepath.Clean(fmt.Sprintf("%s/file.ext", wd)), config.Path)
+	assert.Equal(t, secret.FromFile(filepath.Clean(fmt.Sprintf("%s/file.ext", baseDir))), config.File)
 }
 
 func TestUnmarshalSetsDefaultValues(t *testing.T) {
 	rattlesnake := newRattlesnake("test", "TEST")
 
-	config := rattlesnakeConfigStub{}
+	var config struct {
+		Default string `default:"default-value"`
+	}
+
 	err := rattlesnake.Unmarshal(&config)
 
 	assert.NoError(t, err, "Unmarshal failed unexpectedly")
-	assert.Equal(t, "/test/file", config.Path)
+	assert.Equal(t, "default-value", config.Default)
 }
 
 func TestUnmarshalValidatesValues(t *testing.T) {
 	rattlesnake := newRattlesnake("test", "TEST")
 
-	t.Setenv("TEST_URL", "not_an_url")
-	config := rattlesnakeConfigStub{}
+	config := struct {
+		Url string `validate:"http_url"`
+	}{
+		Url: "invalid-url",
+	}
+
+	err := rattlesnake.Unmarshal(&config)
+
+	assert.Error(t, err, "Unmarshal should fail on validation error")
+	assert.Equal(t, "invalid-url", config.Url)
+}
+
+func TestUnmarshalValidatesSecrets(t *testing.T) {
+	rattlesnake := newRattlesnake("test", "TEST")
+
+	config := struct {
+		Secret secret.Secret `validate:"required"`
+	}{
+		Secret: secret.FromFile("./missing/file"),
+	}
+
 	err := rattlesnake.Unmarshal(&config)
 
 	assert.Error(t, err, "Unmarshal should fail on validation error")
-	assert.Equal(t, "not_an_url", config.Url)
 }
 
 func TestOnConfigChangeRunsHandler(t *testing.T) {
 	rattlesnake := newRattlesnake("test", "TEST")
+
 	configFile := fmt.Sprintf("%s/config.yml", t.TempDir())
 
-	err := rattlesnake.SetConfigFile(configFile)
+	err := test.WriteFile(t, configFile, "{\"value\": \"\"}")
+	assert.NoError(t, err, "writing config file failed unexpectedly")
+
+	err = rattlesnake.SetConfigFile(configFile)
 	assert.NoError(t, err, "SetConfigFile failed unexpectedly")
 
-	err = test.WriteFile(t, configFile, "{\"url\": \"http://example.com\"}")
-	assert.NoError(t, err, "writing config file failed unexpectedly")
+	var config struct {
+		Value string
+	}
 
-	err = rattlesnake.Unmarshal(&rattlesnakeConfigStub{})
+	err = rattlesnake.Unmarshal(&config)
 	assert.NoError(t, err, "Unmarshal failed unexpectedly")
 
 	changed := make(chan bool, 1)
 	rattlesnake.OnConfigChange(func() {
 		changed <- true
 	})
 
-	err = test.WriteFile(t, configFile, "{\"url\": \"http://new.com\"}")
+	err = test.WriteFile(t, configFile, "{\"value\": \"new\"}")
 	assert.NoError(t, err, "writing config file failed unexpectedly")
 
 	assert.True(t, <-changed)