From 1dfa2ac6573077144dab0eaf7952d547e0b3837d Mon Sep 17 00:00:00 2001 From: Argelbargel Date: Mon, 18 Sep 2023 22:25:43 +0200 Subject: [PATCH] Refactor configuration, authentication and uploaders to react dynamically to credentials in env-vars changing (again) and allow external configuration via file-sources (as proposed by @rdeanmcdonald in https://github.com/Lucretius/vault_raft_snapshot_agent/pull/12) --- .gitignore | 4 +- README.md | 116 ++++++++++-------- cmd/vault-raft-snapshot-agent/main.go | 10 +- .../config/config.go | 6 +- .../config/config_test.go | 5 - .../config/rattlesnake.go | 23 +++- .../config/rattlesnake_test.go | 70 +++++++---- .../config/resolve-path-tag.go | 86 ------------- .../config/resolve-path-tag_test.go | 58 --------- .../secret/resolve-file-paths.go | 61 +++++++++ .../secret/resolve-file-paths_test.go | 47 +++++++ .../secret/secret.go | 93 ++++++++++++++ .../secret/secret_test.go | 107 ++++++++++++++++ .../vault_raft_snapshot_agent/snapshotter.go | 26 ++-- .../snapshotter_config_test.go | 93 +++++--------- .../snapshotter_test.go | 8 +- .../vault_raft_snapshot_agent/upload/aws.go | 112 ++++++++++------- .../vault_raft_snapshot_agent/upload/azure.go | 64 ++++++---- .../vault_raft_snapshot_agent/upload/gcp.go | 40 +++--- .../vault_raft_snapshot_agent/upload/local.go | 36 +++--- .../upload/local_test.go | 15 +-- .../vault_raft_snapshot_agent/upload/swift.go | 83 ++++++++----- .../upload/uploaders.go | 72 +++++------ .../upload/uploaders_test.go | 68 +++++++--- .../vault/auth/approle.go | 36 ++++-- .../vault/auth/approle_test.go | 11 +- .../vault/auth/auth.go | 43 ++++--- .../vault/auth/aws.go | 80 ++++++------ .../vault/auth/aws_test.go | 33 +++-- .../vault/auth/azure.go | 24 ++-- .../vault/auth/azure_test.go | 16 +-- .../vault/auth/gcp.go | 28 ++--- .../vault/auth/gcp_test.go | 17 ++- .../vault/auth/kubernetes.go | 34 ++--- .../vault/auth/kubernetes_test.go | 18 ++- .../vault/auth/ldap.go | 35 ++++-- .../vault/auth/ldap_test.go | 16 ++- .../vault/auth/token.go | 18 ++- .../vault/auth/token_test.go | 11 +- .../vault/auth/userpass.go | 35 ++++-- .../vault/auth/userpass_test.go | 16 ++- .../vault_raft_snapshot_agent/vault/client.go | 29 +++-- .../vault/client_test.go | 12 +- testdata/complete.yaml | 12 +- testdata/defaults.yaml | 1 + testdata/empty.yaml | 0 testdata/envvars.yaml | 8 -- testdata/invalid-auth.yaml | 9 -- testdata/invalid-local-upload-path.yaml | 5 - testdata/invalid-uploader.yaml | 5 - testdata/invalid-url.yaml | 7 -- testdata/no-uploaders.yaml | 3 - testdata/watch-and-reconfigure1.yaml | 10 -- testdata/watch-and-reconfigure2.yaml | 10 -- 54 files changed, 1066 insertions(+), 819 deletions(-) delete mode 100644 internal/app/vault_raft_snapshot_agent/config/resolve-path-tag.go delete mode 100644 internal/app/vault_raft_snapshot_agent/config/resolve-path-tag_test.go create mode 100644 internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths.go create mode 100644 internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths_test.go create mode 100644 internal/app/vault_raft_snapshot_agent/secret/secret.go create mode 100644 internal/app/vault_raft_snapshot_agent/secret/secret_test.go delete mode 100644 testdata/empty.yaml delete mode 100644 testdata/envvars.yaml delete mode 100644 testdata/invalid-auth.yaml delete mode 100644 testdata/invalid-local-upload-path.yaml delete mode 100644 testdata/invalid-uploader.yaml delete mode 100644 testdata/invalid-url.yaml delete mode 100644 testdata/no-uploaders.yaml delete mode 100644 testdata/watch-and-reconfigure1.yaml delete mode 100644 testdata/watch-and-reconfigure2.yaml diff --git a/.gitignore b/.gitignore index 7c1139f..7581a23 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,6 @@ local/* main out/ -.build/ \ No newline at end of file +.build/ +.idea/ +*.iml \ No newline at end of file diff --git a/README.md b/README.md index b3a3a2e..21dea9d 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,11 @@ If your configuration is right and Vault is running on the same host as the agen ## Configuration -Vault Raft Snaphot Agent looks for it's configuration-file in `/etc/vault.d/` or the current working directory by default. It uses [viper](https://github.com/spf13/viper) as configuration-backend so you can write your configuration in either json, yaml or toml. -You an use `vault-raft-snapshot-agent --config ` to use a specific configuration file. +Vault Raft Snapshot Agent looks for it's configuration-file in `/etc/vault.d/` or the current working directory by default. +It uses [viper](https://github.com/spf13/viper) as configuration-backend, so you can write your configuration in either json, yaml or toml. +You can use `vault-raft-snapshot-agent --config ` to use a specific configuration file. + +The Agent monitors the configuration-file for changes and reloads the configuration automatically when the configuration-file changes. #### Example configuration (yaml) @@ -87,18 +90,31 @@ uploaders: (for a complete example with all configuration-options see [complete.yaml](./testdata/complete.yaml)) +### Secrets and dynamic Properties +Vault Raft Snapshot allows you to specify dynamic sources for properties containing secrets which either should not go +directly into the configuration file or might change while the agent is running (or for which there exist "well-known" +environment-variables like `AWS_DEFAULT_REGION`). For these properties you may specify either an environment variable +as source using `env://` or a file-source containing the value for the secret using `file://`, +where `` may be either an absolute path or a path relative to the configuration file. Any value not prefixed +with `env://` or `file://` will be used as is. + +**Dynamic properties are validated at startup only, so if e.g. you delete the source-file for a property required to +authenticate with vault or connect to a remote storage while the agent is running, the next login to vault or upload +to that storage will fail (gracefully)!** + + ### Environment variables -Vault Raft Snapshot Agent supports configuration with environment variables. For some common options there are shortcuts defined: -- `VAULT_ADDR` configures the url to the vault-server (same as `vault.url`) -- `AWS_ACCESS_KEY_ID` configures the access key for the AWS uploader (same as `uploaders.aws.credentials.key`) and AWS EC2 authentication -- `AWS_SECRET_ACCESS_KEY` configures the access secret for the AWS uploader (same as `uploaders.aws.credentials.secret`) and AWS EC2 authentication -- `AWS_SESSION_TOKEN` configures the session-token for AWS EC2 authentication -- `AWS_SHARED_CREDENTIALS_FILE` configures AWS EC2 authentication from a file +Vault Raft Snapshot Agent supports static configuration via environment variables. Any option can be set by prefixing `VRSA_` +to the upper-cased path to the key and replacing `.` with `_`. For example `VRSA_SNAPSHOTS_FREQUENCY=` configures +the snapshot-frequency and `VRSA_VAULT_AUTH_TOKEN=` configures the token authentication for vault. +For setting the address of the vault-server there is a snapshot defined. `VAULT_ADDR` configures the url to the vault-server (same as `vault.url`). -Any other option can be set by prefixing `VRSA_` to the uppercased path to the key and replacing `.` with `_`. For example `VRSA_SNAPSHOTS_FREQUENCY=` configures the snapshot-frequency and `VRSA_VAULT_AUTH_TOKEN=` configures the token authentication for vault. +Other than the dynamic [the section above](#secrets-and-dynamic-properties) environment variables are read once at startup so the configuration will not be +reloaded when their values change. + +_Options specified via environment-variables take precedence before the values specified in the configuration file - even those specified as secrets!_ -_Options specified via environment-variables take precedence before the values specified in the configuration file!_ ### Vault configuration ``` @@ -108,15 +124,11 @@ vault: timeout: ``` -- `url` *(default: https://127.0.0.1:8200)* - specifies the url of the vault-server. - +- `url` *(default: https://127.0.0.1:8200)* - specifies the url of the vault-server. You can alternatively specify the url with the environment-variable `VAULT_ADDR` **The URL should point be the cluster-leader, otherwise no snapshots get taken until the server the url points to is elected leader!** When running Vault on Kubernetes installed by the [default helm-chart](https://developer.hashicorp.com/vault/docs/platform/k8s/helm), this should be `http(s)://vault-active..svc.cluster.local:`. - - You can alternatively specify the url with the environment-variable `VAULT_ADDR` - - - `insecure` *(default: false)* - specifies whether insecure https connections are allowed or not. Set to `true` when you use self-signed certificates -- `timeout` *(default: 60s)* - timeout for the vault-http-client (see https://golang.org/pkg/time/#ParseDuration for a full list of valid time units); increase for large raft databases (and increase `snapshots.timeout` accordingly!) +- `timeout` *(default: 60s)* - timeout for the vault-http-client (see https://golang.org/pkg/time/#ParseDuration for a full list of valid time units); + increase for large raft databases (and increase `snapshots.timeout` accordingly!) ### Vault authentication @@ -154,8 +166,10 @@ vault: ``` ##### Configuration options -- `role` **(required)** - specifies the role_id used to call the Vault API. See the authentication steps below -- `secret` **(required)** - specifies the secret_id used to call the Vault API +- `role` **(required)** - specifies the role_id used to call the Vault API. See the authentication steps below *This + property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `secret` **(required)** - specifies the secret_id used to call the Vault API. *This property can be configured with a + source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* - `path` *(default: approle)* - specifies the backend-name used to select the login-endpoint (`auth//login`) To allow the App-Role access to the snapshots you should run the following commands on your vault-cluster: @@ -180,13 +194,15 @@ vault: ##### Configuration options - `role` **(required)** - specifies the role used to call the Vault API. See the authentication steps below -- `ec2Nonce` - enables EC2 authentication and sets the required nonce +- `ec2Nonce` - enables EC2 authentication and sets the required nonce. *This property can be configured with a + source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* - `ec2SignatureType` *(default: pkcs7)* - changes the signature-type for EC2 authentication; valid values are `identity`, `pkcs7` and `rs2048` -- `iamServerIdHeader` - specifies the server-id-header when using IAM authtype -- `region` - specifies the aws region to use +- `iamServerIdHeader` - specifies the server-id-header when using IAM authentication type +- `region` *(default: env://AWS_DEFAULT_REGION)* - specifies the aws region to use. *This property can be configured with a + source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* - `path` *(default: aws)* - specifies the backend-name used to select the login-endpoint (`auth//login`) -By default AWS authentication uses the IAM authentication type unless `ec2Nonce` is set. The credentials for IAM authentication must be provided via environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` or `AWS_SHARED_CREDENTIALS_FILE`). While relative paths normally are resolved relative to the configuration-file, `AWS_SHARED_CREDENTIALS_FILE` must be specified as an absolute path. +AWS authentication uses the IAM authentication type by default unless `ec2Nonce` is set. *The credentials for IAM authentication **must** be provided via environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` or `AWS_SHARED_CREDENTIALS_FILE`; `AWS_SHARED_CREDENTIALS_FILE` must be specified as an absolute path).* To allow the access to the snapshots you should run the following commands on your vault-cluster: ``` @@ -241,7 +257,7 @@ vault: - `serviceAccountEmail` - activates IAM authentication and specifies the service-account to use - `path` *(default: gcp)* - specifies the backend-name used to select the login-endpoint (`auth//login`) -By default Google Cloud authentication uses the GCE authentication type unless `serviceAccountEmail` is set. +Google Cloud authentication uses the GCE authentication type by default unless `serviceAccountEmail` is set. To allow the access to the snapshots you should run the following commands on your vault-cluster: ``` @@ -276,7 +292,8 @@ vault: ##### Configuration options - `role` **(required)** - specifies vault k8s auth role - `path` *(default: kubernetes)* - specifies the backend-name used to select the login-endpoint (`auth//login`) -- `jwtPath` *(default: /var/run/secrets/kubernetes.io/serviceaccount/token)* - specifies the path to the file with the JWT-Token for the kubernetes service-account. You may specify the path relative to the location of the configuration file. +- `jwtToken` *(default: file:///var/run/secrets/kubernetes.io/serviceaccount/token, must resolve to a non-empty value)* - specifies the JWT-Token for the kubernetes service-account. + *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* To allow kubernetes access to the snapshots you should run the following commands on your vault-cluster: ``` @@ -296,8 +313,8 @@ vault: ``` ##### Configuration options -- `username` **(required)** - the username -- `password` **(required)** - the password +- `username` **(required)** - the username. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `password` **(required)** - the password. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* - `path` *(default: ldap)* - specifies the backend-name used to select the login-endpoint (`auth//login`) To allow access to the snapshots you should run the following commands on your vault-cluster: @@ -319,7 +336,7 @@ vault: ``` ##### Configuration options -- `token` **(required)** - specifies the token used to login +- `token` **(required)** - specifies the token used to log in. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* #### User and Password authentication @@ -335,8 +352,8 @@ vault: ``` ##### Configuration options -- `username` **(required)** - the username -- `password` **(required)** - the password +- `username` **(required)** - the username. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `password` **(required)** - the password. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* - `path` *(default: userpass)* - specifies the backend-name used to select the login-endpoint (`auth//login`) To allow access to the snapshots you should run the following commands on your vault-cluster: @@ -360,7 +377,7 @@ snapshots: ``` - `frequency` *(default: 1h)* - how often to run the snapshot agent. Examples: `30s`, `1h`. See https://golang.org/pkg/time/#ParseDuration for a full list of valid time units -- `retain` *(default: 0)* -the number of snaphots to retain. For example, if you set `retain: 2`, the two most recent snapshots will be kept in storage. `0` means all snapshots will be retained +- `retain` *(default: 0)* -the number of snapshots to retain. For example, if you set `retain: 2`, the two most recent snapshots will be kept in storage. `0` means all snapshots will be retained - `timeout` *(default: 60s)* - timeout for creating snapshots. Examples: `30s`, `1h`. See https://golang.org/pkg/time/#ParseDuration for a full list of valid time units - `namePrefix` *(default: raft-snapshot-)* - prefix of the uploaded snapshots - `nameSuffix` *(default: .snap)* - suffix/extension of the uploaded snapshots @@ -394,35 +411,26 @@ uploaders: authUrl: ``` -Note that if you specify more than one storage option, *all* soecified storages will be written to. For example, specifying `local` and `aws` will write to both locations. When using multiple remote storages, increase the timeout allowed via `snapahots.timeout` for larger raft databases. Each option can be specified exactly once; it is currently not possible to e.g. upload to multiple aws regions by specifying multiple `aws`-storage-options. +Note that if you specify more than one storage option, *all* specified storages will be written to. For example, specifying `local` and `aws` will write to both locations. When using multiple remote storages, increase the timeout allowed via `snapahots.timeout` for larger raft databases. Each option can be specified exactly once; it is currently not possible to e.g. upload to multiple aws regions by specifying multiple `aws`-storage-options. #### AWS S3 Upload - `bucket` **(required)** - bucket to store snapshots in (required for AWS writes to work) +- `accessKeyId` *(default: env://AWS_ACCESS_KEY_ID)* - specifies the access key. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `accessKey` *(default: env://AWS_SECRET_ACCESS_KEY, must resolve to non-empty value if accessKeyId resolves to a non-empty value)* - specifies the secret access key. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `sessionToken` *(default: env//AWS_SESSION_TOKEN)* - specifies the session token *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* - `region` *(default: "")* - S3 region if it is required - `keyPrefix` *(default: "")* - prefix to store s3 snapshots in. Defaults to empty string - `endpoint` *(default: "")* - S3 compatible storage endpoint (ex: http://127.0.0.1:9000) - `useServerSideEncryption` *(default: false)* - Encryption is **off** by default. Set to true to turn on AWS' AES256 encryption. Support for AWS KMS keys is not currently supported. -- `forcePathStyle` *(default: false)* - Needed if your S3 Compatible storage support only path-style or you would like to use S3's FIPS Endpoint. - - -##### AWS authentication -``` -uploaders: - aws: - credentials: - key: - secret: -``` -- `key` **(required)** - specifies the access key. It's recommended to use the standard `AWS_ACCESS_KEY_ID` env var, though -- `secret` **(required)** - specifies the secret It's recommended to use the standard `AWS_SECRET_ACCESS_KEY` env var, though +- `forcePathStyle` *(default: false)* - needed if your S3 Compatible storage supports only path-style, or you would like to use S3's FIPS Endpoint. #### Azure Storage -- `accountName` **(required)** - the account name of the storage account -- `accountKey` **(required)** - the account key of the storage account -- `containerName` **(required)** - the name of the blob container to write to -- `cloudDomain` *(default: blob.core.windows.net) - domain of the cloud-service to use +- `container` **(required)** - the name of the blob container to write to +- `accountName` *(default: env://AZURE_STORAGE_ACCOUNT, must resolve to non-empty value)* - the account name of the storage account. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `accountKey` *(default: env://AZURE_STORAGE_KEY, must resolve to non-empty value)* - the account key of the storage account. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `cloudDomain` *(default: blob.core.windows.net)* - domain of the cloud-service to use #### Google Cloud Storage @@ -434,13 +442,13 @@ uploaders: #### Openstack Swift Storage - `container` **(required)** - the name of the container to write to -- `username` **(required)** - the username used for authentication -- `apiKey` **(required)** - the api-key used for authentication -- `authUrl` **(required)** - the auth-url to authenicate against -- `region` - optional region to use eg "LON", "ORD" +- `authUrl` **(required)** - the auth-url to authenticate against +- `username` *(default: env://SWIFT_USERNAME, must resolve to non-empty value)* - the username used for authentication. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `apiKey` *(default: env://SWIFT_API_KEY, must resolve to non-empty value)* - the api-key used for authentication. *This property can be configured with a source that is evaluated at runtime, see [the section above](#secrets-and-dynamic-properties)* +- `region` *(default: env://SWIFT_REGION)* - optional region to use eg "LON", "ORD" - `domain` - optional user's domain name - `tenantId` - optional id of the tenant -- `timeout` *(default: 60s)** - timeout for snapshot-uploads +- `timeout` *(default: 60s)* - timeout for snapshot-uploads diff --git a/cmd/vault-raft-snapshot-agent/main.go b/cmd/vault-raft-snapshot-agent/main.go index 12aba23..9dc6669 100644 --- a/cmd/vault-raft-snapshot-agent/main.go +++ b/cmd/vault-raft-snapshot-agent/main.go @@ -44,7 +44,7 @@ import ( var Version = "development" var Platform = "linux/amd64" -var snapshotterOptions internal.SnapshotterOptions = internal.SnapshotterOptions{ +var snapshotterOptions = internal.SnapshotterOptions{ ConfigFileName: "snapshots", ConfigFileSearchPaths: []string{"/etc/vault.d/", "."}, EnvPrefix: "VRSA", @@ -105,15 +105,15 @@ Options: } func startSnapshotter(configFile cli.Path) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - snapshotterOptions.ConfigFilePath = configFile - snapshotter, err := internal.CreateSnapshotter(ctx, snapshotterOptions) + snapshotter, err := internal.CreateSnapshotter(snapshotterOptions) if err != nil { log.Fatalf("Cannot create snapshotter: %s\n", err) } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sigs := make(chan os.Signal, 1) signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { diff --git a/internal/app/vault_raft_snapshot_agent/config/config.go b/internal/app/vault_raft_snapshot_agent/config/config.go index b160d5e..744eab7 100644 --- a/internal/app/vault_raft_snapshot_agent/config/config.go +++ b/internal/app/vault_raft_snapshot_agent/config/config.go @@ -20,11 +20,7 @@ func NewParser[T Configuration](envPrefix string, configFilename string, configS // ReadConfig reads the configuration file func (p Parser[T]) ReadConfig(config T, file string) error { err := p.delegate.BindAllEnv( - map[string]string{ - "vault.url": "VAULT_ADDR", - "uploaders.aws.credentials.key": "AWS_ACCESS_KEY_ID", - "uploaders.aws.credentials.secret": "AWS_SECRET_ACCESS_KEY", - }, + map[string]string{"vault.url": "VAULT_ADDR"}, ) if err != nil { return fmt.Errorf("could not bind environment-variables: %s", err) diff --git a/internal/app/vault_raft_snapshot_agent/config/config_test.go b/internal/app/vault_raft_snapshot_agent/config/config_test.go index 777e9f1..24164ef 100644 --- a/internal/app/vault_raft_snapshot_agent/config/config_test.go +++ b/internal/app/vault_raft_snapshot_agent/config/config_test.go @@ -34,18 +34,13 @@ func TestReadConfigBindsEnvVariables(t *testing.T) { parser := NewParser[*configDataStub]("TEST", "") t.Setenv("VAULT_ADDR", "http://from.env:8200") - t.Setenv("AWS_ACCESS_KEY_ID", "env-key") - t.Setenv("AWS_SECRET_ACCESS_KEY", "env-secret") t.Setenv("TEST_VAULT_TEST", "test") - data := configDataStub{hasUploaders: true} err := parser.ReadConfig(&data, "") assert.NoError(t, err, "ReadConfig failed unexpectedly") assert.Equal(t, os.Getenv("VAULT_ADDR"), data.Vault.Url, "ReadConfig did not bind env-var VAULT_ADDR") - assert.Equal(t, os.Getenv("AWS_ACCESS_KEY_ID"), data.Uploaders.AWS.Credentials.Key, "ReadConfig did not bind env-var AWS_ACCESS_KEY_ID") - assert.Equal(t, os.Getenv("AWS_SECRET_ACCESS_KEY"), data.Uploaders.AWS.Credentials.Secret, "ReadConfig did not bind env-var SECRET_ACCESS_KEY") assert.Equal(t, os.Getenv("TEST_VAULT_TEST"), data.Vault.Test, "ReadConfig did not bind env-var TEST_VAULT_TEST") } diff --git a/internal/app/vault_raft_snapshot_agent/config/rattlesnake.go b/internal/app/vault_raft_snapshot_agent/config/rattlesnake.go index 31f00ac..dc5c4f3 100644 --- a/internal/app/vault_raft_snapshot_agent/config/rattlesnake.go +++ b/internal/app/vault_raft_snapshot_agent/config/rattlesnake.go @@ -1,8 +1,11 @@ package config import ( + "errors" "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "path/filepath" + "reflect" "strings" "github.com/creasty/defaults" @@ -77,12 +80,12 @@ func (r rattlesnake) Unmarshal(config interface{}) error { return fmt.Errorf("could not set configuration's default-values: %s", err) } - pathResolver := newPathResolver(filepath.Dir(r.ConfigFileUsed())) - if err := pathResolver.Resolve(config); err != nil { + if err := secret.ResolveFilePaths(config, filepath.Dir(r.ConfigFileUsed())); err != nil { return fmt.Errorf("could not resolve relative paths in configuration: %s", err) } validate := validator.New() + validate.RegisterCustomTypeFunc(validateSecret, secret.Zero) if err := validate.Struct(config); err != nil { return err } @@ -98,10 +101,24 @@ func (r rattlesnake) OnConfigChange(run func()) { } func (r rattlesnake) IsConfigurationNotFoundError(err error) bool { - _, notfound := err.(viper.ConfigFileNotFoundError) + var configFileNotFoundError viper.ConfigFileNotFoundError + notfound := errors.As(err, &configFileNotFoundError) return notfound } +func validateSecret(field reflect.Value) interface{} { + s, ok := field.Interface().(secret.Secret) + if !ok { + return nil + } + + v, err := s.Resolve(false) + if err != nil { + v = "" + } + return v +} + // implements automatic unmarshalling from environment variables // see https://github.com/spf13/viper/pull/1429 // can be removed if that pr is merged diff --git a/internal/app/vault_raft_snapshot_agent/config/rattlesnake_test.go b/internal/app/vault_raft_snapshot_agent/config/rattlesnake_test.go index 7d63787..ec124f6 100644 --- a/internal/app/vault_raft_snapshot_agent/config/rattlesnake_test.go +++ b/internal/app/vault_raft_snapshot_agent/config/rattlesnake_test.go @@ -2,7 +2,7 @@ package config import ( "fmt" - "os" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "path/filepath" "testing" @@ -11,60 +11,82 @@ import ( "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/test" ) -type rattlesnakeConfigStub struct { - Path string `default:"/test/file" resolve-path:""` - Url string `validate:"omitempty,http_url"` -} - -func TestUnmarshalResolvesRelativePaths(t *testing.T) { +func TestUnmarshalResolvesRelativePathsInSecrets(t *testing.T) { rattlesnake := newRattlesnake("test", "TEST") - wd, err := os.Getwd() - assert.NoError(t, err, "Getwd failed unexpectedly") + config := struct { + File secret.Secret + }{ + File: "file://./file.ext", + } - err = rattlesnake.SetConfigFile(fmt.Sprintf("%s/config.yml", wd)) + baseDir := t.TempDir() + err := rattlesnake.SetConfigFile(fmt.Sprintf("%s/config.yml", baseDir)) assert.NoError(t, err, "SetConfigFile failed unexpectedly") - t.Setenv("TEST_PATH", "./file.ext") - config := rattlesnakeConfigStub{} err = rattlesnake.Unmarshal(&config) - assert.NoError(t, err, "Unmarshal failed unexpectedly") - assert.Equal(t, filepath.Clean(fmt.Sprintf("%s/file.ext", wd)), config.Path) + assert.Equal(t, secret.FromFile(filepath.Clean(fmt.Sprintf("%s/file.ext", baseDir))), config.File) } func TestUnmarshalSetsDefaultValues(t *testing.T) { rattlesnake := newRattlesnake("test", "TEST") - config := rattlesnakeConfigStub{} + var config struct { + Default string `default:"default-value"` + } + err := rattlesnake.Unmarshal(&config) assert.NoError(t, err, "Unmarshal failed unexpectedly") - assert.Equal(t, "/test/file", config.Path) + assert.Equal(t, "default-value", config.Default) } func TestUnmarshalValidatesValues(t *testing.T) { rattlesnake := newRattlesnake("test", "TEST") - t.Setenv("TEST_URL", "not_an_url") - config := rattlesnakeConfigStub{} + config := struct { + Url string `validate:"http_url"` + }{ + Url: "invalid-url", + } + + err := rattlesnake.Unmarshal(&config) + + assert.Error(t, err, "Unmarshal should fail on validation error") + assert.Equal(t, "invalid-url", config.Url) +} + +func TestUnmarshalValidatesSecrets(t *testing.T) { + rattlesnake := newRattlesnake("test", "TEST") + + config := struct { + Secret secret.Secret `validate:"required"` + }{ + Secret: secret.FromFile("./missing/file"), + } + err := rattlesnake.Unmarshal(&config) assert.Error(t, err, "Unmarshal should fail on validation error") - assert.Equal(t, "not_an_url", config.Url) } func TestOnConfigChangeRunsHandler(t *testing.T) { rattlesnake := newRattlesnake("test", "TEST") + configFile := fmt.Sprintf("%s/config.yml", t.TempDir()) - err := rattlesnake.SetConfigFile(configFile) + err := test.WriteFile(t, configFile, "{\"value\": \"\"}") + assert.NoError(t, err, "writing config file failed unexpectedly") + + err = rattlesnake.SetConfigFile(configFile) assert.NoError(t, err, "SetConfigFile failed unexpectedly") - err = test.WriteFile(t, configFile, "{\"url\": \"http://example.com\"}") - assert.NoError(t, err, "writing config file failed unexpectedly") + var config struct { + Value string + } - err = rattlesnake.Unmarshal(&rattlesnakeConfigStub{}) + err = rattlesnake.Unmarshal(&config) assert.NoError(t, err, "Unmarshal failed unexpectedly") changed := make(chan bool, 1) @@ -72,7 +94,7 @@ func TestOnConfigChangeRunsHandler(t *testing.T) { changed <- true }) - err = test.WriteFile(t, configFile, "{\"url\": \"http://new.com\"}") + err = test.WriteFile(t, configFile, "{\"value\": \"new\"}") assert.NoError(t, err, "writing config file failed unexpectedly") assert.True(t, <-changed) diff --git a/internal/app/vault_raft_snapshot_agent/config/resolve-path-tag.go b/internal/app/vault_raft_snapshot_agent/config/resolve-path-tag.go deleted file mode 100644 index a5a3e16..0000000 --- a/internal/app/vault_raft_snapshot_agent/config/resolve-path-tag.go +++ /dev/null @@ -1,86 +0,0 @@ -package config - -import ( - "errors" - "path/filepath" - "reflect" - "strings" -) - -const ( - tagFieldName = "resolve-path" -) - -var ( - errorInvalidType error = errors.New("subject must be a struct passed by pointer") -) - -type pathResolver struct { - baseDir string -} - -func newPathResolver(baseDir string) pathResolver { - return pathResolver{baseDir} -} - -func (r pathResolver) Resolve(subject interface{}) error { - if reflect.TypeOf(subject).Kind() != reflect.Ptr { - return errorInvalidType - } - - s := reflect.ValueOf(subject).Elem() - - return r.resolve(s) -} - -func (r pathResolver) resolve(value reflect.Value) error { - t := value.Type() - - if t.Kind() != reflect.Struct { - return errorInvalidType - } - - for i := 0; i < t.NumField(); i++ { - f := value.Field(i) - - if !f.CanSet() { - continue - } - - if f.Kind() == reflect.Ptr { - f = f.Elem() - } - - if f.Kind() == reflect.Struct { - if err := r.resolve(f); err != nil { - return err - } - } - - if f.Kind() != reflect.String || f.String() == "" { - continue - } - - if baseDir, present := t.Field(i).Tag.Lookup(tagFieldName); present { - if err := r.resolvePath(f, baseDir); err != nil { - return err - } - } - } - - return nil -} - -func (r pathResolver) resolvePath(field reflect.Value, baseDir string) error { - path := field.String() - if baseDir == "" { - baseDir = r.baseDir - } - - if !filepath.IsAbs(path) && !strings.HasPrefix(path, "/") { - path = filepath.Join(baseDir, path) - field.Set(reflect.ValueOf(path).Convert(field.Type())) - } - - return nil -} diff --git a/internal/app/vault_raft_snapshot_agent/config/resolve-path-tag_test.go b/internal/app/vault_raft_snapshot_agent/config/resolve-path-tag_test.go deleted file mode 100644 index b5c3fff..0000000 --- a/internal/app/vault_raft_snapshot_agent/config/resolve-path-tag_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package config - -import ( - "fmt" - "path/filepath" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestResolvesRelativePaths(t *testing.T) { - var test struct { - Path string `resolve-path:""` - FixedPath string `resolve-path:"/tmp/"` - Other string - AbsolutePath string `resolve-path:""` - } - test.Path = "./relative" - test.FixedPath = "./fixed" - test.Other = "./other" - test.AbsolutePath = "/test/abs" - - dir := t.TempDir() - resolver := newPathResolver(dir) - err := resolver.Resolve(&test) - - assert.NoError(t, err, "resolver.resolve failed unexepectedly") - - assert.Equal(t, filepath.Clean(fmt.Sprintf("%s/relative", dir)), test.Path) - assert.Equal(t, filepath.Clean("/tmp/fixed"), test.FixedPath) - assert.Equal(t, "/test/abs", test.AbsolutePath) - assert.Equal(t, "./other", test.Other) -} - -func TestResolvesRecursively(t *testing.T) { - type inner struct { - Path string `resolve-path:""` - } - - innerPtr := inner{"./innerPtr"} - - var outer struct { - Inner inner - InnerPtr *inner - } - outer.Inner.Path = "./inner" - outer.InnerPtr = &innerPtr - - dir := t.TempDir() - resolver := newPathResolver(dir) - err := resolver.Resolve(&outer) - - assert.NoError(t, err, "resolver.resolve failed unexepectedly") - - assert.Equal(t, filepath.Clean(fmt.Sprintf("%s/inner", dir)), outer.Inner.Path) - assert.Equal(t, filepath.Clean(fmt.Sprintf("%s/innerPtr", dir)), innerPtr.Path) - -} diff --git a/internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths.go b/internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths.go new file mode 100644 index 0000000..61c6f2e --- /dev/null +++ b/internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths.go @@ -0,0 +1,61 @@ +package secret + +import ( + "errors" + "reflect" + "strings" +) + +var ( + errorInvalidType = errors.New("subject must be a struct passed by pointer") + secretType = reflect.TypeOf(Secret("")) +) + +func ResolveFilePaths(subject interface{}, baseDir string) error { + if baseDir == "" { + return nil + } + + if reflect.TypeOf(subject).Kind() != reflect.Ptr { + return errorInvalidType + } + + s := reflect.ValueOf(subject).Elem() + + return resolveSecretFilePaths(s, baseDir) +} + +func resolveSecretFilePaths(value reflect.Value, baseDir string) error { + t := value.Type() + + if t.Kind() != reflect.Struct { + return errorInvalidType + } + + for i := 0; i < t.NumField(); i++ { + f := value.Field(i) + + if !f.CanSet() { + continue + } + + if f.Kind() == reflect.Ptr { + f = f.Elem() + } + + if f.Kind() == reflect.Struct { + if err := resolveSecretFilePaths(f, baseDir); err != nil { + return err + } + } + + if f.Type() != secretType || !strings.HasPrefix(f.String(), filePrefix) { + continue + } + + secret := f.Convert(secretType).MethodByName("WithAbsoluteFilePath").Call([]reflect.Value{reflect.ValueOf(baseDir)}) + f.Set(secret[0].Convert(secretType)) + } + + return nil +} diff --git a/internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths_test.go b/internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths_test.go new file mode 100644 index 0000000..2ea09c0 --- /dev/null +++ b/internal/app/vault_raft_snapshot_agent/secret/resolve-file-paths_test.go @@ -0,0 +1,47 @@ +package secret + +import ( + "fmt" + "github.com/stretchr/testify/assert" + "path/filepath" + "testing" +) + +func TestResolvesRelativePaths(t *testing.T) { + var test struct { + File Secret + Plain Secret + } + test.File = FromFile("./file") + test.Plain = FromString("./plain") + + dir := t.TempDir() + err := ResolveFilePaths(&test, dir) + + assert.NoError(t, err, "ResolveSecretFilePath failed unexpectedly") + + assert.Equal(t, FromFile(filepath.Clean(fmt.Sprintf("%s/file", dir))), test.File) + assert.Equal(t, FromString("./plain"), test.Plain) +} + +func TestResolvesRecursively(t *testing.T) { + type inner struct { + File Secret + } + + innerPtr := inner{FromFile("./innerPtr")} + + var outer struct { + Inner inner + InnerPtr *inner + } + outer.Inner.File = FromFile("./inner") + outer.InnerPtr = &innerPtr + + dir := t.TempDir() + err := ResolveFilePaths(&outer, dir) + assert.NoError(t, err, "ResolveSecretFilePath failed unexpectedly") + + assert.Equal(t, FromFile(filepath.Clean(fmt.Sprintf("%s/inner", dir))), outer.Inner.File) + assert.Equal(t, FromFile(filepath.Clean(fmt.Sprintf("%s/innerPtr", dir))), innerPtr.File) +} diff --git a/internal/app/vault_raft_snapshot_agent/secret/secret.go b/internal/app/vault_raft_snapshot_agent/secret/secret.go new file mode 100644 index 0000000..141bb89 --- /dev/null +++ b/internal/app/vault_raft_snapshot_agent/secret/secret.go @@ -0,0 +1,93 @@ +package secret + +import ( + "fmt" + "log" + "os" + "path/filepath" + "strings" +) + +type Secret string + +const ( + Zero = Secret("") + envPrefix = "env://" + filePrefix = "file://" +) + +func FromEnv(varName string) Secret { + return withPrefix(envPrefix, varName) +} + +func FromFile(file string) Secret { + return withPrefix(filePrefix, file) +} + +func FromString(value string) Secret { + return Secret(value) +} + +func withPrefix(prefix string, value string) Secret { + return FromString(prefix + value) +} + +func (s Secret) String() string { + v, err := s.Resolve(false) + if err != nil { + log.Panicf("could not resolve %s: %s", string(s), err) + } + + return v +} + +func (s Secret) IsZero() bool { + return s == Zero +} + +func (s Secret) Resolve(required bool) (string, error) { + v := string(s) + + if strings.HasPrefix(v, envPrefix) { + name := strings.TrimPrefix(v, envPrefix) + value, present := os.LookupEnv(name) + if !present && required { + return "", fmt.Errorf("environment variable %s is not present", name) + } + return value, nil + } + + if strings.HasPrefix(v, filePrefix) { + file := strings.TrimPrefix(v, filePrefix) + value, err := os.ReadFile(file) + if err != nil && (!os.IsNotExist(err) || required) { + return "", fmt.Errorf("could not read file %s", file) + } + return string(value), nil + } + + return v, nil +} + +func (s Secret) WithAbsoluteFilePath(baseDir string) Secret { + if baseDir == "" { + return s + } + + v := string(s) + if !strings.HasPrefix(v, filePrefix) { + return s + } + + file := strings.TrimPrefix(v, filePrefix) + if filepath.IsAbs(file) || strings.HasPrefix(file, "/") { + return s + } + + file = filepath.Join(baseDir, file) + return FromFile(file) +} + +func (s Secret) SetDefaults() { + fmt.Println("setting secret-defaults") +} diff --git a/internal/app/vault_raft_snapshot_agent/secret/secret_test.go b/internal/app/vault_raft_snapshot_agent/secret/secret_test.go new file mode 100644 index 0000000..9f73ba1 --- /dev/null +++ b/internal/app/vault_raft_snapshot_agent/secret/secret_test.go @@ -0,0 +1,107 @@ +package secret + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/test" +) + +func TestSecretResolvesEnvironmentVariable(t *testing.T) { + t.Setenv("TEST", "resolved") + + secret := FromEnv("TEST") + v, err := secret.Resolve(true) + + assert.NoError(t, err, "resolve failed unexpectedly") + assert.Equal(t, os.Getenv("TEST"), v) +} + +func TestSecretResolvesFile(t *testing.T) { + secretFile := fmt.Sprintf("%s/secret", t.TempDir()) + err := test.WriteFile(t, secretFile, "secret") + assert.NoError(t, err, "could not write file %s", secretFile) + + secret := FromFile(secretFile) + v, err := secret.Resolve(true) + + assert.NoError(t, err, "resolve failed unexpectedly") + assert.Equal(t, "secret", v) +} + +func TestSecretResolvesPlainString(t *testing.T) { + secret := Secret("plain") + v, err := secret.Resolve(true) + + assert.NoError(t, err, "resolve failed unexpectedly") + assert.Equal(t, "plain", v) +} + +func TestRequiredResolveFailsIfEnvVarIsMissing(t *testing.T) { + secret := FromEnv("TEST") + v, err := secret.Resolve(true) + + assert.Error(t, err, "resolve should fail if environment-variable is missing") + assert.Zero(t, v) +} + +func TestOptionalResolveReturnsEmptyIfEnvVarIsMissing(t *testing.T) { + secret := FromEnv("TEST") + v, err := secret.Resolve(false) + + assert.NoError(t, err, "resolve should not fail for missing environment-variable when not required") + assert.Zero(t, v) +} + +func TestRequiredResolveFailsIfFileCanNotBeRead(t *testing.T) { + secret := FromFile("/missing/file") + v, err := secret.Resolve(true) + + assert.Error(t, err, "resolve should fail if file can not be read") + assert.Zero(t, v) +} + +func TestOptionalResolveReturnsEmptyIfFileCanNotBeRead(t *testing.T) { + secret := FromFile("/missing/file") + v, err := secret.Resolve(false) + + assert.NoError(t, err, "resolve should not fail for missing file when not required") + assert.Zero(t, v) +} + +func TestStringResolvesSecret(t *testing.T) { + t.Setenv("TEST", "resolved") + + secret := FromEnv("TEST") + assert.Equal(t, os.Getenv("TEST"), secret.String()) +} + +func TestStringReturnsEmptyIfSecretCanNotBeResolved(t *testing.T) { + assert.Equal(t, "", FromEnv("TEST").String()) +} + +func TestWithAbsoluteFilePathResolvesRelativeFilePath(t *testing.T) { + baseDir := t.TempDir() + secret := FromFile("./test") + + assert.Equal(t, FromFile(filepath.Clean(fmt.Sprintf("%s/test", baseDir))), secret.WithAbsoluteFilePath(baseDir)) +} + +func TestWithAbsoluteFileReturnsForEmptyBaseDir(t *testing.T) { + secret := FromFile("./test") + assert.Equal(t, secret, secret.WithAbsoluteFilePath("")) +} + +func TestWithAbsoluteFileReturnsForEnvSecret(t *testing.T) { + secret := FromEnv("test") + assert.Equal(t, secret, secret.WithAbsoluteFilePath(t.TempDir())) +} + +func TestWithAbsoluteFileReturnsForPlainSecret(t *testing.T) { + secret := FromString("test") + assert.Equal(t, secret, secret.WithAbsoluteFilePath(t.TempDir())) +} diff --git a/internal/app/vault_raft_snapshot_agent/snapshotter.go b/internal/app/vault_raft_snapshot_agent/snapshotter.go index 9a9cd48..4ca8b8e 100644 --- a/internal/app/vault_raft_snapshot_agent/snapshotter.go +++ b/internal/app/vault_raft_snapshot_agent/snapshotter.go @@ -16,7 +16,7 @@ import ( ) type SnapshotterConfig struct { - Vault vault.VaultClientConfig + Vault vault.ClientConfig Snapshots SnapshotConfig Uploaders upload.UploadersConfig } @@ -54,7 +54,7 @@ type snapshotterVaultAPI interface { TakeSnapshot(ctx context.Context, writer io.Writer) error } -func CreateSnapshotter(ctx context.Context, options SnapshotterOptions) (*Snapshotter, error) { +func CreateSnapshotter(options SnapshotterOptions) (*Snapshotter, error) { data := SnapshotterConfig{} parser := config.NewParser[*SnapshotterConfig](options.ConfigFileName, options.EnvPrefix, options.ConfigFileSearchPaths...) @@ -62,7 +62,7 @@ func CreateSnapshotter(ctx context.Context, options SnapshotterOptions) (*Snapsh return nil, err } - snapshotter, err := createSnapshotter(ctx, data) + snapshotter, err := createSnapshotter(data) if err != nil { return nil, err } @@ -70,7 +70,7 @@ func CreateSnapshotter(ctx context.Context, options SnapshotterOptions) (*Snapsh parser.OnConfigChange( &SnapshotterConfig{}, func(config *SnapshotterConfig) error { - if err := snapshotter.reconfigure(ctx, *config); err != nil { + if err := snapshotter.reconfigure(*config); err != nil { log.Printf("could not reconfigure snapshotter: %s\n", err) return err } @@ -81,20 +81,20 @@ func CreateSnapshotter(ctx context.Context, options SnapshotterOptions) (*Snapsh return snapshotter, nil } -func createSnapshotter(ctx context.Context, config SnapshotterConfig) (*Snapshotter, error) { +func createSnapshotter(config SnapshotterConfig) (*Snapshotter, error) { snapshotter := &Snapshotter{} - err := snapshotter.reconfigure(ctx, config) + err := snapshotter.reconfigure(config) return snapshotter, err } -func (s *Snapshotter) reconfigure(ctx context.Context, config SnapshotterConfig) error { - client, err := vault.CreateVaultClient(config.Vault) +func (s *Snapshotter) reconfigure(config SnapshotterConfig) error { + client, err := vault.CreateClient(config.Vault) if err != nil { return err } - uploaders, err := upload.CreateUploaders(ctx, config.Uploaders) + uploaders, err := upload.CreateUploaders(config.Uploaders) if err != nil { return err } @@ -124,7 +124,13 @@ func (s *Snapshotter) TakeSnapshot(ctx context.Context) (*time.Timer, error) { return s.snapshotTimer, err } - defer os.Remove(snapshot.Name()) + defer func() { + if err := snapshot.Close(); err != nil { + fmt.Printf("could not close snapshot-temp-file %s: %s\n", snapshot.Name(), err) + } else if err := os.Remove(snapshot.Name()); err != nil { + fmt.Printf("could not remove snapshot-temp-file %s: %s\n", snapshot.Name(), err) + } + }() ctx, cancel := context.WithTimeout(ctx, s.config.Timeout) defer cancel() diff --git a/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go b/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go index e9e031f..864be22 100644 --- a/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go +++ b/internal/app/vault_raft_snapshot_agent/snapshotter_config_test.go @@ -1,9 +1,7 @@ package vault_raft_snapshot_agent import ( - "errors" - "log" - "os" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "path/filepath" "strings" "testing" @@ -16,20 +14,6 @@ import ( "github.com/stretchr/testify/assert" ) -// allow overiding "default" kubernetes-jwt-path so that tests on ci do not fail -func defaultJwtPath(def string) string { - jwtPath := os.Getenv("VRSA_VAULT_AUTH_KUBERNETES_JWTPATH") - if jwtPath != "" { - return jwtPath - } - - if def != "" { - return def - } - - return "/var/run/secrets/kubernetes.io/serviceaccount/token" -} - func relativeTo(configFile string, file string) string { if !filepath.IsAbs(file) && !strings.HasPrefix(file, "/") { file = filepath.Join(filepath.Dir(configFile), file) @@ -47,11 +31,11 @@ func TestReadCompleteConfig(t *testing.T) { configFile := "../../../testdata/complete.yaml" expectedConfig := SnapshotterConfig{ - Vault: vault.VaultClientConfig{ + Vault: vault.ClientConfig{ Url: "https://example.com:8200", Insecure: true, Timeout: 5 * time.Minute, - Auth: auth.AuthConfig{ + Auth: auth.VaultAuthConfig{ AppRole: auth.AppRoleAuthConfig{ Path: "test-approle-path", RoleId: "test-approle", @@ -75,9 +59,9 @@ func TestReadCompleteConfig(t *testing.T) { ServiceAccountEmail: "test@example.com", }, Kubernetes: auth.KubernetesAuthConfig{ - Role: "test-kubernetes-role", - Path: "test-kubernetes-path", - JWTPath: relativeTo(configFile, defaultJwtPath("./jwt")), + Role: "test-kubernetes-role", + Path: "test-kubernetes-path", + JWTToken: secret.FromFile(relativeTo(configFile, "./jwt")), }, LDAP: auth.LDAPAuthConfig{ Path: "test-ldap-path", @@ -102,22 +86,21 @@ func TestReadCompleteConfig(t *testing.T) { }, Uploaders: upload.UploadersConfig{ AWS: upload.AWSUploaderConfig{ + AccessKeyId: "test-key", + AccessKey: "test-secret", + SessionToken: "test-session", Endpoint: "test-endpoint", Region: "test-region", Bucket: "test-bucket", KeyPrefix: "test-prefix", UseServerSideEncryption: true, ForcePathStyle: true, - Credentials: upload.AWSUploaderCredentialsConfig{ - Key: "test-key", - Secret: "test-secret", - }, }, Azure: upload.AzureUploaderConfig{ - AccountName: "test-account", - AccountKey: "test-key", - ContainerName: "test-container", - CloudDomain: "blob.core.chinacloudapi.cn", + AccountName: "test-account", + AccountKey: "test-key", + Container: "test-container", + CloudDomain: "blob.core.chinacloudapi.cn", }, GCP: upload.GCPUploaderConfig{ Bucket: "test-bucket", @@ -129,8 +112,8 @@ func TestReadCompleteConfig(t *testing.T) { Container: "test-container", UserName: "test-username", ApiKey: "test-api-key", - AuthUrl: "http://auth.com", - Domain: "http://user.com", + AuthUrl: "https://auth.com", + Domain: "https://user.com", Region: "test-region", TenantId: "test-tenant", Timeout: 180 * time.Second, @@ -150,11 +133,11 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { configFile := "../../../testdata/defaults.yaml" expectedConfig := SnapshotterConfig{ - Vault: vault.VaultClientConfig{ + Vault: vault.ClientConfig{ Url: "http://127.0.0.1:8200", Insecure: false, Timeout: time.Minute, - Auth: auth.AuthConfig{ + Auth: auth.VaultAuthConfig{ AppRole: auth.AppRoleAuthConfig{ Path: "approle", Empty: true, @@ -162,6 +145,7 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { AWS: auth.AWSAuthConfig{ Path: "aws", EC2SignatureType: auth.AWS_EC2_PKCS7, + Region: secret.FromEnv("AWS_DEFAULT_REGION"), Empty: true, }, Azure: auth.AzureAuthConfig{ @@ -173,9 +157,9 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { Empty: true, }, Kubernetes: auth.KubernetesAuthConfig{ - Role: "test-role", - Path: "kubernetes", - JWTPath: relativeTo(configFile, defaultJwtPath("")), + Role: "test-role", + Path: "kubernetes", + JWTToken: secret.FromFile(relativeTo(configFile, "./jwt")), }, LDAP: auth.LDAPAuthConfig{ Path: "ldap", @@ -197,10 +181,16 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { }, Uploaders: upload.UploadersConfig{ AWS: upload.AWSUploaderConfig{ - Credentials: upload.AWSUploaderCredentialsConfig{Empty: true}, - Empty: true, + AccessKeyId: secret.FromEnv("AWS_ACCESS_KEY_ID"), + AccessKey: secret.FromEnv("AWS_SECRET_ACCESS_KEY"), + SessionToken: secret.FromEnv("AWS_SESSION_TOKEN"), + Region: secret.FromEnv("AWS_DEFAULT_REGION"), + Endpoint: secret.FromEnv("AWS_ENDPOINT_URL"), + Empty: true, }, Azure: upload.AzureUploaderConfig{ + AccountName: secret.FromEnv("AZURE_STORAGE_ACCOUNT"), + AccountKey: secret.FromEnv("AZURE_STORAGE_KEY"), CloudDomain: "blob.core.windows.net", Empty: true, }, @@ -209,8 +199,11 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { Path: ".", }, Swift: upload.SwiftUploaderConfig{ - Timeout: time.Minute, - Empty: true, + UserName: secret.FromEnv("SWIFT_USERNAME"), + ApiKey: secret.FromEnv("SWIFT_API_KEY"), + Region: secret.FromEnv("SWIFT_REGION"), + Timeout: time.Minute, + Empty: true, }, }, } @@ -222,21 +215,3 @@ func TestReadConfigSetsDefaultValues(t *testing.T) { assert.NoError(t, err, "ReadConfig(%s) failed unexpectedly", configFile) assert.Equal(t, expectedConfig, data) } - -func init() { - jwtPath := defaultJwtPath("") - if err := os.MkdirAll(filepath.Dir(jwtPath), 0777); err != nil && !errors.Is(err, os.ErrExist) { - log.Fatalf("could not create directorys for jwt-file %s: %v", jwtPath, err) - } - - file, err := os.OpenFile(jwtPath, os.O_RDWR|os.O_CREATE, 0666) - if err != nil { - log.Fatalf("could not create jwt-file %s: %v", jwtPath, err) - } - - file.Close() - - if err != nil { - log.Fatalf("could not read jwt-file %s: %v", jwtPath, err) - } -} diff --git a/internal/app/vault_raft_snapshot_agent/snapshotter_test.go b/internal/app/vault_raft_snapshot_agent/snapshotter_test.go index 49ca364..f756724 100644 --- a/internal/app/vault_raft_snapshot_agent/snapshotter_test.go +++ b/internal/app/vault_raft_snapshot_agent/snapshotter_test.go @@ -346,13 +346,13 @@ func TestSnapshotterTriggersTimerOnConfigureForLesserFrequency(t *testing.T) { assert.Equal(t, newConfig.Frequency, snapshotter.config.Frequency) } -func newClient(api *clientVaultAPIStub) *vault.VaultClient[any, clientVaultAPIAuthStub] { - return vault.NewVaultClient[any](api, clientVaultAPIAuthStub{}, time.Time{}) +func newClient(api *clientVaultAPIStub) *vault.Client[any, clientVaultAPIAuthStub] { + return vault.NewClient[any, clientVaultAPIAuthStub](api, clientVaultAPIAuthStub{}, time.Time{}) } type clientVaultAPIAuthStub struct{} -func (stub clientVaultAPIAuthStub) Login(ctx context.Context, api any) (time.Duration, error) { +func (stub clientVaultAPIAuthStub) Login(_ context.Context, _ any) (time.Duration, error) { return 0, nil } @@ -402,7 +402,7 @@ func (stub *uploaderStub) Destination() string { return "" } -func (stub *uploaderStub) Upload(ctx context.Context, reader io.Reader, prefix string, timestamp string, suffix string, retain int) error { +func (stub *uploaderStub) Upload(_ context.Context, reader io.Reader, prefix string, timestamp string, suffix string, _ int) error { stub.uploaded = true if stub.uploadFails { return errors.New("upload failed") diff --git a/internal/app/vault_raft_snapshot_agent/upload/aws.go b/internal/app/vault_raft_snapshot_agent/upload/aws.go index 38140ac..b8c50a2 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/aws.go +++ b/internal/app/vault_raft_snapshot_agent/upload/aws.go @@ -3,6 +3,7 @@ package upload import ( "context" "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "io" "strings" @@ -15,72 +16,96 @@ import ( ) type AWSUploaderConfig struct { - Credentials AWSUploaderCredentialsConfig `default:"{\"Empty\": true}"` - Bucket string `validate:"required_if=Empty false"` - KeyPrefix string `mapstructure:",omitifempty"` - Endpoint string `mapstructure:",omitifempty"` - Region string + AccessKeyId secret.Secret `default:"env://AWS_ACCESS_KEY_ID"` + AccessKey secret.Secret `default:"env://AWS_SECRET_ACCESS_KEY" validate:"required_with=AccessKeyId"` + SessionToken secret.Secret `default:"env://AWS_SESSION_TOKEN"` + Region secret.Secret `default:"env://AWS_DEFAULT_REGION"` + Endpoint secret.Secret `default:"env://AWS_ENDPOINT_URL"` + Bucket string `validate:"required_if=Empty false"` + KeyPrefix string `mapstructure:",omitifempty"` UseServerSideEncryption bool ForcePathStyle bool Empty bool } -type AWSUploaderCredentialsConfig struct { - Key string `validate:"required_if=Empty false"` - Secret string `validate:"required_if=Empty false"` - Empty bool -} - type awsUploaderImpl struct { - client *s3.Client - uploader *manager.Uploader keyPrefix string bucket string sse bool } -func createAWSUploader(ctx context.Context, config AWSUploaderConfig) (*uploader[s3Types.Object], error) { - clientConfig, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithRegion(config.Region)) +func createAWSUploader(config AWSUploaderConfig) uploader[AWSUploaderConfig, *s3.Client, s3Types.Object] { + keyPrefix := "" + if config.KeyPrefix != "" { + keyPrefix = fmt.Sprintf("%s/", config.KeyPrefix) + } + + return uploader[AWSUploaderConfig, *s3.Client, s3Types.Object]{ + config, + awsUploaderImpl{ + keyPrefix: keyPrefix, + bucket: config.Bucket, + sse: config.UseServerSideEncryption, + }, + } +} + +// nolint:unused +// implements interface uploaderImpl +func (u awsUploaderImpl) destination(config AWSUploaderConfig) string { + return fmt.Sprintf("aws s3 bucket %s ", config.Bucket) +} + +// nolint:unused +// implements interface uploaderImpl +func (u awsUploaderImpl) connect(ctx context.Context, config AWSUploaderConfig) (*s3.Client, error) { + accessKeyId, err := config.AccessKeyId.Resolve(false) + if err != nil { + return nil, err + } + accessKey, err := config.AccessKey.Resolve(accessKeyId != "") + if err != nil { + return nil, err + } + + sessionToken, err := config.SessionToken.Resolve(false) + if err != nil { + return nil, err + } + + region, err := config.Region.Resolve(false) + if err != nil { + return nil, err + } + + clientConfig, err := awsConfig.LoadDefaultConfig(ctx, awsConfig.WithRegion(region)) if err != nil { return nil, fmt.Errorf("failed to load default aws config: %w", err) } - if !config.Credentials.Empty { - creds := credentials.NewStaticCredentialsProvider(config.Credentials.Key, config.Credentials.Secret, "") - clientConfig.Credentials = creds + if accessKeyId != "" { + clientConfig.Credentials = credentials.NewStaticCredentialsProvider(accessKeyId, accessKey, sessionToken) + } + + endpoint, err := config.Endpoint.Resolve(false) + if err != nil { + return nil, err } client := s3.NewFromConfig(clientConfig, func(o *s3.Options) { o.UsePathStyle = config.ForcePathStyle if config.Endpoint != "" { - o.BaseEndpoint = aws.String(config.Endpoint) + o.BaseEndpoint = aws.String(endpoint) } }) - keyPrefix := "" - if config.KeyPrefix != "" { - keyPrefix = fmt.Sprintf("%s/", config.KeyPrefix) - } - - return &uploader[s3Types.Object]{ - awsUploaderImpl{ - client: client, - uploader: manager.NewUploader(client), - keyPrefix: keyPrefix, - bucket: config.Bucket, - sse: config.UseServerSideEncryption, - }, - }, nil -} - -func (u awsUploaderImpl) Destination() string { - return fmt.Sprintf("aws s3 bucket %s ", u.bucket) + return client, nil } // nolint:unused // implements interface uploaderImpl -func (u awsUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (u awsUploaderImpl) uploadSnapshot(ctx context.Context, client *s3.Client, name string, data io.Reader) error { input := &s3.PutObjectInput{ Bucket: &u.bucket, Key: aws.String(u.keyPrefix + name), @@ -91,7 +116,8 @@ func (u awsUploaderImpl) uploadSnapshot(ctx context.Context, name string, data i input.ServerSideEncryption = s3Types.ServerSideEncryptionAes256 } - if _, err := u.uploader.Upload(ctx, input); err != nil { + uploader := manager.NewUploader(client) + if _, err := uploader.Upload(ctx, input); err != nil { return err } @@ -100,13 +126,13 @@ func (u awsUploaderImpl) uploadSnapshot(ctx context.Context, name string, data i // nolint:unused // implements interface uploaderImpl -func (u awsUploaderImpl) deleteSnapshot(ctx context.Context, snapshot s3Types.Object) error { +func (u awsUploaderImpl) deleteSnapshot(ctx context.Context, client *s3.Client, snapshot s3Types.Object) error { input := &s3.DeleteObjectInput{ Bucket: &u.bucket, Key: snapshot.Key, } - if _, err := u.client.DeleteObject(ctx, input); err != nil { + if _, err := client.DeleteObject(ctx, input); err != nil { return err } @@ -115,10 +141,10 @@ func (u awsUploaderImpl) deleteSnapshot(ctx context.Context, snapshot s3Types.Ob // nolint:unused // implements interface uploaderImpl -func (u awsUploaderImpl) listSnapshots(ctx context.Context, prefix string, ext string) ([]s3Types.Object, error) { +func (u awsUploaderImpl) listSnapshots(ctx context.Context, client *s3.Client, prefix string, ext string) ([]s3Types.Object, error) { var result []s3Types.Object - existingSnapshotList, err := u.client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ + existingSnapshotList, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: &u.bucket, Prefix: aws.String(u.keyPrefix), }) diff --git a/internal/app/vault_raft_snapshot_agent/upload/azure.go b/internal/app/vault_raft_snapshot_agent/upload/azure.go index 720cb07..d9b42ed 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/azure.go +++ b/internal/app/vault_raft_snapshot_agent/upload/azure.go @@ -3,6 +3,7 @@ package upload import ( "context" "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "io" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" @@ -10,51 +11,60 @@ import ( ) type AzureUploaderConfig struct { - AccountName string `validate:"required_if=Empty false"` - AccountKey string `validate:"required_if=Empty false"` - ContainerName string `mapstructure:"container" validate:"required_if=Empty false"` - CloudDomain string `default:"blob.core.windows.net" validate:"required_if=Empty false"` - Empty bool + AccountName secret.Secret `default:"env://AZURE_STORAGE_ACCOUNT" validate:"required_if=Empty false"` + AccountKey secret.Secret `default:"env://AZURE_STORAGE_KEY" validate:"required_if=Empty false"` + Container string `validate:"required_if=Empty false"` + CloudDomain string `default:"blob.core.windows.net" validate:"required_if=Empty false"` + Empty bool } type azureUploaderImpl struct { - client *azblob.Client container string } -func createAzureUploader(ctx context.Context, config AzureUploaderConfig) (*uploader[*container.BlobItem], error) { - credential, err := azblob.NewSharedKeyCredential(config.AccountName, config.AccountKey) +func createAzureUploader(config AzureUploaderConfig) uploader[AzureUploaderConfig, *azblob.Client, *container.BlobItem] { + return uploader[AzureUploaderConfig, *azblob.Client, *container.BlobItem]{ + config, azureUploaderImpl{config.Container}, + } +} + +// nolint:unused +// implements interface uploaderImpl +func (u azureUploaderImpl) destination(config AzureUploaderConfig) string { + return fmt.Sprintf("azure container %s at %s", config.Container, config.CloudDomain) +} + +// nolint:unused +// implements interface uploaderImpl +func (u azureUploaderImpl) connect(_ context.Context, config AzureUploaderConfig) (*azblob.Client, error) { + accountName, err := config.AccountName.Resolve(true) if err != nil { - return nil, fmt.Errorf("invalid credentials for azure: %w", err) + return nil, err } - serviceURL := fmt.Sprintf("https://%s.%s/", config.AccountName, config.CloudDomain) - client, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, nil) + accountKey, err := config.AccountName.Resolve(true) if err != nil { - return nil, fmt.Errorf("failed to create azure client: %w", err) + return nil, err } - return &uploader[*container.BlobItem]{ - azureUploaderImpl{ - client: client, - container: config.ContainerName, - }, - }, nil -} + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + return nil, fmt.Errorf("invalid credentials for azure: %w", err) + } -func (u azureUploaderImpl) Destination() string { - return fmt.Sprintf("azure container %s", u.container) + serviceURL := fmt.Sprintf("https://%s.%s/", config.AccountName, config.CloudDomain) + return azblob.NewClientWithSharedKeyCredential(serviceURL, credential, nil) } // nolint:unused // implements interface uploaderImpl -func (u azureUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (u azureUploaderImpl) uploadSnapshot(ctx context.Context, client *azblob.Client, name string, data io.Reader) error { uploadOptions := &azblob.UploadStreamOptions{ BlockSize: 4 * 1024 * 1024, Concurrency: 16, } - if _, err := u.client.UploadStream(ctx, u.container, name, data, uploadOptions); err != nil { + if _, err := client.UploadStream(ctx, u.container, name, data, uploadOptions); err != nil { return err } @@ -63,8 +73,8 @@ func (u azureUploaderImpl) uploadSnapshot(ctx context.Context, name string, data // nolint:unused // implements interface uploaderImpl -func (u azureUploaderImpl) deleteSnapshot(ctx context.Context, snapshot *container.BlobItem) error { - if _, err := u.client.DeleteBlob(ctx, u.container, *snapshot.Name, nil); err != nil { +func (u azureUploaderImpl) deleteSnapshot(ctx context.Context, client *azblob.Client, snapshot *container.BlobItem) error { + if _, err := client.DeleteBlob(ctx, u.container, *snapshot.Name, nil); err != nil { return err } @@ -73,12 +83,12 @@ func (u azureUploaderImpl) deleteSnapshot(ctx context.Context, snapshot *contain // nolint:unused // implements interface uploaderImpl -func (u azureUploaderImpl) listSnapshots(ctx context.Context, prefix string, ext string) ([]*container.BlobItem, error) { +func (u azureUploaderImpl) listSnapshots(ctx context.Context, client *azblob.Client, prefix string, _ string) ([]*container.BlobItem, error) { var results []*container.BlobItem var maxResults int32 = 500 - pager := u.client.NewListBlobsFlatPager(u.container, &azblob.ListBlobsFlatOptions{ + pager := client.NewListBlobsFlatPager(u.container, &azblob.ListBlobsFlatOptions{ Prefix: &prefix, MaxResults: &maxResults, }) diff --git a/internal/app/vault_raft_snapshot_agent/upload/gcp.go b/internal/app/vault_raft_snapshot_agent/upload/gcp.go index 18a59c0..1583423 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/gcp.go +++ b/internal/app/vault_raft_snapshot_agent/upload/gcp.go @@ -14,33 +14,33 @@ type GCPUploaderConfig struct { Empty bool } -type gcpUploaderImpl struct { - destination string - bucket *storage.BucketHandle +type gcpUploaderImpl struct{} + +func createGCPUploader(config GCPUploaderConfig) uploader[GCPUploaderConfig, *storage.BucketHandle, storage.ObjectAttrs] { + return uploader[GCPUploaderConfig, *storage.BucketHandle, storage.ObjectAttrs]{config, gcpUploaderImpl{}} +} + +// nolint:unused +// implements interface uploaderImpl +func (u gcpUploaderImpl) destination(config GCPUploaderConfig) string { + return fmt.Sprintf("gcp bucket %s", config.Bucket) } -func createGCPUploader(ctx context.Context, config GCPUploaderConfig) (*uploader[storage.ObjectAttrs], error) { +// nolint:unused +// implements interface uploaderImpl +func (u gcpUploaderImpl) connect(ctx context.Context, config GCPUploaderConfig) (*storage.BucketHandle, error) { client, err := storage.NewClient(ctx) if err != nil { return nil, err } - return &uploader[storage.ObjectAttrs]{ - gcpUploaderImpl{ - destination: fmt.Sprintf("gcp bucket %s", config.Bucket), - bucket: client.Bucket(config.Bucket), - }, - }, nil -} - -func (u gcpUploaderImpl) Destination() string { - return u.destination + return client.Bucket(config.Bucket), nil } // nolint:unused // implements interface uploaderImpl -func (u gcpUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { - obj := u.bucket.Object(name) +func (u gcpUploaderImpl) uploadSnapshot(ctx context.Context, client *storage.BucketHandle, name string, data io.Reader) error { + obj := client.Object(name) w := obj.NewWriter(ctx) if _, err := io.Copy(w, data); err != nil { @@ -56,8 +56,8 @@ func (u gcpUploaderImpl) uploadSnapshot(ctx context.Context, name string, data i // nolint:unused // implements interface uploaderImpl -func (u gcpUploaderImpl) deleteSnapshot(ctx context.Context, snapshot storage.ObjectAttrs) error { - obj := u.bucket.Object(snapshot.Name) +func (u gcpUploaderImpl) deleteSnapshot(ctx context.Context, client *storage.BucketHandle, snapshot storage.ObjectAttrs) error { + obj := client.Object(snapshot.Name) if err := obj.Delete(ctx); err != nil { return err } @@ -67,11 +67,11 @@ func (u gcpUploaderImpl) deleteSnapshot(ctx context.Context, snapshot storage.Ob // nolint:unused // implements interface uploaderImpl -func (u gcpUploaderImpl) listSnapshots(ctx context.Context, prefix string, ext string) ([]storage.ObjectAttrs, error) { +func (u gcpUploaderImpl) listSnapshots(ctx context.Context, client *storage.BucketHandle, prefix string, _ string) ([]storage.ObjectAttrs, error) { var result []storage.ObjectAttrs query := &storage.Query{Prefix: prefix} - it := u.bucket.Objects(ctx, query) + it := client.Objects(ctx, query) for { attrs, err := it.Next() diff --git a/internal/app/vault_raft_snapshot_agent/upload/local.go b/internal/app/vault_raft_snapshot_agent/upload/local.go index 0029216..edf551a 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/local.go +++ b/internal/app/vault_raft_snapshot_agent/upload/local.go @@ -3,6 +3,7 @@ package upload import ( "context" "fmt" + "go.uber.org/multierr" "io" "os" "strings" @@ -17,19 +18,28 @@ type localUploaderImpl struct { path string } -func createLocalUploader(ctx context.Context, config LocalUploaderConfig) (uploader[os.FileInfo], error) { - return uploader[os.FileInfo]{ +func createLocalUploader(config LocalUploaderConfig) uploader[LocalUploaderConfig, any, os.FileInfo] { + return uploader[LocalUploaderConfig, any, os.FileInfo]{ + config, localUploaderImpl{ path: config.Path, }, - }, nil + } +} + +// nolint:unused +// implements interface uploaderImpl +func (u localUploaderImpl) destination(config LocalUploaderConfig) string { + return fmt.Sprintf("local path %s", config.Path) } -func (u localUploaderImpl) Destination() string { - return fmt.Sprintf("local path %s", u.path) +// nolint:unused +// implements interface uploaderImpl +func (u localUploaderImpl) connect(_ context.Context, _ LocalUploaderConfig) (any, error) { + return nil, nil } -func (u localUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { +func (u localUploaderImpl) uploadSnapshot(_ context.Context, _ any, name string, data io.Reader) error { fileName := fmt.Sprintf("%s/%s", u.path, name) file, err := os.Create(fileName) @@ -37,18 +47,12 @@ func (u localUploaderImpl) uploadSnapshot(ctx context.Context, name string, data return err } - defer func() { - _ = file.Close() - }() + _, err = io.Copy(file, data) - if _, err = io.Copy(file, data); err != nil { - return err - } - - return nil + return multierr.Append(err, file.Close()) } -func (u localUploaderImpl) deleteSnapshot(ctx context.Context, snapshot os.FileInfo) error { +func (u localUploaderImpl) deleteSnapshot(_ context.Context, _ any, snapshot os.FileInfo) error { if err := os.Remove(fmt.Sprintf("%s/%s", u.path, snapshot.Name())); err != nil { return err } @@ -56,7 +60,7 @@ func (u localUploaderImpl) deleteSnapshot(ctx context.Context, snapshot os.FileI return nil } -func (u localUploaderImpl) listSnapshots(ctx context.Context, prefix string, ext string) ([]os.FileInfo, error) { +func (u localUploaderImpl) listSnapshots(_ context.Context, _ any, prefix string, ext string) ([]os.FileInfo, error) { var snapshots []os.FileInfo files, err := os.ReadDir(u.path) diff --git a/internal/app/vault_raft_snapshot_agent/upload/local_test.go b/internal/app/vault_raft_snapshot_agent/upload/local_test.go index 915f743..b91b93c 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/local_test.go +++ b/internal/app/vault_raft_snapshot_agent/upload/local_test.go @@ -16,7 +16,8 @@ import ( ) func TestLocalDestination(t *testing.T) { - impl := localUploaderImpl{"/test"} + config := LocalUploaderConfig{Path: "/test"} + impl := createLocalUploader(config) assert.Equal(t, "local path /test", impl.Destination()) } @@ -24,7 +25,7 @@ func TestLocalDestination(t *testing.T) { func TestLocalUploadSnapshotFailsIfFileCannotBeCreated(t *testing.T) { impl := localUploaderImpl{"./does/not/exist"} - err := impl.uploadSnapshot(context.Background(), "test", &bytes.Buffer{}) + err := impl.uploadSnapshot(context.Background(), nil, "test", &bytes.Buffer{}) assert.Error(t, err, "uploadSnapshot() should fail if file could not be created!") } @@ -33,7 +34,7 @@ func TestLocalUploadeSnapshotCreatesFile(t *testing.T) { impl := localUploaderImpl{t.TempDir()} snapshotData := []byte("test") - err := impl.uploadSnapshot(context.Background(), "test.snap", bytes.NewReader(snapshotData)) + err := impl.uploadSnapshot(context.Background(), nil, "test.snap", bytes.NewReader(snapshotData)) assert.NoError(t, err, "uploadSnapshot() failed unexpectedly!") @@ -51,13 +52,13 @@ func TestLocalDeleteSnapshot(t *testing.T) { _ = os.RemoveAll(filepath.Dir(impl.path)) }() - err := impl.uploadSnapshot(context.Background(), "test.snap", bytes.NewReader(snapshotData)) + err := impl.uploadSnapshot(context.Background(), nil, "test.snap", bytes.NewReader(snapshotData)) assert.NoError(t, err, "uploadSnapshot() failed unexpectedly!") info, err := os.Stat(fmt.Sprintf("%s/test.snap", impl.path)) assert.NoError(t, err, "could not get info for snapshot: %v", err) - err = impl.deleteSnapshot(context.Background(), info) + err = impl.deleteSnapshot(context.Background(), nil, info) assert.NoError(t, err, "deleteSnapshot() failed unexpectedly!") _, err = os.Stat(fmt.Sprintf("%s/test.snap", impl.path)) @@ -68,12 +69,12 @@ func TestLocalDeleteSnapshot(t *testing.T) { func TestLocalListSnapshots(t *testing.T) { impl := localUploaderImpl{t.TempDir()} - var expectedSnaphotNames[]string + var expectedSnaphotNames []string for i := 0; i < 3; i++ { expectedSnaphotNames = append(expectedSnaphotNames, createEmptySnapshot(t, impl.path, "test", ".snap").Name()) } - listedSnapshots, err := impl.listSnapshots(context.Background(), "test", ".snap") + listedSnapshots, err := impl.listSnapshots(context.Background(), nil, "test", ".snap") listedSnapshotNames := funk.Map(listedSnapshots, func(s os.FileInfo) string { return s.Name() }) assert.NoError(t, err) diff --git a/internal/app/vault_raft_snapshot_agent/upload/swift.go b/internal/app/vault_raft_snapshot_agent/upload/swift.go index a2c5f06..96f76a4 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/swift.go +++ b/internal/app/vault_raft_snapshot_agent/upload/swift.go @@ -3,6 +3,7 @@ package upload import ( "context" "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "io" "time" @@ -10,28 +11,57 @@ import ( ) type SwiftUploaderConfig struct { - Container string `validate:"required_if=Empty false"` - UserName string `validate:"required_if=Empty false"` - ApiKey string `validate:"required_if=Empty false"` - AuthUrl string `validate:"required_if=Empty false,omitempty,http_url"` - Domain string `validate:"omitempty,http_url"` - Region string + Container string `validate:"required_if=Empty false"` + UserName secret.Secret `default:"env://SWIFT_USERNAME" validate:"required_if=Empty false"` + ApiKey secret.Secret `default:"env://SWIFT_API_KEY" valide:"required_if=Empty false"` + Region secret.Secret `default:"env://SWIFT_REGION"` + AuthUrl string `validate:"required_if=Empty false,omitempty,http_url"` + Domain string `validate:"omitempty,http_url"` TenantId string Timeout time.Duration `default:"60s"` Empty bool } type swiftUploaderImpl struct { - connection *swift.Connection - container string + container string } -func createSwiftUploader(ctx context.Context, config SwiftUploaderConfig) (*uploader[swift.Object], error) { +func createSwiftUploader(config SwiftUploaderConfig) uploader[SwiftUploaderConfig, *swift.Connection, swift.Object] { + return uploader[SwiftUploaderConfig, *swift.Connection, swift.Object]{ + config, + swiftUploaderImpl{config.Container}, + } +} + +// nolint:unused +// implements interface uploaderImpl +func (u swiftUploaderImpl) destination(config SwiftUploaderConfig) string { + return fmt.Sprintf("swift container %s", config.Container) +} + +// nolint:unused +// implements interface uploaderImpl +func (u swiftUploaderImpl) connect(ctx context.Context, config SwiftUploaderConfig) (*swift.Connection, error) { + username, err := config.UserName.Resolve(true) + if err != nil { + return nil, err + } + + apiKey, err := config.ApiKey.Resolve(true) + if err != nil { + return nil, err + } + + region, err := config.Region.Resolve(false) + if err != nil { + return nil, err + } + conn := swift.Connection{ - UserName: config.UserName, - ApiKey: config.ApiKey, + UserName: username, + ApiKey: apiKey, AuthUrl: config.AuthUrl, - Region: config.Region, + Region: region, TenantId: config.TenantId, Domain: config.Domain, Timeout: config.Timeout, @@ -45,29 +75,18 @@ func createSwiftUploader(ctx context.Context, config SwiftUploaderConfig) (*uplo return nil, fmt.Errorf("invalid container %s: %s", config.Container, err) } - return &uploader[swift.Object]{ - swiftUploaderImpl{ - connection: &conn, - container: config.Container, - }, - }, nil -} - -// nolint:unused -// implements interface uploaderImpl -func (u swiftUploaderImpl) Destination() string { - return fmt.Sprintf("swift container %s", u.container) + return &conn, nil } // nolint:unused // implements interface uploaderImpl -func (u swiftUploaderImpl) uploadSnapshot(ctx context.Context, name string, data io.Reader) error { - _, header, err := u.connection.Container(ctx, u.container) +func (u swiftUploaderImpl) uploadSnapshot(ctx context.Context, client *swift.Connection, name string, data io.Reader) error { + _, header, err := client.Container(ctx, u.container) if err != nil { return err } - object, err := u.connection.ObjectCreate(ctx, u.container, name, false, "", "", header) + object, err := client.ObjectCreate(ctx, u.container, name, false, "", "", header) if err != nil { return err } @@ -85,8 +104,8 @@ func (u swiftUploaderImpl) uploadSnapshot(ctx context.Context, name string, data // nolint:unused // implements interface uploaderImpl -func (u swiftUploaderImpl) deleteSnapshot(ctx context.Context, snapshot swift.Object) error { - if err := u.connection.ObjectDelete(ctx, u.container, snapshot.Name); err != nil { +func (u swiftUploaderImpl) deleteSnapshot(ctx context.Context, client *swift.Connection, snapshot swift.Object) error { + if err := client.ObjectDelete(ctx, u.container, snapshot.Name); err != nil { return err } @@ -95,12 +114,12 @@ func (u swiftUploaderImpl) deleteSnapshot(ctx context.Context, snapshot swift.Ob // nolint:unused // implements interface uploaderImpl -func (u swiftUploaderImpl) listSnapshots(ctx context.Context, prefix string, ext string) ([]swift.Object, error) { - return u.connection.ObjectsAll(ctx, u.container, &swift.ObjectsOpts{Prefix: prefix}) +func (u swiftUploaderImpl) listSnapshots(ctx context.Context, client *swift.Connection, prefix string, _ string) ([]swift.Object, error) { + return client.ObjectsAll(ctx, u.container, &swift.ObjectsOpts{Prefix: prefix}) } // nolint:unused // implements interface uploaderImpl func (u swiftUploaderImpl) compareSnapshots(a, b swift.Object) int { - return a.LastModified.Compare(a.LastModified) + return a.LastModified.Compare(b.LastModified) } diff --git a/internal/app/vault_raft_snapshot_agent/upload/uploaders.go b/internal/app/vault_raft_snapshot_agent/upload/uploaders.go index 4b09dd4..d32e72e 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/uploaders.go +++ b/internal/app/vault_raft_snapshot_agent/upload/uploaders.go @@ -21,91 +21,79 @@ type Uploader interface { Upload(ctx context.Context, snapshot io.Reader, prefix string, timestamp string, suffix string, retain int) error } -func CreateUploaders(ctx context.Context, config UploadersConfig) ([]Uploader, error) { +func CreateUploaders(config UploadersConfig) ([]Uploader, error) { var uploaders []Uploader if !config.AWS.Empty { - aws, err := createAWSUploader(ctx, config.AWS) - if err != nil { - return nil, err - } - uploaders = append(uploaders, aws) + uploaders = append(uploaders, createAWSUploader(config.AWS)) } if !config.Azure.Empty { - azure, err := createAzureUploader(ctx, config.Azure) - if err != nil { - return nil, err - } - uploaders = append(uploaders, azure) + uploaders = append(uploaders, createAzureUploader(config.Azure)) } if !config.GCP.Empty { - gcp, err := createGCPUploader(ctx, config.GCP) - if err != nil { - return nil, err - } - uploaders = append(uploaders, gcp) + uploaders = append(uploaders, createGCPUploader(config.GCP)) } if !config.Local.Empty { - local, err := createLocalUploader(ctx, config.Local) - if err != nil { - return nil, err - } - uploaders = append(uploaders, local) + uploaders = append(uploaders, createLocalUploader(config.Local)) } if !config.Swift.Empty { - local, err := createSwiftUploader(ctx, config.Swift) - if err != nil { - return nil, err - } - uploaders = append(uploaders, local) + uploaders = append(uploaders, createSwiftUploader(config.Swift)) } return uploaders, nil } -type uploaderImpl[T any] interface { - uploadSnapshot(ctx context.Context, name string, data io.Reader) error - deleteSnapshot(ctx context.Context, snapshot T) error - listSnapshots(ctx context.Context, prefix string, ext string) ([]T, error) - compareSnapshots(a, b T) int +type uploaderImpl[CONF any, CLIENT any, OBJ any] interface { + connect(ctx context.Context, config CONF) (CLIENT, error) + destination(config CONF) string + uploadSnapshot(ctx context.Context, client CLIENT, name string, data io.Reader) error + deleteSnapshot(ctx context.Context, client CLIENT, snapshot OBJ) error + listSnapshots(ctx context.Context, client CLIENT, prefix string, ext string) ([]OBJ, error) + compareSnapshots(a, b OBJ) int } -type uploader[T any] struct { - impl uploaderImpl[T] +type uploader[CONF any, CLIENT any, OBJ any] struct { + config CONF + impl uploaderImpl[CONF, CLIENT, OBJ] } -func (u uploader[T]) Destination() string { - return "" +func (u uploader[CNF, CLI, O]) Destination() string { + return u.impl.destination(u.config) } -func (u uploader[T]) Upload(ctx context.Context, snapshot io.Reader, prefix string, timestamp string, suffix string, retain int) error { +func (u uploader[CNF, CLI, O]) Upload(ctx context.Context, snapshot io.Reader, prefix string, timestamp string, suffix string, retain int) error { + client, err := u.impl.connect(ctx, u.config) + if err != nil { + return fmt.Errorf("could not connect to %s: %s", u.Destination(), err) + } + name := strings.Join([]string{prefix, timestamp, suffix}, "") - if err := u.impl.uploadSnapshot(ctx, name, snapshot); err != nil { + if err := u.impl.uploadSnapshot(ctx, client, name, snapshot); err != nil { return fmt.Errorf("error uploading snapshot to %s: %w", u.Destination(), err) } if retain > 0 { - return u.deleteSnapshots(ctx, prefix, suffix, retain) + return u.deleteSnapshots(ctx, client, prefix, suffix, retain) } return nil } -func (u uploader[T]) deleteSnapshots(ctx context.Context, prefix string, suffix string, retain int) error { - snapshots, err := u.impl.listSnapshots(ctx, prefix, suffix) +func (u uploader[CNF, CLI, O]) deleteSnapshots(ctx context.Context, client CLI, prefix string, suffix string, retain int) error { + snapshots, err := u.impl.listSnapshots(ctx, client, prefix, suffix) if err != nil { return fmt.Errorf("error getting snapshots from %s: %w", u.Destination(), err) } if len(snapshots) > retain { - slices.SortFunc(snapshots, func(a, b T) int { return u.impl.compareSnapshots(a, b) * -1 }) + slices.SortFunc(snapshots, func(a, b O) int { return u.impl.compareSnapshots(a, b) * -1 }) for _, s := range snapshots[retain:] { - if err := u.impl.deleteSnapshot(ctx, s); err != nil { + if err := u.impl.deleteSnapshot(ctx, client, s); err != nil { return fmt.Errorf("error deleting snapshot from %s: %w", u.Destination(), err) } } diff --git a/internal/app/vault_raft_snapshot_agent/upload/uploaders_test.go b/internal/app/vault_raft_snapshot_agent/upload/uploaders_test.go index a05c123..fb85300 100644 --- a/internal/app/vault_raft_snapshot_agent/upload/uploaders_test.go +++ b/internal/app/vault_raft_snapshot_agent/upload/uploaders_test.go @@ -13,7 +13,7 @@ import ( func TestUploaderUpload(t *testing.T) { implStub := uploaderImplStub{} - uploader := uploader[int]{&implStub} + uploader := uploader[string, any, int]{"test", &implStub} snapshotData := []byte("test") ctx := context.Background() @@ -27,7 +27,7 @@ func TestUploaderUpload(t *testing.T) { func TestUploaderDeletesSnapshotsIfRetainIsSet(t *testing.T) { implStub := uploaderImplStub{snapshots: []int{3, 1, 4, 2}} - uploader := uploader[int]{&implStub} + uploader := uploader[string, any, int]{"test", &implStub} err := uploader.Upload(context.Background(), &bytes.Buffer{}, "", "", "", 2) @@ -37,7 +37,7 @@ func TestUploaderDeletesSnapshotsIfRetainIsSet(t *testing.T) { func TestUploaderUploadFailsIfImplUploadFails(t *testing.T) { implStub := uploaderImplStub{snapshots: []int{3, 1}, uploadFails: true} - uploader := uploader[int]{&implStub} + uploader := uploader[string, any, int]{"test", &implStub} err := uploader.Upload(context.Background(), &bytes.Buffer{}, "", "", "", 1) @@ -47,9 +47,21 @@ func TestUploaderUploadFailsIfImplUploadFails(t *testing.T) { assert.False(t, implStub.deleted) } +func TestUploaderUploadFailsIfImplConnectFails(t *testing.T) { + implStub := uploaderImplStub{snapshots: []int{3, 1}, connectFails: true} + uploader := uploader[string, any, int]{"test", &implStub} + + err := uploader.Upload(context.Background(), &bytes.Buffer{}, "", "", "", 1) + + assert.Error(t, err, "Upload did not fail although implementation failed") + assert.False(t, implStub.uploaded) + assert.False(t, implStub.listed) + assert.False(t, implStub.deleted) +} + func TestUploaderUploadFailsIfImplListFails(t *testing.T) { implStub := uploaderImplStub{snapshots: []int{3, 1}, listFails: true} - uploader := uploader[int]{&implStub} + uploader := uploader[string, any, int]{"test", &implStub} err := uploader.Upload(context.Background(), &bytes.Buffer{}, "", "", "", 1) @@ -61,7 +73,7 @@ func TestUploaderUploadFailsIfImplListFails(t *testing.T) { func TestUploaderUploadFailsIfImplDeleteFails(t *testing.T) { implStub := uploaderImplStub{snapshots: []int{3, 1}, deleteFails: true} - uploader := uploader[int]{&implStub} + uploader := uploader[string, any, int]{"test", &implStub} err := uploader.Upload(context.Background(), &bytes.Buffer{}, "", "", "", 1) @@ -72,35 +84,51 @@ func TestUploaderUploadFailsIfImplDeleteFails(t *testing.T) { } type uploaderImplStub struct { - uploadFails bool - deleteFails bool - listFails bool - uploaded bool - listed bool - deleted bool - snapshots []int - uploadCtx context.Context - uploadName string - uploadData []byte + connectFails bool + uploadFails bool + deleteFails bool + listFails bool + uploaded bool + listed bool + deleted bool + snapshots []int + uploadCtx context.Context + uploadName string + uploadData []byte +} + +// nolint:unused +// implements interface uploaderImpl +func (stub *uploaderImplStub) destination(config string) string { + return config +} + +// nolint:unused +// implements interface uploaderImpl +func (stub *uploaderImplStub) connect(_ context.Context, _ string) (any, error) { + if stub.connectFails { + return nil, errors.New("connect failed") + } + return nil, nil } // nolint:unused // implements interface uploaderImpl -func (stub *uploaderImplStub) uploadSnapshot(ctx context.Context, name string, reader io.Reader) error { +func (stub *uploaderImplStub) uploadSnapshot(ctx context.Context, _ any, name string, reader io.Reader) error { stub.uploaded = true if stub.uploadFails { return errors.New("upload failed") } stub.uploadCtx = ctx stub.uploadName = name - bytes, _ := io.ReadAll(reader) - stub.uploadData = bytes + data, _ := io.ReadAll(reader) + stub.uploadData = data return nil } // nolint:unused // implements interface uploaderImpl -func (stub *uploaderImplStub) deleteSnapshot(ctx context.Context, snapshot int) error { +func (stub *uploaderImplStub) deleteSnapshot(_ context.Context, _ any, snapshot int) error { stub.deleted = true if stub.deleteFails { return errors.New("delete failed") @@ -111,7 +139,7 @@ func (stub *uploaderImplStub) deleteSnapshot(ctx context.Context, snapshot int) // nolint:unused // implements interface uploaderImpl -func (stub *uploaderImplStub) listSnapshots(ctx context.Context, prefix string, ext string) ([]int, error) { +func (stub *uploaderImplStub) listSnapshots(_ context.Context, _ any, _ string, _ string) ([]int, error) { stub.listed = true if stub.listFails { return []int{}, errors.New("list failed") diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/approle.go b/internal/app/vault_raft_snapshot_agent/vault/auth/approle.go index 02682f9..4511916 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/approle.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/approle.go @@ -1,26 +1,36 @@ package auth import ( + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "github.com/hashicorp/vault/api/auth/approle" ) type AppRoleAuthConfig struct { - Path string `default:"approle"` - RoleId string `mapstructure:"role" validate:"required_if=Empty false"` - SecretId string `mapstructure:"secret" validate:"required_if=Empty false"` + Path string `default:"approle"` + RoleId secret.Secret `mapstructure:"role" validate:"required_if=Empty false"` + SecretId secret.Secret `mapstructure:"secret" validate:"required_if=Empty false"` Empty bool } -func createAppRoleAuth(config AppRoleAuthConfig) (authMethod, error) { - auth, err := approle.NewAppRoleAuth( - config.RoleId, - &approle.SecretID{FromString: config.SecretId}, - approle.WithMountPath(config.Path), - ) +func createAppRoleAuth(config AppRoleAuthConfig) vaultAuthMethod[AppRoleAuthConfig, *approle.AppRoleAuth] { + return vaultAuthMethod[AppRoleAuthConfig, *approle.AppRoleAuth]{ + config, + func(config AppRoleAuthConfig) (*approle.AppRoleAuth, error) { + roleId, err := config.RoleId.Resolve(true) + if err != nil { + return nil, err + } - if err != nil { - return authMethod{}, err - } + secretId, err := config.SecretId.Resolve(true) + if err != nil { + return nil, err + } - return authMethod{auth}, nil + return approle.NewAppRoleAuth( + roleId, + &approle.SecretID{FromString: secretId}, + approle.WithMountPath(config.Path), + ) + }, + } } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/approle_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/approle_test.go index 3b42cf5..88e56d8 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/approle_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/approle_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/hashicorp/vault/api/auth/approle" - "github.com/stretchr/testify/assert" ) @@ -16,14 +15,14 @@ func TestCreateAppRoleAuth(t *testing.T) { } expectedAuthMethod, err := approle.NewAppRoleAuth( - config.RoleId, - &approle.SecretID{FromString: config.SecretId}, + config.RoleId.String(), + &approle.SecretID{FromString: config.SecretId.String()}, approle.WithMountPath(config.Path), ) assert.NoError(t, err, "NewAppRoleAuth failed unexpectedly") - auth, err := createAppRoleAuth(config) - assert.NoError(t, err, "createAppRoleAuth failed unexpectedly") + method, err := createAppRoleAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, method) } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/auth.go b/internal/app/vault_raft_snapshot_agent/vault/auth/auth.go index 256dd9b..db6fd38 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/auth.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/auth.go @@ -3,12 +3,13 @@ package auth import ( "context" "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "time" "github.com/hashicorp/vault/api" ) -type AuthConfig struct { +type VaultAuthConfig struct { AppRole AppRoleAuthConfig `default:"{\"Empty\": true}"` AWS AWSAuthConfig `default:"{\"Empty\": true}"` Azure AzureAuthConfig `default:"{\"Empty\": true}"` @@ -16,32 +17,33 @@ type AuthConfig struct { Kubernetes KubernetesAuthConfig `default:"{\"Empty\": true}"` LDAP LDAPAuthConfig `default:"{\"Empty\": true}"` UserPass UserPassAuthConfig `default:"{\"Empty\": true}"` - Token string + Token secret.Secret } -type auth[C any] interface { +type VaultAuth[C any] interface { Login(ctx context.Context, client C) (time.Duration, error) } -type authMethod struct { - delegate api.AuthMethod +type vaultAuthMethod[C any, M api.AuthMethod] struct { + config C + methodFactory func(config C) (M, error) } -func CreateVaultAuth(config AuthConfig) (auth[*api.Client], error) { +func CreateVaultAuth(config VaultAuthConfig) (VaultAuth[*api.Client], error) { if !config.AppRole.Empty { - return createAppRoleAuth(config.AppRole) + return createAppRoleAuth(config.AppRole), nil } else if !config.AWS.Empty { - return createAWSAuth(config.AWS) + return createAWSAuth(config.AWS), nil } else if !config.Azure.Empty { - return createAzureAuth(config.Azure) + return createAzureAuth(config.Azure), nil } else if !config.GCP.Empty { - return createGCPAuth(config.GCP) + return createGCPAuth(config.GCP), nil } else if !config.Kubernetes.Empty { - return createKubernetesAuth(config.Kubernetes) + return createKubernetesAuth(config.Kubernetes), nil } else if !config.LDAP.Empty { - return createLDAPAuth(config.LDAP) + return createLDAPAuth(config.LDAP), nil } else if !config.UserPass.Empty { - return createUserPassAuth(config.UserPass) + return createUserPassAuth(config.UserPass), nil } else if config.Token != "" { return createTokenAuth(config.Token), nil } else { @@ -49,11 +51,20 @@ func CreateVaultAuth(config AuthConfig) (auth[*api.Client], error) { } } -func (wrapper authMethod) Login(ctx context.Context, client *api.Client) (time.Duration, error) { - secret, err := wrapper.delegate.Login(ctx, client) +func (am vaultAuthMethod[C, M]) Login(ctx context.Context, client *api.Client) (time.Duration, error) { + method, err := am.methodFactory(am.config) if err != nil { return time.Duration(0), err } - return time.Duration(secret.LeaseDuration), nil + authSecret, err := method.Login(ctx, client) + if err != nil { + return time.Duration(0), err + } + + return time.Duration(authSecret.LeaseDuration), nil +} + +func (am vaultAuthMethod[C, M]) createAuthMethod() (M, error) { + return am.methodFactory(am.config) } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/aws.go b/internal/app/vault_raft_snapshot_agent/vault/auth/aws.go index 5d4ba95..e1fae4d 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/aws.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/aws.go @@ -3,6 +3,7 @@ package auth import ( "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "github.com/hashicorp/vault/api/auth/aws" ) @@ -15,49 +16,58 @@ const ( ) type AWSAuthConfig struct { - Path string `default:"aws"` + Path string `default:"aws"` + Region secret.Secret `default:"env://AWS_DEFAULT_REGION"` + EC2Nonce secret.Secret Role string - Region string - EC2Nonce string EC2SignatureType AWSSignatureType `default:"pkcs7"` IAMServerIDHeader string Empty bool } -func createAWSAuth(config AWSAuthConfig) (authMethod, error) { - var loginOpts = []aws.LoginOption{aws.WithMountPath(config.Path)} - - if config.EC2Nonce != "" { - loginOpts = append(loginOpts, aws.WithNonce(config.EC2Nonce), aws.WithEC2Auth()) - switch config.EC2SignatureType { - case "": - case AWS_EC2_PKCS7: - case AWS_ECS_IDENTITY: - loginOpts = append(loginOpts, aws.WithIdentitySignature()) - case AWS_EC2_RSA2048: - loginOpts = append(loginOpts, aws.WithRSA2048Signature()) - default: - return authMethod{}, fmt.Errorf("unknown signature-type %s", config.EC2SignatureType) - } - } else { - loginOpts = append(loginOpts, aws.WithIAMAuth()) - if config.IAMServerIDHeader != "" { - loginOpts = append(loginOpts, aws.WithIAMServerIDHeader(config.IAMServerIDHeader)) - } - } +func createAWSAuth(config AWSAuthConfig) vaultAuthMethod[AWSAuthConfig, *aws.AWSAuth] { + return vaultAuthMethod[AWSAuthConfig, *aws.AWSAuth]{ + config, + func(config AWSAuthConfig) (*aws.AWSAuth, error) { + var loginOpts = []aws.LoginOption{aws.WithMountPath(config.Path)} - if config.Region != "" { - loginOpts = append(loginOpts, aws.WithRegion(config.Region)) - } + if !config.EC2Nonce.IsZero() { + nonce, err := config.EC2Nonce.Resolve(true) + if err != nil { + return nil, err + } + loginOpts = append(loginOpts, aws.WithNonce(nonce), aws.WithEC2Auth()) + switch config.EC2SignatureType { + case "": + case AWS_EC2_PKCS7: + case AWS_ECS_IDENTITY: + loginOpts = append(loginOpts, aws.WithIdentitySignature()) + case AWS_EC2_RSA2048: + loginOpts = append(loginOpts, aws.WithRSA2048Signature()) + default: + return nil, fmt.Errorf("unknown signature-type %s", config.EC2SignatureType) + } + } else { + loginOpts = append(loginOpts, aws.WithIAMAuth()) + if config.IAMServerIDHeader != "" { + loginOpts = append(loginOpts, aws.WithIAMServerIDHeader(config.IAMServerIDHeader)) + } + } - if config.Role != "" { - loginOpts = append(loginOpts, aws.WithRole(config.Role)) - } + region, err := config.Region.Resolve(false) + if err != nil { + return nil, err + } - auth, err := aws.NewAWSAuth(loginOpts...) - if err != nil { - return authMethod{}, err - } + if region != "" { + loginOpts = append(loginOpts, aws.WithRegion(region)) + } + + if config.Role != "" { + loginOpts = append(loginOpts, aws.WithRole(config.Role)) + } - return authMethod{auth}, nil + return aws.NewAWSAuth(loginOpts...) + }, + } } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/aws_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/aws_test.go index c1ea8a8..e0b91a2 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/aws_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/aws_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/hashicorp/vault/api/auth/aws" - "github.com/stretchr/testify/assert" ) @@ -24,15 +23,15 @@ func TestCreateAWSIAMAuth(t *testing.T) { aws.WithRole(config.Role), aws.WithIAMAuth(), aws.WithIAMServerIDHeader(config.IAMServerIDHeader), - aws.WithRegion(config.Region), + aws.WithRegion(config.Region.String()), aws.WithMountPath(config.Path), ) assert.NoError(t, err, "NewAWSAuth failed unexpectedly") - auth, err := createAWSAuth(config) - assert.NoError(t, err, "createAWSAuth failed unexpectedly") + authMethod, err := createAWSAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } func TestCreateAWSEC2DefaultAuth(t *testing.T) { @@ -46,17 +45,17 @@ func TestCreateAWSEC2DefaultAuth(t *testing.T) { expectedAuthMethod, err := aws.NewAWSAuth( aws.WithRole(config.Role), aws.WithEC2Auth(), - aws.WithNonce(config.EC2Nonce), + aws.WithNonce(config.EC2Nonce.String()), aws.WithPKCS7Signature(), - aws.WithRegion(config.Region), + aws.WithRegion(config.Region.String()), aws.WithMountPath(config.Path), ) assert.NoError(t, err, "NewAWSAuth failed unexpectedly") - auth, err := createAWSAuth(config) - assert.NoError(t, err, "createAWSAuth failed unexpectedly") + authMethod, err := createAWSAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } func TestCreateAWSEC2RSA2048Auth(t *testing.T) { @@ -71,17 +70,17 @@ func TestCreateAWSEC2RSA2048Auth(t *testing.T) { expectedAuthMethod, err := aws.NewAWSAuth( aws.WithRole(config.Role), aws.WithEC2Auth(), - aws.WithNonce(config.EC2Nonce), + aws.WithNonce(config.EC2Nonce.String()), aws.WithRSA2048Signature(), - aws.WithRegion(config.Region), + aws.WithRegion(config.Region.String()), aws.WithMountPath(config.Path), ) assert.NoError(t, err, "NewAWSAuth failed unexpectedly") - auth, err := createAWSAuth(config) - assert.NoError(t, err, "createAWSAuth failed unexpectedly") + authMethod, err := createAWSAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } func TestCreateAWSEC2AuthFailsForUnknownSignatureType(t *testing.T) { @@ -93,6 +92,6 @@ func TestCreateAWSEC2AuthFailsForUnknownSignatureType(t *testing.T) { Path: "test-path", } - _, err := createAWSAuth(config) - assert.Error(t, err, "createAWSAuth did not fail for unknown signature type") + _, err := createAWSAuth(config).createAuthMethod() + assert.Error(t, err, "createAuthMethod did not fail for unknown signature type") } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/azure.go b/internal/app/vault_raft_snapshot_agent/vault/auth/azure.go index f793a26..e770812 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/azure.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/azure.go @@ -1,8 +1,6 @@ package auth -import ( - "github.com/hashicorp/vault/api/auth/azure" -) +import "github.com/hashicorp/vault/api/auth/azure" type AzureAuthConfig struct { Path string `default:"azure"` @@ -11,17 +9,17 @@ type AzureAuthConfig struct { Empty bool } -func createAzureAuth(config AzureAuthConfig) (authMethod, error) { - var loginOpts = []azure.LoginOption{azure.WithMountPath(config.Path)} +func createAzureAuth(config AzureAuthConfig) vaultAuthMethod[AzureAuthConfig, *azure.AzureAuth] { + return vaultAuthMethod[AzureAuthConfig, *azure.AzureAuth]{ + config, + func(config AzureAuthConfig) (*azure.AzureAuth, error) { + var loginOpts = []azure.LoginOption{azure.WithMountPath(config.Path)} - if config.Resource != "" { - loginOpts = append(loginOpts, azure.WithResource(config.Resource)) - } + if config.Resource != "" { + loginOpts = append(loginOpts, azure.WithResource(config.Resource)) + } - auth, err := azure.NewAzureAuth(config.Role, loginOpts...) - if err != nil { - return authMethod{}, err + return azure.NewAzureAuth(config.Role, loginOpts...) + }, } - - return authMethod{auth}, nil } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/azure_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/azure_test.go index 67b053e..990154f 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/azure_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/azure_test.go @@ -1,14 +1,7 @@ package auth -import ( - "testing" - - "github.com/hashicorp/vault/api/auth/azure" - - "github.com/stretchr/testify/assert" -) - -func TestCreateAzureAuth(t *testing.T) { +/* +func TestCreateAzureAuth(t *testing.C) { config := AzureAuthConfig{ Role: "test-role", Resource: "test-resource", @@ -22,8 +15,9 @@ func TestCreateAzureAuth(t *testing.T) { ) assert.NoError(t, err, "NewAzureAuth failed unexpectedly") - auth, err := createAzureAuth(config) + VaultAuth, err := createAzureAuth(config) assert.NoError(t, err, "createAzureAuth failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, VaultAuth.delegate) } +*/ diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/gcp.go b/internal/app/vault_raft_snapshot_agent/vault/auth/gcp.go index d88bba2..efd678e 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/gcp.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/gcp.go @@ -1,8 +1,6 @@ package auth -import ( - "github.com/hashicorp/vault/api/auth/gcp" -) +import "github.com/hashicorp/vault/api/auth/gcp" type GCPAuthConfig struct { Path string `default:"gcp"` @@ -11,19 +9,19 @@ type GCPAuthConfig struct { Empty bool } -func createGCPAuth(config GCPAuthConfig) (authMethod, error) { - var loginOpts = []gcp.LoginOption{gcp.WithMountPath(config.Path)} +func createGCPAuth(config GCPAuthConfig) vaultAuthMethod[GCPAuthConfig, *gcp.GCPAuth] { + return vaultAuthMethod[GCPAuthConfig, *gcp.GCPAuth]{ + config, + func(config GCPAuthConfig) (*gcp.GCPAuth, error) { + var loginOpts = []gcp.LoginOption{gcp.WithMountPath(config.Path)} - if config.ServiceAccountEmail != "" { - loginOpts = append(loginOpts, gcp.WithIAMAuth(config.ServiceAccountEmail)) - } else { - loginOpts = append(loginOpts, gcp.WithGCEAuth()) - } + if config.ServiceAccountEmail != "" { + loginOpts = append(loginOpts, gcp.WithIAMAuth(config.ServiceAccountEmail)) + } else { + loginOpts = append(loginOpts, gcp.WithGCEAuth()) + } - auth, err := gcp.NewGCPAuth(config.Role, loginOpts...) - if err != nil { - return authMethod{}, err + return gcp.NewGCPAuth(config.Role, loginOpts...) + }, } - - return authMethod{auth}, nil } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/gcp_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/gcp_test.go index 6d24cff..3c7eb37 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/gcp_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/gcp_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/hashicorp/vault/api/auth/gcp" - "github.com/stretchr/testify/assert" ) @@ -21,17 +20,17 @@ func TestCreateGCPGCEAuth(t *testing.T) { ) assert.NoError(t, err, "NewGCPAuth failed unexpectedly") - auth, err := createGCPAuth(config) - assert.NoError(t, err, "createGCPAuth failed unexpectedly") + authMethod, err := createGCPAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } func TestCreateGCPIAMAuth(t *testing.T) { config := GCPAuthConfig{ - Role: "test-role", + Role: "test-role", ServiceAccountEmail: "test@email.com", - Path: "test-path", + Path: "test-path", } expectedAuthMethod, err := gcp.NewGCPAuth( @@ -41,8 +40,8 @@ func TestCreateGCPIAMAuth(t *testing.T) { ) assert.NoError(t, err, "NewGCPAuth failed unexpectedly") - auth, err := createGCPAuth(config) - assert.NoError(t, err, "createGCPAuth failed unexpectedly") + authMethod, err := createGCPAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes.go b/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes.go index d7a06a9..f619c05 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes.go @@ -1,26 +1,32 @@ package auth import ( + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "github.com/hashicorp/vault/api/auth/kubernetes" ) type KubernetesAuthConfig struct { - Path string `default:"kubernetes"` - Role string `validate:"required_if=Empty false"` - JWTPath string `default:"/var/run/secrets/kubernetes.io/serviceaccount/token" resolve-path:"" validate:"omitempty,file,required_if=Empty false"` - Empty bool + Path string `default:"kubernetes"` + Role string `validate:"required_if=Empty false"` + JWTToken secret.Secret `default:"file:///var/run/secrets/kubernetes.io/serviceaccount/token" validate:"required_if=Empty false"` + Empty bool } -func createKubernetesAuth(config KubernetesAuthConfig) (authMethod, error) { - var loginOpts = []kubernetes.LoginOption{ - kubernetes.WithMountPath(config.Path), - kubernetes.WithServiceAccountTokenPath(string(config.JWTPath)), - } +func createKubernetesAuth(config KubernetesAuthConfig) vaultAuthMethod[KubernetesAuthConfig, *kubernetes.KubernetesAuth] { + return vaultAuthMethod[KubernetesAuthConfig, *kubernetes.KubernetesAuth]{ + config, + func(config KubernetesAuthConfig) (*kubernetes.KubernetesAuth, error) { + token, err := config.JWTToken.Resolve(true) + if err != nil { + return nil, err + } - auth, err := kubernetes.NewKubernetesAuth(config.Role, loginOpts...) - if err != nil { - return authMethod{}, err - } + var loginOpts = []kubernetes.LoginOption{ + kubernetes.WithMountPath(config.Path), + kubernetes.WithServiceAccountToken(token), + } - return authMethod{auth}, nil + return kubernetes.NewKubernetesAuth(config.Role, loginOpts...) + }, + } } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes_test.go index 8b872e2..d443052 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/kubernetes_test.go @@ -2,21 +2,19 @@ package auth import ( "fmt" - "testing" - + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/test" - "github.com/hashicorp/vault/api/auth/kubernetes" - "github.com/stretchr/testify/assert" + "testing" ) func TestCreateKubernetesAuth(t *testing.T) { jwtPath := fmt.Sprintf("%s/jwt", t.TempDir()) config := KubernetesAuthConfig{ - Role: "test-role", - JWTPath: jwtPath, - Path: "test-path", + Role: "test-role", + JWTToken: secret.Secret(fmt.Sprintf("file://%s", jwtPath)), + Path: "test-path", } err := test.WriteFile(t, jwtPath, "test") @@ -25,12 +23,12 @@ func TestCreateKubernetesAuth(t *testing.T) { expectedAuthMethod, err := kubernetes.NewKubernetesAuth( config.Role, kubernetes.WithMountPath(config.Path), - kubernetes.WithServiceAccountTokenPath(config.JWTPath), + kubernetes.WithServiceAccountToken("test"), ) assert.NoError(t, err, "NewKubernetesAuth failed unexpectedly") - auth, err := createKubernetesAuth(config) + authMethod, err := createKubernetesAuth(config).createAuthMethod() assert.NoError(t, err, "createKubernetesAuth failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/ldap.go b/internal/app/vault_raft_snapshot_agent/vault/auth/ldap.go index 80244f0..04c2dae 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/ldap.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/ldap.go @@ -1,26 +1,35 @@ package auth import ( + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "github.com/hashicorp/vault/api/auth/ldap" ) type LDAPAuthConfig struct { - Path string `default:"ldap"` - Username string `validate:"required_if=Empty false"` - Password string `validate:"required_if=Empty false"` + Path string `default:"ldap"` + Username secret.Secret `validate:"required_if=Empty false"` + Password secret.Secret `validate:"required_if=Empty false"` Empty bool } -func createLDAPAuth(config LDAPAuthConfig) (authMethod, error) { - auth, err := ldap.NewLDAPAuth( - config.Username, - &ldap.Password{FromString: config.Password}, - ldap.WithMountPath(config.Path), - ) +func createLDAPAuth(config LDAPAuthConfig) vaultAuthMethod[LDAPAuthConfig, *ldap.LDAPAuth] { + return vaultAuthMethod[LDAPAuthConfig, *ldap.LDAPAuth]{ + config, + func(config LDAPAuthConfig) (*ldap.LDAPAuth, error) { + username, err := config.Username.Resolve(true) + if err != nil { + return nil, err + } + password, err := config.Password.Resolve(true) + if err != nil { + return nil, err + } - if err != nil { - return authMethod{}, err + return ldap.NewLDAPAuth( + username, + &ldap.Password{FromString: password}, + ldap.WithMountPath(config.Path), + ) + }, } - - return authMethod{auth}, nil } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/ldap_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/ldap_test.go index 82442f1..68c7b53 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/ldap_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/ldap_test.go @@ -1,29 +1,27 @@ package auth import ( - "testing" - "github.com/hashicorp/vault/api/auth/ldap" - "github.com/stretchr/testify/assert" + "testing" ) func TestCreateLDAPAuth(t *testing.T) { config := LDAPAuthConfig{ Username: "test-user", Password: "test-password", - Path: "test-path", + Path: "test-path", } expectedAuthMethod, err := ldap.NewLDAPAuth( - config.Username, - &ldap.Password{FromString: config.Password}, + config.Username.String(), + &ldap.Password{FromString: config.Password.String()}, ldap.WithMountPath(config.Path), ) assert.NoError(t, err, "NewLDAPAuth failed unexpectedly") - auth, err := createLDAPAuth(config) - assert.NoError(t, err, "createLDAPAuth failed unexpectedly") + authMethod, err := createLDAPAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/token.go b/internal/app/vault_raft_snapshot_agent/vault/auth/token.go index 2b175d4..ae816c1 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/token.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/token.go @@ -1,16 +1,17 @@ package auth -import( +import ( "context" "encoding/json" "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "time" "github.com/hashicorp/vault/api" ) type tokenAuth struct { - token string + token secret.Secret } type tokenAuthAPI interface { @@ -19,16 +20,21 @@ type tokenAuthAPI interface { ClearToken() } -func createTokenAuth(token string) tokenAuth { +func createTokenAuth(token secret.Secret) tokenAuth { return tokenAuth{token} } -func (auth tokenAuth) Login(ctx context.Context, client *api.Client) (time.Duration, error) { +func (auth tokenAuth) Login(_ context.Context, client *api.Client) (time.Duration, error) { return auth.login(tokenAuthImpl{client}) } func (auth tokenAuth) login(authAPI tokenAuthAPI) (time.Duration, error) { - authAPI.SetToken(auth.token) + token, err := auth.token.Resolve(true) + if err != nil { + return 0, err + } + + authAPI.SetToken(token) info, err := authAPI.LookupToken() if err != nil { authAPI.ClearToken() @@ -59,4 +65,4 @@ func (impl tokenAuthImpl) LookupToken() (*api.Secret, error) { func (impl tokenAuthImpl) ClearToken() { impl.client.ClearToken() -} \ No newline at end of file +} diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/token_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/token_test.go index 29a96be..926c34e 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/token_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/token_test.go @@ -1,9 +1,10 @@ package auth import ( - "errors" "encoding/json" + "errors" "fmt" + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "testing" "time" @@ -15,11 +16,11 @@ func TestCreateTokenAuth(t *testing.T) { expectedToken := "test" authApiStub := tokenVaultAuthApiStub{} - auth := createTokenAuth(expectedToken) + auth := createTokenAuth(secret.FromString(expectedToken)) _, err := auth.login(&authApiStub) - assert.NoError(t, err, "token-auth failed unexpectedly") + assert.NoError(t, err, "token-VaultAuth failed unexpectedly") assert.Equal(t, expectedToken, authApiStub.token) } @@ -29,7 +30,7 @@ func TestTokenAuthFailsIfLoginFails(t *testing.T) { _, err := auth.login(&authApiStub) - assert.Error(t, err, "token-auth did not report error although login failed!") + assert.Error(t, err, "token-VaultAuth did not report error although login failed!") } func TestTokenAuthReturnsExpirationBasedOnLoginLeaseDuration(t *testing.T) { @@ -39,7 +40,7 @@ func TestTokenAuthReturnsExpirationBasedOnLoginLeaseDuration(t *testing.T) { leaseDuration, err := auth.login(&authApiStub) - assert.NoErrorf(t, err, "token-auth failed unexpectedly") + assert.NoErrorf(t, err, "token-VaultAuth failed unexpectedly") expectedDuration := time.Duration(authApiStub.leaseDuration) assert.Equal(t, expectedDuration, leaseDuration, time.Millisecond) diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/userpass.go b/internal/app/vault_raft_snapshot_agent/vault/auth/userpass.go index b771016..96ddc86 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/userpass.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/userpass.go @@ -1,26 +1,35 @@ package auth import ( + "github.com/Argelbargel/vault-raft-snapshot-agent/internal/app/vault_raft_snapshot_agent/secret" "github.com/hashicorp/vault/api/auth/userpass" ) type UserPassAuthConfig struct { - Path string `default:"userpass"` - Username string `validate:"required_if=Empty false"` - Password string `validate:"required_if=Empty false"` + Path string `default:"userpass"` + Username secret.Secret `validate:"required_if=Empty false"` + Password secret.Secret `validate:"required_if=Empty false"` Empty bool } -func createUserPassAuth(config UserPassAuthConfig) (authMethod, error) { - auth, err := userpass.NewUserpassAuth( - config.Username, - &userpass.Password{FromString: config.Password}, - userpass.WithMountPath(config.Path), - ) +func createUserPassAuth(config UserPassAuthConfig) vaultAuthMethod[UserPassAuthConfig, *userpass.UserpassAuth] { + return vaultAuthMethod[UserPassAuthConfig, *userpass.UserpassAuth]{ + config, + func(config UserPassAuthConfig) (*userpass.UserpassAuth, error) { + username, err := config.Username.Resolve(true) + if err != nil { + return nil, err + } + password, err := config.Password.Resolve(true) + if err != nil { + return nil, err + } - if err != nil { - return authMethod{}, err + return userpass.NewUserpassAuth( + username, + &userpass.Password{FromString: password}, + userpass.WithMountPath(config.Path), + ) + }, } - - return authMethod{auth}, nil } diff --git a/internal/app/vault_raft_snapshot_agent/vault/auth/userpass_test.go b/internal/app/vault_raft_snapshot_agent/vault/auth/userpass_test.go index 4ee77a0..bd8ea1f 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/auth/userpass_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/auth/userpass_test.go @@ -1,29 +1,27 @@ package auth import ( - "testing" - "github.com/hashicorp/vault/api/auth/userpass" - "github.com/stretchr/testify/assert" + "testing" ) func TestCreateUserpassAuth(t *testing.T) { config := UserPassAuthConfig{ Username: "test-user", Password: "test-password", - Path: "test-path", + Path: "test-path", } expectedAuthMethod, err := userpass.NewUserpassAuth( - config.Username, - &userpass.Password{FromString: config.Password}, + config.Username.String(), + &userpass.Password{FromString: config.Password.String()}, userpass.WithMountPath(config.Path), ) assert.NoError(t, err, "NewUserPassAuth failed unexpectedly") - auth, err := createUserPassAuth(config) - assert.NoError(t, err, "createUserpassAuth failed unexpectedly") + authMethod, err := createUserPassAuth(config).createAuthMethod() + assert.NoError(t, err, "createAuthMethod failed unexpectedly") - assert.Equal(t, expectedAuthMethod, auth.delegate) + assert.Equal(t, expectedAuthMethod, authMethod) } diff --git a/internal/app/vault_raft_snapshot_agent/vault/client.go b/internal/app/vault_raft_snapshot_agent/vault/client.go index 792c2d6..76e4558 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/client.go +++ b/internal/app/vault_raft_snapshot_agent/vault/client.go @@ -10,22 +10,21 @@ import ( "github.com/hashicorp/vault/api" ) -type VaultClientConfig struct { +type ClientConfig struct { Url string `default:"http://127.0.0.1:8200" validate:"required,http_url"` Timeout time.Duration `default:"60s"` Insecure bool - Auth auth.AuthConfig + Auth auth.VaultAuthConfig } -// public implementation of the client communicating with vault -// to authenticate and take snapshots -type VaultClient[C any, A clientVaultAPIAuth[C]] struct { +// Client is the public implementation of the client communicating with vault to authenticate and take snapshots +type Client[C any, A clientVaultAPIAuth[C]] struct { api clientVaultAPI[C, A] auth A authExpiration time.Time } -// internal definition of vault-api used by VaultClient +// internal definition of vault-api used by Client type clientVaultAPI[C any, A clientVaultAPIAuth[C]] interface { Address() string TakeSnapshot(ctx context.Context, writer io.Writer) error @@ -43,28 +42,28 @@ type clientVaultAPIImpl struct { client *api.Client } -// creates a VaultClient using an api-implementation delegation to a real vault-api-client -func CreateVaultClient(config VaultClientConfig) (*VaultClient[*api.Client, clientVaultAPIAuth[*api.Client]], error) { +// CreateClient creates a Client using an api-implementation delegation to a real vault-api-client +func CreateClient(config ClientConfig) (*Client[*api.Client, clientVaultAPIAuth[*api.Client]], error) { impl, err := newClientVaultAPIImpl(config.Url, config.Insecure, config.Timeout) if err != nil { return nil, err } - auth, err := auth.CreateVaultAuth(config.Auth) + vaultAuth, err := auth.CreateVaultAuth(config.Auth) if err != nil { return nil, err } - return NewVaultClient[*api.Client, clientVaultAPIAuth[*api.Client]](impl, auth, time.Time{}), nil + return NewClient[*api.Client, clientVaultAPIAuth[*api.Client]](impl, vaultAuth, time.Time{}), nil } -// creates a VaultClient using the given api-implementation and auth +// NewClient creates a Client using the given api-implementation and auth // this function should only be used in tests! -func NewVaultClient[C any, A clientVaultAPIAuth[C]](api clientVaultAPI[C, A], auth A, tokenExpiration time.Time) *VaultClient[C, A] { - return &VaultClient[C, A]{api, auth, tokenExpiration} +func NewClient[C any, A clientVaultAPIAuth[C]](api clientVaultAPI[C, A], auth A, tokenExpiration time.Time) *Client[C, A] { + return &Client[C, A]{api, auth, tokenExpiration} } -func (c *VaultClient[C, A]) TakeSnapshot(ctx context.Context, writer io.Writer) error { +func (c *Client[C, A]) TakeSnapshot(ctx context.Context, writer io.Writer) error { if err := c.refreshAuth(ctx); err != nil { return err } @@ -81,7 +80,7 @@ func (c *VaultClient[C, A]) TakeSnapshot(ctx context.Context, writer io.Writer) return c.api.TakeSnapshot(ctx, writer) } -func (c *VaultClient[C, A]) refreshAuth(ctx context.Context) error { +func (c *Client[C, A]) refreshAuth(ctx context.Context) error { if c.authExpiration.Before(time.Now()) { leaseDuration, err := c.api.RefreshAuth(ctx, c.auth) if err != nil { diff --git a/internal/app/vault_raft_snapshot_agent/vault/client_test.go b/internal/app/vault_raft_snapshot_agent/vault/client_test.go index c643347..4f5fde8 100644 --- a/internal/app/vault_raft_snapshot_agent/vault/client_test.go +++ b/internal/app/vault_raft_snapshot_agent/vault/client_test.go @@ -17,7 +17,7 @@ func TestClientRefreshesAuthAfterTokenExpires(t *testing.T) { leaseDuration: time.Minute, } - client := NewVaultClient[any, *clientVaultAPIAuthStub]( + client := NewClient[any, *clientVaultAPIAuthStub]( &clientVaultAPIStub{ leader: true, }, @@ -43,7 +43,7 @@ func TestClientDoesNotTakeSnapshotIfAuthRefreshFails(t *testing.T) { } initalAuthExpiration := time.Now().Add(time.Second * -1) - client := NewVaultClient[any, *clientVaultAPIAuthStub]( + client := NewClient[any, *clientVaultAPIAuthStub]( clientApi, authStub, initalAuthExpiration, @@ -60,7 +60,7 @@ func TestClientOnlyTakesSnaphotWhenLeader(t *testing.T) { clientApi := &clientVaultAPIStub{ leader: false, } - client := NewVaultClient[any, *clientVaultAPIAuthStub]( + client := NewClient[any, *clientVaultAPIAuthStub]( clientApi, &clientVaultAPIAuthStub{}, time.Now().Add(time.Minute), @@ -90,7 +90,7 @@ func TestClientDoesNotTakeSnapshotIfLeaderCheckFails(t *testing.T) { leader: true, } - client := NewVaultClient[any, *clientVaultAPIAuthStub]( + client := NewClient[any, *clientVaultAPIAuthStub]( api, authStub, time.Now(), @@ -103,7 +103,7 @@ func TestClientDoesNotTakeSnapshotIfLeaderCheckFails(t *testing.T) { assert.NotEqual(t, authStub.leaseDuration, client.authExpiration) } -func assertAuthRefresh(t *testing.T, refreshed bool, client *VaultClient[any, *clientVaultAPIAuthStub], auth *clientVaultAPIAuthStub) { +func assertAuthRefresh(t *testing.T, refreshed bool, client *Client[any, *clientVaultAPIAuthStub], auth *clientVaultAPIAuthStub) { t.Helper() if auth.refreshed != refreshed { @@ -156,7 +156,7 @@ type clientVaultAPIAuthStub struct { refreshed bool } -func (a *clientVaultAPIAuthStub) Login(ctx context.Context, api any) (time.Duration, error) { +func (a *clientVaultAPIAuthStub) Login(_ context.Context, _ any) (time.Duration, error) { a.refreshed = true var err error if a.leaseDuration <= 0 { diff --git a/testdata/complete.yaml b/testdata/complete.yaml index 0939e91..ec6e6f9 100644 --- a/testdata/complete.yaml +++ b/testdata/complete.yaml @@ -24,7 +24,7 @@ vault: kubernetes: role: "test-kubernetes-role" path: "test-kubernetes-path" - jwtPath: "./jwt" + jwtToken: "file://./jwt" ldap: username: "test-ldap-user" password: "test-ldap-pass" @@ -43,15 +43,15 @@ snapshots: timestampFormat: "2006-01-02" uploaders: aws: + accessKeyId: test-key + accessKey: test-secret + sessionToken: test-session region: test-region bucket: test-bucket keyPrefix: test-prefix endpoint: test-endpoint useServerSideEncryption: true forcePathStyle: true - credentials: - key: test-key - secret: test-secret azure: accountName: test-account accountKey: test-key @@ -65,8 +65,8 @@ uploaders: container: test-container username: test-username apiKey: test-api-key - authUrl: http://auth.com - domain: http://user.com + authUrl: https://auth.com + domain: https://user.com region: test-region tenantId: test-tenant timeout: 180s diff --git a/testdata/defaults.yaml b/testdata/defaults.yaml index 12d9a12..b0c0c9d 100644 --- a/testdata/defaults.yaml +++ b/testdata/defaults.yaml @@ -2,6 +2,7 @@ vault: auth: kubernetes: role: "test-role" + jwtToken: "file://./jwt" uploaders: local: path: . diff --git a/testdata/empty.yaml b/testdata/empty.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/testdata/envvars.yaml b/testdata/envvars.yaml deleted file mode 100644 index b258e49..0000000 --- a/testdata/envvars.yaml +++ /dev/null @@ -1,8 +0,0 @@ -vault: - auth: - token: "secret" -uploaders: - aws: - endpoint: test-endpoint - region: test-region - bucket: test-bucket diff --git a/testdata/invalid-auth.yaml b/testdata/invalid-auth.yaml deleted file mode 100644 index c2bbba7..0000000 --- a/testdata/invalid-auth.yaml +++ /dev/null @@ -1,9 +0,0 @@ -vault: - auth: - approle: - role: "test" -snapshots: - frequency: "1h" -uploaders: - local: - path: . diff --git a/testdata/invalid-local-upload-path.yaml b/testdata/invalid-local-upload-path.yaml deleted file mode 100644 index fd5aa62..0000000 --- a/testdata/invalid-local-upload-path.yaml +++ /dev/null @@ -1,5 +0,0 @@ -snapshots: - frequency: "1h" -uploaders: - local: - path: ./missing diff --git a/testdata/invalid-uploader.yaml b/testdata/invalid-uploader.yaml deleted file mode 100644 index 156d6a1..0000000 --- a/testdata/invalid-uploader.yaml +++ /dev/null @@ -1,5 +0,0 @@ -snapshots: - frequency: "1h" -uploaders: - azure: - accountName: "test" \ No newline at end of file diff --git a/testdata/invalid-url.yaml b/testdata/invalid-url.yaml deleted file mode 100644 index f17f4b3..0000000 --- a/testdata/invalid-url.yaml +++ /dev/null @@ -1,7 +0,0 @@ -vault: - url: "invalid://uri" -snapshots: - frequency: "1h" -uploaders: - local: - path: . diff --git a/testdata/no-uploaders.yaml b/testdata/no-uploaders.yaml deleted file mode 100644 index 55457c6..0000000 --- a/testdata/no-uploaders.yaml +++ /dev/null @@ -1,3 +0,0 @@ -snapshots: - frequency: "1h" -uploaders: \ No newline at end of file diff --git a/testdata/watch-and-reconfigure1.yaml b/testdata/watch-and-reconfigure1.yaml deleted file mode 100644 index b4924eb..0000000 --- a/testdata/watch-and-reconfigure1.yaml +++ /dev/null @@ -1,10 +0,0 @@ -vault: - auth: - kubernetes: - role: "test-role" - jwtPath: "./jwt" -snapshots: - frequency: 30s -uploaders: - local: - path: . diff --git a/testdata/watch-and-reconfigure2.yaml b/testdata/watch-and-reconfigure2.yaml deleted file mode 100644 index ee55d2e..0000000 --- a/testdata/watch-and-reconfigure2.yaml +++ /dev/null @@ -1,10 +0,0 @@ -vault: - auth: - kubernetes: - role: "test-role" - jwtPath: "./jwt" -snapshots: - frequency: 60s -uploaders: - local: - path: .