Skip to content

Commit

Permalink
feat(remote_state): implement outputs fetching from gcs
Browse files Browse the repository at this point in the history 
This implements outputs fetching from the GCS backend, in a similar fashion to
what has been implemented for S3.

I've also added some misc fixes:
* correct typo: steateBody -> stateBody
* drop unused Path parameter on GCS config (it's not supported)

Tested on my architecture, everything seems to work!

Ref: https://developer.hashicorp.com/terraform/language/settings/backends/gcs
  • Loading branch information
@TPXP
TPXP committed Aug 7, 2024
1 parent 02ad89f commit cf09d17
Show file tree
Hide file tree
Showing 2 changed files with 83 additions and 25 deletions.
63 changes: 61 additions & 2 deletions config/dependency.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package config
import (
"bufio"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
Expand Down Expand Up @@ -813,6 +814,16 @@ func getTerragruntOutputJsonFromRemoteState(
}
ctx.TerragruntOptions.Logger.Debugf("Retrieved output from %s as json: %s using s3 bucket", targetConfigPath, jsonBytes)
return jsonBytes, nil
case "gcs":
jsonBytes, err := getTerragruntOutputJsonFromRemoteStateGCS(
targetTGOptions,
remoteState,
)
if err != nil {
return nil, err
}
ctx.TerragruntOptions.Logger.Debugf("Retrieved output from %s as json: %s using GCS bucket", targetConfigPath, jsonBytes)
return jsonBytes, nil
default:
ctx.TerragruntOptions.Logger.Errorf("FetchDependencyOutputFromState is not supported for backend %s, falling back to normal method", backend)
}
Expand Down Expand Up @@ -886,11 +897,59 @@ func getTerragruntOutputJsonFromRemoteStateS3(terragruntOptions *options.Terragr
terragruntOptions.Logger.Warnf("Failed to close remote state response %v", err)
}
}(result.Body)
steateBody, err := io.ReadAll(result.Body)
stateBody, err := io.ReadAll(result.Body)
if err != nil {
return nil, err
}
jsonState := string(stateBody)
jsonMap := make(map[string]interface{})
err = json.Unmarshal([]byte(jsonState), &jsonMap)
if err != nil {
return nil, err
}
jsonOutputs, err := json.Marshal(jsonMap["outputs"])
if err != nil {
return nil, err
}
return jsonOutputs, nil
}

// getTerragruntOutputJsonFromRemoteStateGCS pulls the output directly from an GCS bucket without calling Terraform
func getTerragruntOutputJsonFromRemoteStateGCS(terragruntOptions *options.TerragruntOptions, remoteState *remote.RemoteState) ([]byte, error) {
terragruntOptions.Logger.Debugf("Fetching outputs directly from gcs://%s/%s/default.tfstate", remoteState.Config["bucket"], remoteState.Config["prefix"], )

gcsConfigExtended, err := remote.ParseExtendedGCSConfig(remoteState.Config)
if err != nil {
return nil, err
}

if err := remote.ValidateGCSConfig(gcsConfigExtended); err != nil {
return nil, err
}

var gcsConfig = gcsConfigExtended.RemoteStateConfigGCS

gcsClient, err := remote.CreateGCSClient(gcsConfig)
if err != nil {
return nil, err
}

bucket := gcsClient.Bucket(gcsConfig.Bucket)
object := bucket.Object(gcsConfig.Prefix + "/default.tfstate");

reader, err := object.NewReader(context.Background())

if err != nil {
return nil, err
}

defer reader.Close()

stateBody, err := io.ReadAll(reader)
if err != nil {
return nil, err
}
jsonState := string(steateBody)
jsonState := string(stateBody)
jsonMap := make(map[string]interface{})
err = json.Unmarshal([]byte(jsonState), &jsonMap)
if err != nil {
Expand Down
45 changes: 22 additions & 23 deletions remote/remote_state_gcs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
* has to create them.
*/
type ExtendedRemoteStateConfigGCS struct {
remoteStateConfigGCS RemoteStateConfigGCS
RemoteStateConfigGCS RemoteStateConfigGCS

Project string `mapstructure:"project"`
Location string `mapstructure:"location"`
Expand All @@ -56,7 +56,6 @@ type RemoteStateConfigGCS struct {
Credentials string `mapstructure:"credentials"`
AccessToken string `mapstructure:"access_token"`
Prefix string `mapstructure:"prefix"`
Path string `mapstructure:"path"`
EncryptionKey string `mapstructure:"encryption_key"`

ImpersonateServiceAccount string `mapstructure:"impersonate_service_account"`
Expand Down Expand Up @@ -168,16 +167,16 @@ func (gcsInitializer GCSInitializer) buildInitializerCacheKey(gcsConfig *RemoteS
// Initialize the remote state GCS bucket specified in the given config. This function will validate the config
// parameters, create the GCS bucket if it doesn't already exist, and check that versioning is enabled.
func (gcsInitializer GCSInitializer) Initialize(ctx context.Context, remoteState *RemoteState, terragruntOptions *options.TerragruntOptions) error {
gcsConfigExtended, err := parseExtendedGCSConfig(remoteState.Config)
gcsConfigExtended, err := ParseExtendedGCSConfig(remoteState.Config)
if err != nil {
return err
}

if err := validateGCSConfig(gcsConfigExtended); err != nil {
if err := ValidateGCSConfig(gcsConfigExtended); err != nil {
return err
}

var gcsConfig = gcsConfigExtended.remoteStateConfigGCS
var gcsConfig = gcsConfigExtended.RemoteStateConfigGCS

cacheKey := gcsInitializer.buildInitializerCacheKey(&gcsConfig)
if initialized, hit := initializedRemoteStateCache.Get(ctx, cacheKey); initialized && hit {
Expand Down Expand Up @@ -242,7 +241,7 @@ func parseGCSConfig(config map[string]interface{}) (*RemoteStateConfigGCS, error
}

// Parse the given map into a GCS config
func parseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteStateConfigGCS, error) {
func ParseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteStateConfigGCS, error) {
var gcsConfig RemoteStateConfigGCS
var extendedConfig ExtendedRemoteStateConfigGCS

Expand All @@ -254,14 +253,14 @@ func parseExtendedGCSConfig(config map[string]interface{}) (*ExtendedRemoteState
return nil, errors.WithStackTrace(err)
}

extendedConfig.remoteStateConfigGCS = gcsConfig
extendedConfig.RemoteStateConfigGCS = gcsConfig

return &extendedConfig, nil
}

// Validate all the parameters of the given GCS remote state configuration
func validateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS) error {
var config = extendedConfig.remoteStateConfigGCS
func ValidateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS) error {
var config = extendedConfig.RemoteStateConfigGCS

if config.Bucket == "" {
return errors.WithStackTrace(MissingRequiredGCSRemoteStateConfig("bucket"))
Expand All @@ -273,8 +272,8 @@ func validateGCSConfig(extendedConfig *ExtendedRemoteStateConfigGCS) error {
// If the bucket specified in the given config doesn't already exist, prompt the user to create it, and if the user
// confirms, create the bucket and enable versioning for it.
func createGCSBucketIfNecessary(ctx context.Context, gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
if !DoesGCSBucketExist(gcsClient, &config.remoteStateConfigGCS) {
terragruntOptions.Logger.Debugf("Remote state GCS bucket %s does not exist. Attempting to create it", config.remoteStateConfigGCS.Bucket)
if !DoesGCSBucketExist(gcsClient, &config.RemoteStateConfigGCS) {
terragruntOptions.Logger.Debugf("Remote state GCS bucket %s does not exist. Attempting to create it", config.RemoteStateConfigGCS.Bucket)

// A project must be specified in order for terragrunt to automatically create a storage bucket.
if config.Project == "" {
Expand All @@ -287,18 +286,18 @@ func createGCSBucketIfNecessary(ctx context.Context, gcsClient *storage.Client,
}

if terragruntOptions.FailIfBucketCreationRequired {
return BucketCreationNotAllowed(config.remoteStateConfigGCS.Bucket)
return BucketCreationNotAllowed(config.RemoteStateConfigGCS.Bucket)
}

prompt := fmt.Sprintf("Remote state GCS bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.remoteStateConfigGCS.Bucket)
prompt := fmt.Sprintf("Remote state GCS bucket %s does not exist or you don't have permissions to access it. Would you like Terragrunt to create it?", config.RemoteStateConfigGCS.Bucket)
shouldCreateBucket, err := shell.PromptUserForYesNo(prompt, terragruntOptions)
if err != nil {
return err
}

if shouldCreateBucket {
// To avoid any eventual consistency issues with creating a GCS bucket we use a retry loop.
description := fmt.Sprintf("Create GCS bucket %s", config.remoteStateConfigGCS.Bucket)
description := fmt.Sprintf("Create GCS bucket %s", config.RemoteStateConfigGCS.Bucket)

return util.DoWithRetry(ctx, description, gcpMaxRetries, gcpSleepBetweenRetries, logrus.DebugLevel, func(ctx context.Context) error {
return CreateGCSBucketWithVersioning(gcsClient, config, terragruntOptions)
Expand Down Expand Up @@ -335,7 +334,7 @@ func CreateGCSBucketWithVersioning(gcsClient *storage.Client, config *ExtendedRe
return err
}

if err := WaitUntilGCSBucketExists(gcsClient, &config.remoteStateConfigGCS, terragruntOptions); err != nil {
if err := WaitUntilGCSBucketExists(gcsClient, &config.RemoteStateConfigGCS, terragruntOptions); err != nil {
return err
}

Expand All @@ -348,14 +347,14 @@ func CreateGCSBucketWithVersioning(gcsClient *storage.Client, config *ExtendedRe

func AddLabelsToGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
if config.GCSBucketLabels == nil || len(config.GCSBucketLabels) == 0 {
terragruntOptions.Logger.Debugf("No labels specified for bucket %s.", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("No labels specified for bucket %s.", config.RemoteStateConfigGCS.Bucket)
return nil
}

terragruntOptions.Logger.Debugf("Adding labels to GCS bucket with %s", config.GCSBucketLabels)

ctx := context.Background()
bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)
bucket := gcsClient.Bucket(config.RemoteStateConfigGCS.Bucket)

bucketAttrs := storage.BucketAttrsToUpdate{}

Expand All @@ -375,15 +374,15 @@ func AddLabelsToGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteState

// Create the GCS bucket specified in the given config
func CreateGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfigGCS, terragruntOptions *options.TerragruntOptions) error {
terragruntOptions.Logger.Debugf("Creating GCS bucket %s in project %s", config.remoteStateConfigGCS.Bucket, config.Project)
terragruntOptions.Logger.Debugf("Creating GCS bucket %s in project %s", config.RemoteStateConfigGCS.Bucket, config.Project)

// The project ID to which the bucket belongs. This is only used when creating a new bucket during initialization.
// Since buckets have globally unique names, the project ID is not required to access the bucket during normal
// operation.
projectID := config.Project

ctx := context.Background()
bucket := gcsClient.Bucket(config.remoteStateConfigGCS.Bucket)
bucket := gcsClient.Bucket(config.RemoteStateConfigGCS.Bucket)

bucketAttrs := &storage.BucketAttrs{}

Expand All @@ -393,19 +392,19 @@ func CreateGCSBucket(gcsClient *storage.Client, config *ExtendedRemoteStateConfi
}

if config.SkipBucketVersioning {
terragruntOptions.Logger.Debugf("Versioning is disabled for the remote state GCS bucket %s using 'skip_bucket_versioning' config.", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("Versioning is disabled for the remote state GCS bucket %s using 'skip_bucket_versioning' config.", config.RemoteStateConfigGCS.Bucket)
} else {
terragruntOptions.Logger.Debugf("Enabling versioning on GCS bucket %s", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("Enabling versioning on GCS bucket %s", config.RemoteStateConfigGCS.Bucket)
bucketAttrs.VersioningEnabled = true
}

if config.EnableBucketPolicyOnly {
terragruntOptions.Logger.Debugf("Enabling uniform bucket-level access on GCS bucket %s", config.remoteStateConfigGCS.Bucket)
terragruntOptions.Logger.Debugf("Enabling uniform bucket-level access on GCS bucket %s", config.RemoteStateConfigGCS.Bucket)
bucketAttrs.BucketPolicyOnly = storage.BucketPolicyOnly{Enabled: true}
}

err := bucket.Create(ctx, projectID, bucketAttrs)
return errors.WithStackTraceAndPrefix(err, "Error creating GCS bucket %s", config.remoteStateConfigGCS.Bucket)
return errors.WithStackTraceAndPrefix(err, "Error creating GCS bucket %s", config.RemoteStateConfigGCS.Bucket)
}

// GCP is eventually consistent, so after creating a GCS bucket, this method can be used to wait until the information
Expand Down

0 comments on commit cf09d17

Please sign in to comment.