From 6504d0ea3c18f19076f02ce75167bcc927f61109 Mon Sep 17 00:00:00 2001 From: wai-wong-edb <119956756+wai-wong-edb@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:29:46 +0000 Subject: [PATCH 1/2] feat: wal storage (#608) * feat: wal storage * feat: data sources * feat: read only connection bug * fix: requested changes * fix: azurepremiumstorage example comments --- .../biganimal_cluster/data-source.tf | 4 +++ .../biganimal_faraway_replica/data-source.tf | 4 +++ .../biganimal_cluster/ha/resource.tf | 6 ++++ .../single_node/aws/resource.tf | 6 ++++ .../single_node/azure/resource.tf | 8 ++++- .../single_node/gcp/resource.tf | 6 ++++ .../biganimal_faraway_replica/aws/resource.tf | 5 +++ .../azure/resource.tf | 7 +++- .../cluster_and_faraway_replica/resource.tf | 15 ++++++-- .../biganimal_faraway_replica/gcp/resource.tf | 5 +++ .../biganimal_pgd/aws/data_group/resource.tf | 5 +++ .../resource.tf | 10 ++++++ .../azure/data_group/resource.tf | 10 ++++++ .../resource.tf | 10 ++++++ .../biganimal_pgd/gcp/data_group/resource.tf | 5 +++ .../resource.tf | 10 ++++++ pkg/models/cluster.go | 1 + pkg/models/pgd/api/data_group.go | 1 + pkg/models/pgd/terraform/data_group.go | 1 + pkg/plan_modifier/data_group_custom_diff.go | 5 +++ pkg/provider/common.go | 35 +++++++++++++++++++ pkg/provider/data_source_cluster.go | 1 + pkg/provider/data_source_fareplica.go | 1 + pkg/provider/data_source_pgd.go | 1 + pkg/provider/resource_cluster.go | 21 ++++++++++- pkg/provider/resource_fareplica.go | 19 ++++++++++ pkg/provider/resource_pgd.go | 27 +++++++++++++- 27 files changed, 223 insertions(+), 6 deletions(-) diff --git a/examples/data-sources/biganimal_cluster/data-source.tf b/examples/data-sources/biganimal_cluster/data-source.tf index 0931e502..fc0ed105 100644 --- a/examples/data-sources/biganimal_cluster/data-source.tf +++ b/examples/data-sources/biganimal_cluster/data-source.tf @@ -89,6 +89,10 @@ output "storage" { value = data.biganimal_cluster.this.storage } +output "wal_storage" { + value = data.biganimal_cluster.this.wal_storage +} + output "superuser_access" { value = coalesce(data.biganimal_cluster.this.superuser_access, false) } diff --git a/examples/data-sources/biganimal_faraway_replica/data-source.tf b/examples/data-sources/biganimal_faraway_replica/data-source.tf index 92d9b0af..ca60f853 100644 --- a/examples/data-sources/biganimal_faraway_replica/data-source.tf +++ b/examples/data-sources/biganimal_faraway_replica/data-source.tf @@ -89,6 +89,10 @@ output "storage" { value = data.biganimal_faraway_replica.this.storage } +output "wal_storage" { + value = data.biganimal_faraway_replica.this.wal_storage +} + output "volume_snapshot_backup" { value = data.biganimal_faraway_replica.this.volume_snapshot_backup } diff --git a/examples/resources/biganimal_cluster/ha/resource.tf b/examples/resources/biganimal_cluster/ha/resource.tf index 45600681..bd8bd140 100644 --- a/examples/resources/biganimal_cluster/ha/resource.tf +++ b/examples/resources/biganimal_cluster/ha/resource.tf @@ -68,6 +68,12 @@ resource "biganimal_cluster" "ha_cluster" { size = "4 Gi" } + # wal_storage = { + # volume_type = "gp3" + # volume_properties = "gp3" + # size = "4 Gi" + # } + maintenance_window = { is_enabled = true start_day = 6 diff --git a/examples/resources/biganimal_cluster/single_node/aws/resource.tf b/examples/resources/biganimal_cluster/single_node/aws/resource.tf index 5ca4f7e5..c98105a6 100644 --- a/examples/resources/biganimal_cluster/single_node/aws/resource.tf +++ b/examples/resources/biganimal_cluster/single_node/aws/resource.tf @@ -69,6 +69,12 @@ resource "biganimal_cluster" "single_node_cluster" { size = "4 Gi" } + # wal_storage = { + # volume_type = "gp3" + # volume_properties = "gp3" + # size = "4 Gi" + # } + maintenance_window = { is_enabled = true start_day = 6 diff --git a/examples/resources/biganimal_cluster/single_node/azure/resource.tf b/examples/resources/biganimal_cluster/single_node/azure/resource.tf index fc6d4abe..034ddb78 100644 --- a/examples/resources/biganimal_cluster/single_node/azure/resource.tf +++ b/examples/resources/biganimal_cluster/single_node/azure/resource.tf @@ -66,9 +66,15 @@ resource "biganimal_cluster" "single_node_cluster" { storage = { volume_type = "azurepremiumstorage" volume_properties = "P1" - size = "4 Gi" + size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance } + # wal_storage = { + # volume_type = "azurepremiumstorage" + # volume_properties = "P1" + # size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance + # } + maintenance_window = { is_enabled = true start_day = 6 diff --git a/examples/resources/biganimal_cluster/single_node/gcp/resource.tf b/examples/resources/biganimal_cluster/single_node/gcp/resource.tf index e6ad7932..4e702e43 100644 --- a/examples/resources/biganimal_cluster/single_node/gcp/resource.tf +++ b/examples/resources/biganimal_cluster/single_node/gcp/resource.tf @@ -69,6 +69,12 @@ resource "biganimal_cluster" "single_node_cluster" { size = "10 Gi" } + # wal_storage = { + # volume_type = "pd-ssd" + # volume_properties = "pd-ssd" + # size = "10 Gi" + # } + maintenance_window = { is_enabled = true start_day = 6 diff --git a/examples/resources/biganimal_faraway_replica/aws/resource.tf b/examples/resources/biganimal_faraway_replica/aws/resource.tf index e9826b08..8cf1bbb4 100644 --- a/examples/resources/biganimal_faraway_replica/aws/resource.tf +++ b/examples/resources/biganimal_faraway_replica/aws/resource.tf @@ -66,6 +66,11 @@ resource "biganimal_faraway_replica" "faraway_replica" { volume_properties = "gp3" size = "4 Gi" } + # wal_storage = { + # volume_type = "gp3" + # volume_properties = "gp3" + # size = "4 Gi" + # } private_networking = false region = "ap-south-1" diff --git a/examples/resources/biganimal_faraway_replica/azure/resource.tf b/examples/resources/biganimal_faraway_replica/azure/resource.tf index 7b1c98a4..17f522dd 100644 --- a/examples/resources/biganimal_faraway_replica/azure/resource.tf +++ b/examples/resources/biganimal_faraway_replica/azure/resource.tf @@ -64,8 +64,13 @@ resource "biganimal_faraway_replica" "faraway_replica" { storage = { volume_type = "azurepremiumstorage" volume_properties = "P1" - size = "4 Gi" + size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance } + # wal_storage = { + # volume_type = "azurepremiumstorage" + # volume_properties = "P1" + # size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance + # } private_networking = false region = "australiaeast" diff --git a/examples/resources/biganimal_faraway_replica/cluster_and_faraway_replica/resource.tf b/examples/resources/biganimal_faraway_replica/cluster_and_faraway_replica/resource.tf index 3b4df3bf..931f92a0 100644 --- a/examples/resources/biganimal_faraway_replica/cluster_and_faraway_replica/resource.tf +++ b/examples/resources/biganimal_faraway_replica/cluster_and_faraway_replica/resource.tf @@ -42,9 +42,15 @@ resource "biganimal_cluster" "single_node_cluster" { storage = { volume_type = "azurepremiumstorage" volume_properties = "P1" - size = "4 Gi" + size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance } + # wal_storage = { + # volume_type = "azurepremiumstorage" + # volume_properties = "P1" + # size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance + # } + pg_type = "epas" #valid values ["epas", "pgextended", "postgres]" pg_version = "15" cloud_provider = "azure" @@ -100,8 +106,13 @@ resource "biganimal_faraway_replica" "faraway_replica" { storage = { volume_type = "azurepremiumstorage" volume_properties = "P1" - size = "4 Gi" + size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance } + # wal_storage = { + # volume_type = "azurepremiumstorage" + # volume_properties = "P1" + # size = "4 Gi" # for azurepremiumstorage please check Premium storage disk sizes here: https://learn.microsoft.com/en-us/azure/virtual-machines/premium-storage-performance + # } private_networking = false region = "centralindia" diff --git a/examples/resources/biganimal_faraway_replica/gcp/resource.tf b/examples/resources/biganimal_faraway_replica/gcp/resource.tf index e9994cfd..9381e555 100644 --- a/examples/resources/biganimal_faraway_replica/gcp/resource.tf +++ b/examples/resources/biganimal_faraway_replica/gcp/resource.tf @@ -66,6 +66,11 @@ resource "biganimal_faraway_replica" "faraway_replica" { volume_properties = "pd-ssd" size = "4 Gi" } + # wal_storage = { + # volume_type = "pd-ssd" + # volume_properties = "pd-ssd" + # size = "4 Gi" + # } private_networking = false region = "us-east1" diff --git a/examples/resources/biganimal_pgd/aws/data_group/resource.tf b/examples/resources/biganimal_pgd/aws/data_group/resource.tf index 4a672731..bcab1513 100644 --- a/examples/resources/biganimal_pgd/aws/data_group/resource.tf +++ b/examples/resources/biganimal_pgd/aws/data_group/resource.tf @@ -77,6 +77,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "gp3" size = "32 Gi" } + # wal_storage = { + # volume_type = "gp3" + # volume_properties = "gp3" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } diff --git a/examples/resources/biganimal_pgd/aws/data_groups_with_witness_group/resource.tf b/examples/resources/biganimal_pgd/aws/data_groups_with_witness_group/resource.tf index 8bae42df..479f5530 100644 --- a/examples/resources/biganimal_pgd/aws/data_groups_with_witness_group/resource.tf +++ b/examples/resources/biganimal_pgd/aws/data_groups_with_witness_group/resource.tf @@ -77,6 +77,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "gp3" size = "32 Gi" } + # wal_storage = { + # volume_type = "gp3" + # volume_properties = "gp3" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } @@ -134,6 +139,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "gp3" size = "32 Gi" } + # wal_storage = { + # volume_type = "gp3" + # volume_properties = "gp3" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } diff --git a/examples/resources/biganimal_pgd/azure/data_group/resource.tf b/examples/resources/biganimal_pgd/azure/data_group/resource.tf index 2cecef13..b9a33336 100644 --- a/examples/resources/biganimal_pgd/azure/data_group/resource.tf +++ b/examples/resources/biganimal_pgd/azure/data_group/resource.tf @@ -77,6 +77,16 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "P2" size = "32 Gi" } + storage = { + volume_type = "azurepremiumstorage" + volume_properties = "P2" + size = "32 Gi" + } + # wal_storage = { + # volume_type = "azurepremiumstorage" + # volume_properties = "P2" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } diff --git a/examples/resources/biganimal_pgd/azure/data_groups_with_witness_group/resource.tf b/examples/resources/biganimal_pgd/azure/data_groups_with_witness_group/resource.tf index cb181ec1..13b43fd9 100644 --- a/examples/resources/biganimal_pgd/azure/data_groups_with_witness_group/resource.tf +++ b/examples/resources/biganimal_pgd/azure/data_groups_with_witness_group/resource.tf @@ -77,6 +77,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "P2" size = "32 Gi" } + # wal_storage = { + # volume_type = "azurepremiumstorage" + # volume_properties = "P2" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } @@ -134,6 +139,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "P2" size = "32 Gi" } + # wal_storage = { + # volume_type = "azurepremiumstorage" + # volume_properties = "P2" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } diff --git a/examples/resources/biganimal_pgd/gcp/data_group/resource.tf b/examples/resources/biganimal_pgd/gcp/data_group/resource.tf index 64281147..9899dae9 100644 --- a/examples/resources/biganimal_pgd/gcp/data_group/resource.tf +++ b/examples/resources/biganimal_pgd/gcp/data_group/resource.tf @@ -77,6 +77,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "pd-ssd" size = "32 Gi" } + # wal_storage = { + # volume_type = "pd-ssd" + # volume_properties = "pd-ssd" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } diff --git a/examples/resources/biganimal_pgd/gcp/data_groups_with_witness_group/resource.tf b/examples/resources/biganimal_pgd/gcp/data_groups_with_witness_group/resource.tf index cd2bf536..f3f8ab56 100644 --- a/examples/resources/biganimal_pgd/gcp/data_groups_with_witness_group/resource.tf +++ b/examples/resources/biganimal_pgd/gcp/data_groups_with_witness_group/resource.tf @@ -77,6 +77,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "pd-ssd" size = "32 Gi" } + # wal_storage = { + # volume_type = "pd-ssd" + # volume_properties = "pd-ssd" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } @@ -138,6 +143,11 @@ resource "biganimal_pgd" "pgd_cluster" { volume_properties = "pd-ssd" size = "32 Gi" } + # wal_storage = { + # volume_type = "pd-ssd" + # volume_properties = "pd-ssd" + # size = "32 Gi" + # } pg_type = { pg_type_id = "epas" #valid values ["epas", "pgextended", "postgres]" } diff --git a/pkg/models/cluster.go b/pkg/models/cluster.go index 2684159c..c5bc2a95 100644 --- a/pkg/models/cluster.go +++ b/pkg/models/cluster.go @@ -186,6 +186,7 @@ type Cluster struct { EncryptionKeyIdReq *string `json:"keyId,omitempty"` EncryptionKeyResp *EncryptionKey `json:"encryptionKey,omitempty"` PgIdentity *string `json:"pgIdentity,omitempty"` + WalStorage *Storage `json:"walStorage,omitempty"` } // IsHealthy checks to see if the cluster has the right condition 'biganimal.com/deployed' diff --git a/pkg/models/pgd/api/data_group.go b/pkg/models/pgd/api/data_group.go index 8821c892..b71495f3 100644 --- a/pkg/models/pgd/api/data_group.go +++ b/pkg/models/pgd/api/data_group.go @@ -30,4 +30,5 @@ type DataGroup struct { PeAllowedPrincipalIds *[]string `json:"peAllowedPrincipalIds,omitempty"` RoConnectionUri *string `json:"roConnectionUri,omitempty"` ReadOnlyConnections *bool `json:"readOnlyConnections,omitempty"` + WalStorage *models.Storage `json:"walStorage,omitempty"` } diff --git a/pkg/models/pgd/terraform/data_group.go b/pkg/models/pgd/terraform/data_group.go index 4756d334..aa31b6eb 100644 --- a/pkg/models/pgd/terraform/data_group.go +++ b/pkg/models/pgd/terraform/data_group.go @@ -33,4 +33,5 @@ type DataGroup struct { PeAllowedPrincipalIds types.Set `tfsdk:"pe_allowed_principal_ids"` RoConnectionUri types.String `tfsdk:"ro_connection_uri"` ReadOnlyConnections *bool `tfsdk:"read_only_connections"` + WalStorage *Storage `tfsdk:"wal_storage"` } diff --git a/pkg/plan_modifier/data_group_custom_diff.go b/pkg/plan_modifier/data_group_custom_diff.go index 0df65780..2facbb45 100644 --- a/pkg/plan_modifier/data_group_custom_diff.go +++ b/pkg/plan_modifier/data_group_custom_diff.go @@ -126,6 +126,11 @@ func (m CustomDataGroupDiffModifier) PlanModifyList(ctx context.Context, req pla pDg.Storage.Iops = sDg.Storage.Iops pDg.Storage.Throughput = sDg.Storage.Throughput + if sDg.WalStorage != nil { + pDg.WalStorage.Iops = sDg.WalStorage.Iops + pDg.WalStorage.Throughput = sDg.WalStorage.Throughput + } + // fix to set the correct allowed ip ranges to allow all if a PGD data group has private networking set as true if pDg.PrivateNetworking != nil && *pDg.PrivateNetworking { pDg.AllowedIpRanges = types.SetValueMust(pDg.AllowedIpRanges.ElementType(ctx), []attr.Value{ diff --git a/pkg/provider/common.go b/pkg/provider/common.go index 212db12e..0cbcc659 100644 --- a/pkg/provider/common.go +++ b/pkg/provider/common.go @@ -3,6 +3,9 @@ package provider import ( commonApi "github.com/EnterpriseDB/terraform-provider-biganimal/pkg/models/common/api" commonTerraform "github.com/EnterpriseDB/terraform-provider-biganimal/pkg/models/common/terraform" + "github.com/hashicorp/terraform-plugin-framework/resource/schema" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" + "github.com/hashicorp/terraform-plugin-framework/resource/schema/stringplanmodifier" "github.com/hashicorp/terraform-plugin-framework/types" "github.com/hashicorp/terraform-plugin-framework/types/basetypes" ) @@ -31,3 +34,35 @@ func buildAPIReqAssignTags(tfRsrcTags []commonTerraform.Tag) []commonApi.Tag { } return tags } + +var resourceWal = schema.SingleNestedAttribute{ + Description: "Use a separate storage volume for Write-Ahead Logs (Recommended for high write workloads)", + Optional: true, + Attributes: map[string]schema.Attribute{ + "iops": schema.StringAttribute{ + Description: "IOPS for the selected volume. It can be set to different values depending on your volume type and properties.", + Optional: true, + Computed: true, + PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, + }, + "size": schema.StringAttribute{ + Description: "Size of the volume. It can be set to different values depending on your volume type and properties.", + Required: true, + PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, + }, + "throughput": schema.StringAttribute{ + Description: "Throughput is automatically calculated by BigAnimal based on the IOPS input if it's not provided.", + Optional: true, + Computed: true, + PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, + }, + "volume_properties": schema.StringAttribute{ + Description: "Volume properties in accordance with the selected volume type.", + Required: true, + }, + "volume_type": schema.StringAttribute{ + Description: "Volume type. For Azure: \"azurepremiumstorage\" or \"ultradisk\". For AWS: \"gp3\", \"io2\", org s \"io2-block-express\". For Google Cloud: only \"pd-ssd\".", + Required: true, + }, + }, +} diff --git a/pkg/provider/data_source_cluster.go b/pkg/provider/data_source_cluster.go index fc876772..5dd02003 100644 --- a/pkg/provider/data_source_cluster.go +++ b/pkg/provider/data_source_cluster.go @@ -388,6 +388,7 @@ func (c *clusterDataSource) Schema(ctx context.Context, req datasource.SchemaReq MarkdownDescription: "Cluster connection service name.", Computed: true, }, + "wal_storage": resourceWal, }, } } diff --git a/pkg/provider/data_source_fareplica.go b/pkg/provider/data_source_fareplica.go index d3ca9d21..8ff5b3df 100644 --- a/pkg/provider/data_source_fareplica.go +++ b/pkg/provider/data_source_fareplica.go @@ -271,6 +271,7 @@ func (c *FAReplicaData) Schema(ctx context.Context, req datasource.SchemaRequest }, }, }, + "wal_storage": resourceWal, }, } } diff --git a/pkg/provider/data_source_pgd.go b/pkg/provider/data_source_pgd.go index fb6a5abe..c0f587a9 100644 --- a/pkg/provider/data_source_pgd.go +++ b/pkg/provider/data_source_pgd.go @@ -284,6 +284,7 @@ func (p pgdDataSource) Schema(ctx context.Context, req datasource.SchemaRequest, Description: "Is read-only connections enabled.", Computed: true, }, + "wal_storage": resourceWal, }, }, }, diff --git a/pkg/provider/resource_cluster.go b/pkg/provider/resource_cluster.go index 83023578..5855ae53 100644 --- a/pkg/provider/resource_cluster.go +++ b/pkg/provider/resource_cluster.go @@ -82,6 +82,7 @@ type ClusterResourceModel struct { VolumeSnapshot types.Bool `tfsdk:"volume_snapshot_backup"` Tags []commonTerraform.Tag `tfsdk:"tags"` ServiceName types.String `tfsdk:"service_name"` + WalStorage *StorageResourceModel `tfsdk:"wal_storage"` Timeouts timeouts.Value `tfsdk:"timeouts"` } @@ -272,7 +273,7 @@ func (c *clusterResource) Schema(ctx context.Context, req resource.SchemaRequest Required: true, }, "volume_type": schema.StringAttribute{ - Description: "Volume type. For Azure: \"azurepremiumstorage\" or \"ultradisk\". For AWS: \"gp3\", \"io2\", org s \"io2-block-express\". For Google Cloud: only \"pd-ssd\".", + Description: "Volume type. For Azure: \"azurepremiumstorage\" or \"ultradisk\". For AWS: \"gp3\", \"io2\", or \"io2-block-express\". For Google Cloud: only \"pd-ssd\".", Required: true, }, }, @@ -575,6 +576,7 @@ func (c *clusterResource) Schema(ctx context.Context, req resource.SchemaRequest Computed: true, PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, }, + "wal_storage": resourceWal, }, } } @@ -863,6 +865,13 @@ func readCluster(ctx context.Context, client *api.ClusterClient, tfClusterResour tfClusterResource.SuperuserAccess = types.BoolPointerValue(responseCluster.SuperuserAccess) tfClusterResource.PgIdentity = types.StringPointerValue(responseCluster.PgIdentity) tfClusterResource.VolumeSnapshot = types.BoolPointerValue(responseCluster.VolumeSnapshot) + tfClusterResource.WalStorage = &StorageResourceModel{ + VolumeType: types.StringPointerValue(responseCluster.WalStorage.VolumeTypeId), + VolumeProperties: types.StringPointerValue(responseCluster.WalStorage.VolumePropertiesId), + Size: types.StringPointerValue(responseCluster.WalStorage.Size), + Iops: types.StringPointerValue(responseCluster.WalStorage.Iops), + Throughput: types.StringPointerValue(responseCluster.WalStorage.Throughput), + } if responseCluster.EncryptionKeyResp != nil && *responseCluster.Phase != constants.PHASE_HEALTHY { if !tfClusterResource.PgIdentity.IsNull() && tfClusterResource.PgIdentity.ValueString() != "" { @@ -1102,6 +1111,16 @@ func (c *clusterResource) generateGenericClusterModel(ctx context.Context, clust VolumeSnapshot: clusterResource.VolumeSnapshot.ValueBoolPointer(), } + if clusterResource.WalStorage != nil { + cluster.WalStorage = &models.Storage{ + VolumePropertiesId: clusterResource.WalStorage.VolumeProperties.ValueStringPointer(), + VolumeTypeId: clusterResource.WalStorage.VolumeType.ValueStringPointer(), + Iops: clusterResource.WalStorage.Iops.ValueStringPointer(), + Size: clusterResource.WalStorage.Size.ValueStringPointer(), + Throughput: clusterResource.WalStorage.Throughput.ValueStringPointer(), + } + } + cluster.Extensions = &[]models.ClusterExtension{} if clusterResource.Pgvector.ValueBool() { *cluster.Extensions = append(*cluster.Extensions, models.ClusterExtension{Enabled: true, ExtensionId: "pgvector"}) diff --git a/pkg/provider/resource_fareplica.go b/pkg/provider/resource_fareplica.go index 25012d6b..21b56dc4 100644 --- a/pkg/provider/resource_fareplica.go +++ b/pkg/provider/resource_fareplica.go @@ -65,6 +65,7 @@ type FAReplicaResourceModel struct { TransparentDataEncryptionAction types.String `tfsdk:"transparent_data_encryption_action"` VolumeSnapshot types.Bool `tfsdk:"volume_snapshot_backup"` Tags []commonTerraform.Tag `tfsdk:"tags"` + WalStorage *StorageResourceModel `tfsdk:"wal_storage"` Timeouts timeouts.Value `tfsdk:"timeouts"` } @@ -422,6 +423,7 @@ func (r *FAReplicaResource) Schema(ctx context.Context, req resource.SchemaReque plan_modifier.CustomAssignTags(), }, }, + "wal_storage": resourceWal, }, } } @@ -636,6 +638,13 @@ func readFAReplica(ctx context.Context, client *api.ClusterClient, fAReplicaReso fAReplicaResourceModel.PgVersion = types.StringValue(responseCluster.PgVersion.PgVersionId) fAReplicaResourceModel.PgType = types.StringValue(responseCluster.PgType.PgTypeId) fAReplicaResourceModel.VolumeSnapshot = types.BoolPointerValue(responseCluster.VolumeSnapshot) + fAReplicaResourceModel.WalStorage = &StorageResourceModel{ + VolumeType: types.StringPointerValue(responseCluster.WalStorage.VolumeTypeId), + VolumeProperties: types.StringPointerValue(responseCluster.WalStorage.VolumePropertiesId), + Size: types.StringPointerValue(responseCluster.WalStorage.Size), + Iops: types.StringPointerValue(responseCluster.WalStorage.Iops), + Throughput: types.StringPointerValue(responseCluster.WalStorage.Throughput), + } // pgConfig. If tf resource pg config elem matches with api response pg config elem then add the elem to tf resource pg config newPgConfig := []PgConfigResourceModel{} @@ -770,6 +779,16 @@ func (r *FAReplicaResource) generateGenericFAReplicaModel(ctx context.Context, f BackupRetentionPeriod: fAReplicaResourceModel.BackupRetentionPeriod.ValueStringPointer(), } + if fAReplicaResourceModel.WalStorage != nil { + cluster.WalStorage = &models.Storage{ + VolumePropertiesId: fAReplicaResourceModel.WalStorage.VolumeProperties.ValueStringPointer(), + VolumeTypeId: fAReplicaResourceModel.WalStorage.VolumeType.ValueStringPointer(), + Iops: fAReplicaResourceModel.WalStorage.Iops.ValueStringPointer(), + Size: fAReplicaResourceModel.WalStorage.Size.ValueStringPointer(), + Throughput: fAReplicaResourceModel.WalStorage.Throughput.ValueStringPointer(), + } + } + allowedIpRanges := []models.AllowedIpRange{} for _, ipRange := range fAReplicaResourceModel.AllowedIpRanges { allowedIpRanges = append(allowedIpRanges, models.AllowedIpRange{ diff --git a/pkg/provider/resource_pgd.go b/pkg/provider/resource_pgd.go index cc1792c7..9ce1744e 100644 --- a/pkg/provider/resource_pgd.go +++ b/pkg/provider/resource_pgd.go @@ -415,8 +415,8 @@ func PgdSchema(ctx context.Context) schema.Schema { "read_only_connections": schema.BoolAttribute{ Description: "Is read-only connections enabled.", Optional: true, - Computed: true, }, + "wal_storage": resourceWal, }, }, }, @@ -666,6 +666,11 @@ func (p pgdResource) Create(ctx context.Context, req resource.CreateRequest, res storage := buildRequestStorage(*v.Storage) + var walStorage *models.Storage + if v.WalStorage != nil { + walStorage = buildRequestStorage(*v.WalStorage) + } + if v.PgConfig == nil { v.PgConfig = &[]models.KeyValue{} } @@ -704,6 +709,7 @@ func (p pgdResource) Create(ctx context.Context, req resource.CreateRequest, res ServiceAccountIds: svAccIds, PeAllowedPrincipalIds: principalIds, ReadOnlyConnections: v.ReadOnlyConnections, + WalStorage: walStorage, } *clusterReqBody.Groups = append(*clusterReqBody.Groups, apiDGModel) } @@ -953,6 +959,11 @@ func (p pgdResource) Update(ctx context.Context, req resource.UpdateRequest, res for _, v := range plan.DataGroups { storage := buildRequestStorage(*v.Storage) + var walStorage *models.Storage + if v.WalStorage != nil { + walStorage = buildRequestStorage(*v.WalStorage) + } + groupId := v.GroupId.ValueStringPointer() if v.GroupId.IsUnknown() { groupId = nil @@ -977,6 +988,7 @@ func (p pgdResource) Update(ctx context.Context, req resource.UpdateRequest, res MaintenanceWindow: v.MaintenanceWindow, ServiceAccountIds: svAccIds, PeAllowedPrincipalIds: principalIds, + WalStorage: walStorage, } // signals that it doesn't have an existing group id so this is a new group to add and needs extra fields @@ -1337,6 +1349,18 @@ func buildTFGroupsAs(ctx context.Context, diags *diag.Diagnostics, state tfsdk.S Throughput: types.StringPointerValue(apiRespDgModel.Storage.Throughput), } + // wal storage + var walStorage *terraform.Storage + if apiRespDgModel.WalStorage != nil { + walStorage = &terraform.Storage{ + Size: types.StringPointerValue(apiRespDgModel.WalStorage.Size), + VolumePropertiesId: types.StringPointerValue(apiRespDgModel.WalStorage.VolumePropertiesId), + VolumeTypeId: types.StringPointerValue(apiRespDgModel.WalStorage.VolumeTypeId), + Iops: types.StringPointerValue(apiRespDgModel.WalStorage.Iops), + Throughput: types.StringPointerValue(apiRespDgModel.WalStorage.Throughput), + } + } + // service account ids serviceAccIds := []attr.Value{} if apiRespDgModel.ServiceAccountIds != nil && len(*apiRespDgModel.ServiceAccountIds) != 0 { @@ -1421,6 +1445,7 @@ func buildTFGroupsAs(ctx context.Context, diags *diag.Diagnostics, state tfsdk.S PeAllowedPrincipalIds: types.SetValueMust(types.StringType, principalIds), RoConnectionUri: types.StringPointerValue(apiRespDgModel.RoConnectionUri), ReadOnlyConnections: apiRespDgModel.ReadOnlyConnections, + WalStorage: walStorage, } outPgdTFResource.DataGroups = append(outPgdTFResource.DataGroups, tfDGModel) From 4a868a570ae1293bd64ce29ef92a02bb95d4d089 Mon Sep 17 00:00:00 2001 From: wai-wong-edb <119956756+wai-wong-edb@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:30:29 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20validation=20check.=20Prompt=20to=20?= =?UTF-8?q?remove=20pe=20ids=20and=20service=20ids=20if=20clo=E2=80=A6=20(?= =?UTF-8?q?#607)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: validation check. Prompt to remove pe ids and service ids if cloud provider is not bah * fix: validation for principal ids pgd fix * fix: validation for principal ids pgd bug fix and update validation message --- pkg/plan_modifier/cloud_provider.go | 59 +++++++++++++++++++++ pkg/plan_modifier/data_group_custom_diff.go | 53 +++++++++++++----- pkg/provider/resource_analytics_cluster.go | 5 +- pkg/provider/resource_cluster.go | 5 +- 4 files changed, 104 insertions(+), 18 deletions(-) create mode 100644 pkg/plan_modifier/cloud_provider.go diff --git a/pkg/plan_modifier/cloud_provider.go b/pkg/plan_modifier/cloud_provider.go new file mode 100644 index 00000000..b3bad7fa --- /dev/null +++ b/pkg/plan_modifier/cloud_provider.go @@ -0,0 +1,59 @@ +package plan_modifier + +import ( + "context" + "strings" + + "github.com/hashicorp/terraform-plugin-framework/resource/schema/planmodifier" + "github.com/hashicorp/terraform-plugin-go/tftypes" +) + +func CustomClusterCloudProvider() planmodifier.String { + return customCloudProviderModifier{} +} + +type customCloudProviderModifier struct{} + +func (m customCloudProviderModifier) Description(_ context.Context) string { + return "Once set, the value of this attribute in state will not change." +} + +func (m customCloudProviderModifier) MarkdownDescription(_ context.Context) string { + return "Once set, the value of this attribute in state will not change." +} + +func (m customCloudProviderModifier) PlanModifyString(ctx context.Context, req planmodifier.StringRequest, resp *planmodifier.StringResponse) { + cloudProviderConfig := req.ConfigValue.ValueString() + var configObject map[string]tftypes.Value + + err := req.Config.Raw.As(&configObject) + if err != nil { + resp.Diagnostics.AddError("Mapping config object in custom cloud provider modifier error", err.Error()) + return + } + + if !strings.Contains(cloudProviderConfig, "bah") { + peIds, ok := configObject["pe_allowed_principal_ids"] + if ok && !peIds.IsNull() { + resp.Diagnostics.AddError("your cloud account 'pe_allowed_principal_ids' field not allowed error", + "field 'pe_allowed_principal_ids' should only be set if you are using BigAnimal's cloud account e.g. 'bah:aws', please remove 'pe_allowed_principal_ids'") + return + } + + saIds, ok := configObject["service_account_ids"] + if ok && !saIds.IsNull() { + resp.Diagnostics.AddError("your cloud account 'service_account_ids' field not allowed error", + "field 'service_account_ids' should only be set if you are using BigAnimal's cloud account 'bah:gcp', please remove 'service_account_ids'") + return + } + } + + if strings.Contains(cloudProviderConfig, "bah") && !strings.Contains(cloudProviderConfig, "bah:gcp") { + saIds, ok := configObject["service_account_ids"] + if ok && !saIds.IsNull() { + resp.Diagnostics.AddError("your cloud account 'service_account_ids' field not allowed error", + "you are not using cloud provider 'bah:gcp', field 'service_account_ids' should only be set if you are using cloud provider 'bah:gcp', please remove 'service_account_ids'") + return + } + } +} diff --git a/pkg/plan_modifier/data_group_custom_diff.go b/pkg/plan_modifier/data_group_custom_diff.go index 2facbb45..db207562 100644 --- a/pkg/plan_modifier/data_group_custom_diff.go +++ b/pkg/plan_modifier/data_group_custom_diff.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "reflect" + "strings" "github.com/EnterpriseDB/terraform-provider-biganimal/pkg/models/pgd/terraform" "github.com/hashicorp/terraform-plugin-framework/attr" @@ -32,6 +33,44 @@ func (m CustomDataGroupDiffModifier) MarkdownDescription(_ context.Context) stri // PlanModifyList implements the plan modification logic. func (m CustomDataGroupDiffModifier) PlanModifyList(ctx context.Context, req planmodifier.ListRequest, resp *planmodifier.ListResponse) { + var stateDgsObs []terraform.DataGroup + diag := req.StateValue.ElementsAs(ctx, &stateDgsObs, false) + if diag.ErrorsCount() > 0 { + resp.Diagnostics.Append(diag...) + return + } + + var planDgsObs []terraform.DataGroup + diag = resp.PlanValue.ElementsAs(ctx, &planDgsObs, false) + if diag.ErrorsCount() > 0 { + resp.Diagnostics.Append(diag...) + return + } + + // validations + for _, pDg := range planDgsObs { + // validation to remove principal ids and service account ids if cloud provider is not bah + if !strings.Contains(*pDg.Provider.CloudProviderId, "bah") { + if !pDg.PeAllowedPrincipalIds.IsNull() && len(pDg.PeAllowedPrincipalIds.Elements()) > 0 { + resp.Diagnostics.AddError("your cloud account 'pe_allowed_principal_ids' field not allowed error", + fmt.Sprintf("field 'pe_allowed_principal_ids' for region %v should only be set if you are using BigAnimal's cloud account e.g. 'bah:aws', please remove 'pe_allowed_principal_ids'\n", pDg.Region.RegionId)) + return + } + + if !pDg.ServiceAccountIds.IsNull() && len(pDg.ServiceAccountIds.Elements()) > 0 { + resp.Diagnostics.AddError("your cloud account 'service_account_ids' field not allowed error", + fmt.Sprintf("field 'service_account_ids' for region %v should only be set if you are using BigAnimal's cloud account 'bah:gcp', please remove 'service_account_ids'\n", pDg.Region.RegionId)) + return + } + } else if strings.Contains(*pDg.Provider.CloudProviderId, "bah") && !strings.Contains(*pDg.Provider.CloudProviderId, "bah:gcp") { + if !pDg.ServiceAccountIds.IsNull() && len(pDg.ServiceAccountIds.Elements()) > 0 { + resp.Diagnostics.AddError("your cloud account 'service_account_ids' field not allowed error", + fmt.Sprintf("you are not using BigAnimal's cloud account 'bah:gcp' for region %v, field 'service_account_ids' should only be set if you are using BigAnimal's cloud account 'bah:gcp', please remove 'service_account_ids'", pDg.Region.RegionId)) + return + } + } + } + if req.StateValue.IsNull() { // private networking case when doing create var planDgsObs []terraform.DataGroup @@ -87,20 +126,6 @@ func (m CustomDataGroupDiffModifier) PlanModifyList(ctx context.Context, req pla newDgPlan := []terraform.DataGroup{} - var stateDgsObs []terraform.DataGroup - diag := req.StateValue.ElementsAs(ctx, &stateDgsObs, false) - if diag.ErrorsCount() > 0 { - resp.Diagnostics.Append(diag...) - return - } - - var planDgsObs []terraform.DataGroup - diag = resp.PlanValue.ElementsAs(ctx, &planDgsObs, false) - if diag.ErrorsCount() > 0 { - resp.Diagnostics.Append(diag...) - return - } - // Need to sort the plan according to the state this is so the compare and setting unknowns are correct // https://developer.hashicorp.com/terraform/plugin/framework/resources/plan-modification#caveats // sort the order of the plan the same as the state, state is from the read and plan is from the config diff --git a/pkg/provider/resource_analytics_cluster.go b/pkg/provider/resource_analytics_cluster.go index 51ec7a29..5eabc97a 100644 --- a/pkg/provider/resource_analytics_cluster.go +++ b/pkg/provider/resource_analytics_cluster.go @@ -169,8 +169,9 @@ func (r *analyticsClusterResource) Schema(ctx context.Context, req resource.Sche PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, }, "cloud_provider": schema.StringAttribute{ - Description: "Cloud provider. For example, \"aws\" or \"bah:aws\".", - Required: true, + Description: "Cloud provider. For example, \"aws\" or \"bah:aws\".", + Required: true, + PlanModifiers: []planmodifier.String{plan_modifier.CustomClusterCloudProvider()}, }, "pg_type": schema.StringAttribute{ MarkdownDescription: "Postgres type. For example, \"epas\" or \"pgextended\".", diff --git a/pkg/provider/resource_cluster.go b/pkg/provider/resource_cluster.go index 5855ae53..48b1456e 100644 --- a/pkg/provider/resource_cluster.go +++ b/pkg/provider/resource_cluster.go @@ -326,8 +326,9 @@ func (c *clusterResource) Schema(ctx context.Context, req resource.SchemaRequest PlanModifiers: []planmodifier.String{stringplanmodifier.UseStateForUnknown()}, }, "cloud_provider": schema.StringAttribute{ - Description: "Cloud provider. For example, \"aws\", \"azure\", \"gcp\" or \"bah:aws\", \"bah:gcp\".", - Required: true, + Description: "Cloud provider. For example, \"aws\", \"azure\", \"gcp\" or \"bah:aws\", \"bah:gcp\".", + Required: true, + PlanModifiers: []planmodifier.String{plan_modifier.CustomClusterCloudProvider()}, }, "pg_type": schema.StringAttribute{ MarkdownDescription: "Postgres type. For example, \"epas\", \"pgextended\", or \"postgres\".",