Merge pull request #24 from qonto/add-transaction-log-disk-usage

Integrate WAL files to storage analysis

README.md

@@ -63,6 +63,7 @@ It collect key metrics about:
 | rds_quota_total_storage_bytes | `aws_account_id`, `aws_region` | Maximum total storage for all DB instances |
 | rds_read_iops_average | `aws_account_id`, `aws_region`, `dbidentifier` | Average number of disk read I/O operations per second |
 | rds_read_throughput_bytes | `aws_account_id`, `aws_region`, `dbidentifier` | Average number of bytes read from disk per second |
+| rds_transaction_logs_disk_usage_bytes | `aws_account_id`, `aws_region`, `dbidentifier` | Disk space used by transaction logs (only on PostgreSQL) |
 | rds_replica_lag_seconds | `aws_account_id`, `aws_region`, `dbidentifier` | For read replica configurations, the amount of time a read replica DB instance lags behind the source DB instance. Applies to MariaDB, Microsoft SQL Server, MySQL, Oracle, and PostgreSQL read replicas |
 | rds_replication_slot_disk_usage_average | `aws_account_id`, `aws_region`, `dbidentifier` | Disk space used by replication slot files. Applies to PostgreSQL |
 | rds_swap_usage_bytes | `aws_account_id`, `aws_region`, `dbidentifier` | Amount of swap space used on the DB instance. This metric is not available for SQL Server |

internal/app/cloudwatch/rds.go

@@ -40,6 +40,7 @@ type RdsMetrics struct {
 	ReplicaLag                *float64
 	ReplicationSlotDiskUsage  *float64
 	SwapUsage                 *float64
+	TransactionLogsDiskUsage  *float64
 	WriteIOPS                 *float64
 	WriteThroughput           *float64
 }
@@ -76,6 +77,8 @@ func (m *RdsMetrics) Update(field string, value float64) error {
 		m.ReadThroughput = &value
 	case "WriteThroughput":
 		m.WriteThroughput = &value
+	case "TransactionLogsDiskUsage":
+		m.TransactionLogsDiskUsage = &value
 	default:
 		return fmt.Errorf("can't process '%s' metrics: %w", field, errUnknownMetric)
 	}
@@ -84,8 +87,8 @@ func (m *RdsMetrics) Update(field string, value float64) error {
 }
 
 // getCloudWatchMetricsName returns names of Cloudwatch metrics to collect
-func getCloudWatchMetricsName() [15]string {
-	return [15]string{
+func getCloudWatchMetricsName() [16]string {
+	return [16]string{
 		"CPUUtilization",
 		"DBLoad",
 		"DBLoadCPU",
@@ -99,6 +102,7 @@ func getCloudWatchMetricsName() [15]string {
 		"ReplicaLag",
 		"ReplicationSlotDiskUsage",
 		"SwapUsage",
+		"TransactionLogsDiskUsage",
 		"WriteIOPS",
 		"WriteThroughput",
 	}

internal/app/cloudwatch/rds_test.go

@@ -26,6 +26,7 @@ var db1ExpecteRdsMetrics = cloudwatch.RdsMetrics{
 	ReplicaLag:                aws.Float64(42),
 	ReplicationSlotDiskUsage:  aws.Float64(100),
 	SwapUsage:                 aws.Float64(10),
+	TransactionLogsDiskUsage:  aws.Float64(24),
 	WriteIOPS:                 aws.Float64(11),
 	WriteThroughput:           aws.Float64(12),
 }
@@ -44,6 +45,7 @@ var db2ExpecteRdsMetrics = cloudwatch.RdsMetrics{
 	ReplicaLag:                aws.Float64(42),
 	ReplicationSlotDiskUsage:  aws.Float64(100),
 	SwapUsage:                 aws.Float64(10),
+	TransactionLogsDiskUsage:  aws.Float64(24),
 	WriteIOPS:                 aws.Float64(11),
 	WriteThroughput:           aws.Float64(12),
 }
@@ -116,6 +118,11 @@ func generateMockedMetricsForInstance(id int, m cloudwatch.RdsMetrics) []aws_clo
 			Label:  aws.String("SwapUsage"),
 			Values: []float64{*m.SwapUsage},
 		},
+		{
+			Id:     aws.String(fmt.Sprintf("transactionlogsdiskusage_%d", id)),
+			Label:  aws.String("TransactionLogsDiskUsage"),
+			Values: []float64{*m.TransactionLogsDiskUsage},
+		},
 		{
 			Id:     aws.String(fmt.Sprintf("writeiops_%d", id)),
 			Label:  aws.String("WriteIOPS"),
@@ -173,6 +180,7 @@ func TestGetDBInstanceTypeInformation(t *testing.T) {
 		assert.Equal(t, value.ReplicaLag, result.Instances[id].ReplicaLag, "ReplicaLag mismatch")
 		assert.Equal(t, value.ReplicationSlotDiskUsage, result.Instances[id].ReplicationSlotDiskUsage, "ReplicationSlotDiskUsage mismatch")
 		assert.Equal(t, value.SwapUsage, result.Instances[id].SwapUsage, "SwapUsage mismatch")
+		assert.Equal(t, value.TransactionLogsDiskUsage, result.Instances[id].TransactionLogsDiskUsage, "TransactionLogsDiskUsage mismatch")
 		assert.Equal(t, value.WriteIOPS, result.Instances[id].WriteIOPS, "WriteIOPS mismatch")
 		assert.Equal(t, value.WriteThroughput, result.Instances[id].WriteThroughput, "WriteThroughput mismatch")
 	}

internal/app/exporter/exporter.go

@@ -97,6 +97,7 @@ type rdsCollector struct {
 	usageDBInstances            *prometheus.Desc
 	usageManualSnapshots        *prometheus.Desc
 	exporterBuildInformation    *prometheus.Desc
+	transactionLogsDiskUsage    *prometheus.Desc
 }
 
 func NewCollector(logger slog.Logger, collectorConfiguration Configuration, awsAccountID string, awsRegion string, rdsClient rdsClient, ec2Client EC2Client, cloudWatchClient cloudWatchClient, servicequotasClient servicequotasClient) *rdsCollector {
@@ -235,6 +236,10 @@ func NewCollector(logger slog.Logger, collectorConfiguration Configuration, awsA
 			"Number of active sessions where the wait event type is not CPU",
 			[]string{"aws_account_id", "aws_region", "dbidentifier"}, nil,
 		),
+		transactionLogsDiskUsage: prometheus.NewDesc("rds_transaction_logs_disk_usage_bytes",
+			"Disk space used by transaction logs (only on PostgreSQL)",
+			[]string{"aws_account_id", "aws_region", "dbidentifier"}, nil,
+		),
 		quotaDBInstances: prometheus.NewDesc("rds_quota_max_dbinstances_average",
 			"Maximum number of RDS instances allowed in the AWS account",
 			[]string{"aws_account_id", "aws_region"}, nil,
@@ -471,6 +476,10 @@ func (c *rdsCollector) Collect(ch chan<- prometheus.Metric) {
 			ch <- prometheus.MustNewConstMetric(c.writeThroughput, prometheus.GaugeValue, *instance.WriteThroughput, c.awsAccountID, c.awsRegion, dbidentifier)
 		}
 
+		if instance.TransactionLogsDiskUsage != nil {
+			ch <- prometheus.MustNewConstMetric(c.transactionLogsDiskUsage, prometheus.GaugeValue, *instance.TransactionLogsDiskUsage, c.awsAccountID, c.awsRegion, dbidentifier)
+		}
+
 		if instance.DBLoad != nil {
 			ch <- prometheus.MustNewConstMetric(c.DBLoad, prometheus.GaugeValue, *instance.DBLoad, c.awsAccountID, c.awsRegion, dbidentifier)
 		}