Cache costexplorer API

providers/aws/aws.go

@@ -2,20 +2,19 @@ package aws
 
 import (
 	"context"
+	"encoding/json"
 	"strings"
 	"time"
 
-	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/costexplorer"
-	"github.com/aws/aws-sdk-go-v2/service/costexplorer/types"
 	log "github.com/sirupsen/logrus"
 	"github.com/tailwarden/komiser/models"
 	"github.com/tailwarden/komiser/providers"
 	"github.com/tailwarden/komiser/providers/aws/apigateway"
 	"github.com/tailwarden/komiser/providers/aws/cloudfront"
 	"github.com/tailwarden/komiser/providers/aws/cloudwatch"
-	"github.com/tailwarden/komiser/providers/aws/codecommit"
 	"github.com/tailwarden/komiser/providers/aws/codebuild"
+	"github.com/tailwarden/komiser/providers/aws/codecommit"
 	"github.com/tailwarden/komiser/providers/aws/codedeploy"
 	"github.com/tailwarden/komiser/providers/aws/dynamodb"
 	"github.com/tailwarden/komiser/providers/aws/ec2"
@@ -118,41 +117,27 @@ func FetchResources(ctx context.Context, client providers.ProviderClient, region
 		listOfSupportedRegions = regions
 	}
 
-	costexplorerClient := costexplorer.NewFromConfig(*client.AWSClient)
-	costexplorerOutputList := []*costexplorer.GetCostAndUsageOutput{}
-	var nextPageToken *string
-	for {
-		costexplorerOutput, err := costexplorerClient.GetCostAndUsage(ctx, &costexplorer.GetCostAndUsageInput{
-			Granularity: "DAILY",
-			Metrics:     []string{"UnblendedCost"},
-			TimePeriod: &types.DateInterval{
-				Start: aws.String(utils.BeginningOfMonth(time.Now()).Format("2006-01-02")),
-				End:   aws.String(time.Now().Format("2006-01-02")),
-			},
-			GroupBy: []types.GroupDefinition{
-				{
-					Key:  aws.String("SERVICE"),
-					Type: "DIMENSION",
-				},
-				{
-					Key:  aws.String("REGION"),
-					Type: "DIMENSION",
-				},
-			},
-			NextPageToken: nextPageToken,
-		})
+	var costexplorerOutputList []*costexplorer.GetCostAndUsageOutput
+	if jsonData, err := readCostExplorerCache(); err == nil {
+		err := json.Unmarshal(jsonData, &costexplorerOutputList)
 		if err != nil {
-			log.Warn("Couldn't fetch cost and usage data:", err)
-			break
+			log.Warn("Failed to unmarshal cached cost explorer data:", err)
 		}
-
-		costexplorerOutputList = append(costexplorerOutputList, costexplorerOutput)
-
-		if aws.ToString(costexplorerOutput.NextPageToken) == "" {
-			break
+	} else {
+		costexplorerOutputList, err = getCostexplorerOutput(
+			ctx, client, utils.BeginningMonthsAgo(time.Now(), 6).Format("2006-01-02"), utils.EndingOfLastMonth(time.Now()).Format("2006-01-02"),
+		)
+		if err != nil {
+			log.Warn("Failed to get cost explorer output:", err)
+		}
+		if err := writeCostExplorerCache(costexplorerOutputList); err != nil {
+			log.Warn("Failed to write cost explorer cache:", err)
 		}
+	}
 
-		nextPageToken = costexplorerOutput.NextPageToken
+	costexplorerOutputList, err := getCostexplorerOutput(ctx, client, utils.BeginningOfMonth(time.Now()).Format("2006-01-02"), time.Now().Format("2006-01-02"))
+	if err != nil {
+		log.Warn("Failed to get cost explorer output:", err)
 	}
 	ctxWithCostexplorerOutput := context.WithValue(ctx, awsUtils.CostexplorerKey, costexplorerOutputList)
 	for _, region := range listOfSupportedRegions {

providers/aws/costexplorer.go

@@ -0,0 +1,77 @@
+package aws
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/costexplorer"
+	"github.com/aws/aws-sdk-go-v2/service/costexplorer/types"
+	log "github.com/sirupsen/logrus"
+	"github.com/tailwarden/komiser/providers"
+)
+
+const costExplorerCacheFile = "cost_explorer_cache.json"
+
+// readCostExplorerCache reads cost explorer cache data from the file.
+func readCostExplorerCache() ([]byte, error) {
+	file, err := os.ReadFile(costExplorerCacheFile)
+	if err != nil {
+		return nil, err
+	}
+	return file, nil
+}
+
+// writeCostExplorerCache writes cost explorer cache data to the file.
+func writeCostExplorerCache(data interface{}) error {
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+	err = os.WriteFile(costExplorerCacheFile, jsonData, 0644)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getCostexplorerOutput(ctx context.Context, client providers.ProviderClient, start, end string) ([]*costexplorer.GetCostAndUsageOutput, error) {
+	costexplorerOutputList := []*costexplorer.GetCostAndUsageOutput{}
+	costexplorerClient := costexplorer.NewFromConfig(*client.AWSClient)
+	var nextPageToken *string
+	for {
+		costexplorerOutput, err := costexplorerClient.GetCostAndUsage(ctx, &costexplorer.GetCostAndUsageInput{
+			Granularity: "DAILY",
+			Metrics:     []string{"UnblendedCost"},
+			TimePeriod: &types.DateInterval{
+				Start: aws.String(start),
+				End:   aws.String(end),
+			},
+			GroupBy: []types.GroupDefinition{
+				{
+					Key:  aws.String("SERVICE"),
+					Type: "DIMENSION",
+				},
+				{
+					Key:  aws.String("REGION"),
+					Type: "DIMENSION",
+				},
+			},
+			NextPageToken: nextPageToken,
+		})
+		if err != nil {
+			log.Warn("Couldn't fetch cost and usage data:", err)
+			return nil, err
+		}
+
+		costexplorerOutputList = append(costexplorerOutputList, costexplorerOutput)
+
+		if aws.ToString(costexplorerOutput.NextPageToken) == "" {
+			break
+		}
+
+		nextPageToken = costexplorerOutput.NextPageToken
+	}
+	return costexplorerOutputList, nil
+}

providers/aws/utils/utils.go

@@ -25,7 +25,7 @@ func GetCostAndUsage(ctx context.Context, region string, svcName string) (float6
 	for _, costexplorerOutput := range costexplorerOutputList {
 		for _, group := range costexplorerOutput.ResultsByTime {
 			for _, v := range group.Groups {
-				if v.Keys[0] == svcName && v.Keys[1] == region {
+				if v.Keys[0] == svcName {
 					amt, err := strconv.ParseFloat(*v.Metrics["UnblendedCost"].Amount, 64)
 					if err != nil {
 						return 0, err

utils/datecalc.go

@@ -17,3 +17,19 @@ func EndingOfMonth(date time.Time) time.Time {
 	}
 	return endingOfMonth
 }
+
+func BeginningMonthsAgo(date time.Time, months int) time.Time {
+	beginningOfTargetMonth, err := time.Parse("2006-01-02", date.AddDate(0, -months, -date.Day()+1).Format("2006-01-02"))
+	if err != nil {
+		return time.Now()
+	}
+	return beginningOfTargetMonth
+}
+
+func EndingOfLastMonth(date time.Time) time.Time {
+	endingOfMonth, err := time.Parse("2006-01-02", date.AddDate(0, 0, -date.Day()+1).Format("2006-01-02"))
+	if err != nil {
+		return time.Now()
+	}
+	return endingOfMonth
+}