diff --git a/Makefile b/Makefile index 5ed141c..a312dd5 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ RESET := $(shell tput -T linux sgr0) TITLE := $(BOLD)$(PURPLE) SUCCESS := $(BOLD)$(GREEN) # the quality gate lower threshold for unit test total % coverage (by function statements) -COVERAGE_THRESHOLD := 50 +COVERAGE_THRESHOLD := 45 CLUSTER_NAME=anchore-k8s-inventory-testing diff --git a/README.md b/README.md index 3942ef0..b6543a6 100644 --- a/README.md +++ b/README.md @@ -401,6 +401,17 @@ missing-tag-policy: ignore-not-running: true ``` +### Batching Inventory Report Posting + +Set upper limits for the content that can be contained in a single inventory report POST +to Anchore Enterprise. If the inventory data is greater than the limit then the inventory +report will be broken into smaller requests up to the limit size specified. + +```yaml +inventory-report-limits: + namespaces: 0 # default of 0 means no limit +``` + ### Anchore API configuration Use this section to configure the Anchore Enterprise API endpoint diff --git a/anchore-k8s-inventory.yaml b/anchore-k8s-inventory.yaml index 1b22f3a..e8becf8 100644 --- a/anchore-k8s-inventory.yaml +++ b/anchore-k8s-inventory.yaml @@ -117,6 +117,11 @@ ignore-not-running: true # Only respected if mode is periodic polling-interval-seconds: 300 +# Batch Request configuration +inventory-report-limits: + namespaces: 0 # default of 0 means no limit per report + +# Anchore API Configuration anchore: # url: $ANCHORE_K8S_INVENTORY_ANCHORE_URL # user: $ANCHORE_K8S_INVENTORY_ANCHORE_USER diff --git a/cmd/root.go b/cmd/root.go index 9b10086..aaa151f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -58,20 +58,23 @@ var rootCmd = &cobra.Command{ os.Exit(1) } anErrorOccurred := false - for account, report := range reports { - err = pkg.HandleReport(report, appConfig, account) - if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) { - // Retry with default account - retryAccount := appConfig.AnchoreDetails.Account - if appConfig.AccountRouteByNamespaceLabel.DefaultAccount != "" { - retryAccount = appConfig.AccountRouteByNamespaceLabel.DefaultAccount + for account, reportsForAccount := range reports { + for count, report := range reportsForAccount { + log.Infof("Sending Inventory Report to Anchore Account %s, %d of %d", account, count+1, len(reportsForAccount)) + err = pkg.HandleReport(report, appConfig, account) + if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) { + // Retry with default account + retryAccount := appConfig.AnchoreDetails.Account + if appConfig.AccountRouteByNamespaceLabel.DefaultAccount != "" { + retryAccount = appConfig.AccountRouteByNamespaceLabel.DefaultAccount + } + log.Warnf("Error sending to Anchore Account %s, sending to default account", account) + err = pkg.HandleReport(report, appConfig, retryAccount) + } + if err != nil { + log.Errorf("Failed to handle Image Results: %+v", err) + anErrorOccurred = true } - log.Warnf("Error sending to Anchore Account %s, sending to default account", account) - err = pkg.HandleReport(report, appConfig, retryAccount) - } - if err != nil { - log.Errorf("Failed to handle Image Results: %+v", err) - anErrorOccurred = true } } if anErrorOccurred { diff --git a/internal/config/config.go b/internal/config/config.go index 2faa6a0..9c372f8 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,11 +51,12 @@ type Application struct { MissingRegistryOverride string `mapstructure:"missing-registry-override"` MissingTagPolicy MissingTagConf `mapstructure:"missing-tag-policy"` RunMode mode.Mode - Mode string `mapstructure:"mode"` - IgnoreNotRunning bool `mapstructure:"ignore-not-running"` - PollingIntervalSeconds int `mapstructure:"polling-interval-seconds"` - AnchoreDetails AnchoreInfo `mapstructure:"anchore"` - VerboseInventoryReports bool `mapstructure:"verbose-inventory-reports"` + Mode string `mapstructure:"mode"` + IgnoreNotRunning bool `mapstructure:"ignore-not-running"` + PollingIntervalSeconds int `mapstructure:"polling-interval-seconds"` + InventoryReportLimits InventoryReportLimits `mapstructure:"inventory-report-limits"` + AnchoreDetails AnchoreInfo `mapstructure:"anchore"` + VerboseInventoryReports bool `mapstructure:"verbose-inventory-reports"` } // MissingTagConf details the policy for handling missing tags when reporting images @@ -92,6 +93,11 @@ type KubernetesAPI struct { WorkerPoolSize int `mapstructure:"worker-pool-size"` } +// Details upper limits for the inventory report contents before splitting into batches +type InventoryReportLimits struct { + Namespaces int `mapstructure:"namespaces"` +} + // Information for posting in-use image details to Anchore (or any URL for that matter) type AnchoreInfo struct { URL string `mapstructure:"url"` diff --git a/internal/config/test-fixtures/snapshot/TestDefaultConfigString.golden b/internal/config/test-fixtures/snapshot/TestDefaultConfigString.golden index 8aeefbd..4174aa6 100644 --- a/internal/config/test-fixtures/snapshot/TestDefaultConfigString.golden +++ b/internal/config/test-fixtures/snapshot/TestDefaultConfigString.golden @@ -44,6 +44,8 @@ runmode: 0 mode: adhoc ignorenotrunning: true pollingintervalseconds: 300 +inventoryreportlimits: + namespaces: 0 anchoredetails: url: "" user: "" diff --git a/internal/config/test-fixtures/snapshot/TestEmptyConfigString.golden b/internal/config/test-fixtures/snapshot/TestEmptyConfigString.golden index a4f4174..5fa79f4 100644 --- a/internal/config/test-fixtures/snapshot/TestEmptyConfigString.golden +++ b/internal/config/test-fixtures/snapshot/TestEmptyConfigString.golden @@ -44,6 +44,8 @@ runmode: 0 mode: "" ignorenotrunning: false pollingintervalseconds: 0 +inventoryreportlimits: + namespaces: 0 anchoredetails: url: "" user: "" diff --git a/internal/config/test-fixtures/snapshot/TestSensitiveConfigString.golden b/internal/config/test-fixtures/snapshot/TestSensitiveConfigString.golden index 2f4097c..35dc105 100644 --- a/internal/config/test-fixtures/snapshot/TestSensitiveConfigString.golden +++ b/internal/config/test-fixtures/snapshot/TestSensitiveConfigString.golden @@ -44,6 +44,8 @@ runmode: 0 mode: adhoc ignorenotrunning: true pollingintervalseconds: 300 +inventoryreportlimits: + namespaces: 0 anchoredetails: url: "" user: "" diff --git a/pkg/lib.go b/pkg/lib.go index d61a714..dbb4617 100644 --- a/pkg/lib.go +++ b/pkg/lib.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "os" "regexp" "time" @@ -35,7 +36,7 @@ type channels struct { stopper chan struct{} } -type AccountRoutedReports map[string]inventory.Report +type AccountRoutedReports map[string][]inventory.Report func reportToStdout(report inventory.Report) error { enc := json.NewEncoder(os.Stdout) @@ -99,19 +100,22 @@ func PeriodicallyGetInventoryReport(cfg *config.Application) { if err != nil { log.Errorf("Failed to get Inventory Report: %w", err) } else { - for account, report := range reports { - err := HandleReport(report, cfg, account) - if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) { - // Retry with default account - retryAccount := cfg.AnchoreDetails.Account - if cfg.AccountRouteByNamespaceLabel.DefaultAccount != "" { - retryAccount = cfg.AccountRouteByNamespaceLabel.DefaultAccount + for account, reportsForAccount := range reports { + for count, report := range reportsForAccount { + log.Infof("Sending Inventory Report to Anchore Account %s, %d of %d", account, count+1, len(reportsForAccount)) + err := HandleReport(report, cfg, account) + if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) { + // Retry with default account + retryAccount := cfg.AnchoreDetails.Account + if cfg.AccountRouteByNamespaceLabel.DefaultAccount != "" { + retryAccount = cfg.AccountRouteByNamespaceLabel.DefaultAccount + } + log.Warnf("Error sending to Anchore Account %s, sending to default account", account) + err = HandleReport(report, cfg, retryAccount) + } + if err != nil { + log.Errorf("Failed to handle Inventory Report: %w", err) } - log.Warnf("Error sending to Anchore Account %s, sending to default account", account) - err = HandleReport(report, cfg, retryAccount) - } - if err != nil { - log.Errorf("Failed to handle Inventory Report: %w", err) } } } @@ -322,19 +326,43 @@ func GetAccountRoutedNamespaces(defaultAccount string, namespaces []inventory.Na return accountRoutesForAllNamespaces } +func GetNamespacesBatches(namespaces []inventory.Namespace, batchSize int) [][]inventory.Namespace { + batches := make([][]inventory.Namespace, 0) + if batchSize <= 0 { + return append(batches, namespaces) + } + for i := 0; i < len(namespaces); i += batchSize { + end := i + batchSize + if end > len(namespaces) { + end = len(namespaces) + } + batches = append(batches, namespaces[i:end]) + } + return batches +} + func GetInventoryReports(cfg *config.Application) (AccountRoutedReports, error) { log.Info("Starting image inventory collection") reports := AccountRoutedReports{} + batchSize := cfg.InventoryReportLimits.Namespaces + if batchSize > 0 { + log.Infof("Batching namespaces into groups of %d", batchSize) + } namespaces, _ := GetAllNamespaces(cfg) if len(cfg.AccountRoutes) == 0 && cfg.AccountRouteByNamespaceLabel.LabelKey == "" { - allNamespacesReport, err := GetInventoryReportForNamespaces(cfg, namespaces) - if err != nil { - return AccountRoutedReports{}, err + for batchCount, batch := range GetNamespacesBatches(namespaces, batchSize) { + if batchSize > 0 { + log.Infof("Collecting batch %d of %d for account %s", batchCount+1, int(math.Ceil(float64(len(namespaces))/float64(batchSize))), cfg.AnchoreDetails.Account) + } + batchNamespacesReport, err := GetInventoryReportForNamespaces(cfg, batch) + if err != nil { + return AccountRoutedReports{}, err + } + reports[cfg.AnchoreDetails.Account] = append(reports[cfg.AnchoreDetails.Account], batchNamespacesReport) } - reports[cfg.AnchoreDetails.Account] = allNamespacesReport } else { accountRoutesForAllNamespaces := GetAccountRoutedNamespaces(cfg.AnchoreDetails.Account, namespaces, cfg.AccountRoutes, cfg.AccountRouteByNamespaceLabel) @@ -348,11 +376,16 @@ func GetInventoryReports(cfg *config.Application) (AccountRoutedReports, error) // Get inventory reports for each account for account, namespaces := range accountRoutesForAllNamespaces { - accountReport, err := GetInventoryReportForNamespaces(cfg, namespaces) - if err != nil { - return AccountRoutedReports{}, err + for batchCount, batch := range GetNamespacesBatches(namespaces, batchSize) { + if batchSize > 0 { + log.Infof("Collecting inventory batch %d of %d for account %s", batchCount+1, int(math.Ceil(float64(len(namespaces))/float64(batchSize))), account) + } + accountReport, err := GetInventoryReportForNamespaces(cfg, batch) + if err != nil { + return AccountRoutedReports{}, err + } + reports[account] = append(reports[account], accountReport) } - reports[account] = accountReport } } diff --git a/pkg/lib_test.go b/pkg/lib_test.go index b41dd86..3c18686 100644 --- a/pkg/lib_test.go +++ b/pkg/lib_test.go @@ -5,7 +5,6 @@ import ( "github.com/anchore/k8s-inventory/internal/config" "github.com/anchore/k8s-inventory/pkg/inventory" - "github.com/stretchr/testify/assert" ) @@ -223,3 +222,73 @@ func TestGetAccountRoutedNamespaces(t *testing.T) { }) } } + +func TestGetNamespacesBatches(t *testing.T) { + type args struct { + namespaces []inventory.Namespace + batchSize int + } + tests := []struct { + name string + args args + want [][]inventory.Namespace + }{ + { + name: "empty namespaces", + args: args{ + namespaces: []inventory.Namespace{}, + batchSize: 10, + }, + want: [][]inventory.Namespace{}, + }, + { + name: "single batch", + args: args{ + namespaces: TestNamespaces, + batchSize: 10, + }, + want: [][]inventory.Namespace{ + TestNamespaces, + }, + }, + { + name: "multiple batches", + args: args{ + namespaces: TestNamespaces, + batchSize: 2, + }, + want: [][]inventory.Namespace{ + {TestNamespace1, TestNamespace2}, + {TestNamespace3, TestNamespace4}, + }, + }, + { + name: "multiple batches with remainder", + args: args{ + namespaces: append(TestNamespaces, TestNamespace5), + batchSize: 2, + }, + want: [][]inventory.Namespace{ + {TestNamespace1, TestNamespace2}, + {TestNamespace3, TestNamespace4}, + {TestNamespace5}, + }, + }, + { + name: "no batches configured (batch size 0)", + args: args{ + namespaces: TestNamespaces, + batchSize: 0, + }, + want: [][]inventory.Namespace{ + TestNamespaces, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := GetNamespacesBatches(tt.args.namespaces, tt.args.batchSize) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/test/integration/get_images_test.go b/test/integration/get_images_test.go index b1fbbc5..7edc7cf 100644 --- a/test/integration/get_images_test.go +++ b/test/integration/get_images_test.go @@ -14,6 +14,8 @@ const ( ) // Assumes that the hello-world helm chart in ./fixtures was installed (basic nginx container) +// +//nolint:gocognit func TestGetImageResults(t *testing.T) { cmd.InitAppConfig() reports, err := pkg.GetInventoryReports(cmd.GetAppConfig()) @@ -21,37 +23,39 @@ func TestGetImageResults(t *testing.T) { t.Fatalf("failed to get image results: %v", err) } - for _, report := range reports { - if report.ServerVersionMetadata == nil { - t.Errorf("Failed to include Server Version Metadata in report") - } - - if report.Timestamp == "" { - t.Errorf("Failed to include Timestamp in report") - } + for _, reportsForAccount := range reports { + for _, report := range reportsForAccount { + if report.ServerVersionMetadata == nil { + t.Errorf("Failed to include Server Version Metadata in report") + } - foundIntegrationTestNamespace := false - for _, item := range report.Namespaces { - if item.Name != IntegrationTestNamespace { - continue + if report.Timestamp == "" { + t.Errorf("Failed to include Timestamp in report") } - foundIntegrationTestNamespace = true - foundIntegrationTestImage := false - for _, image := range report.Containers { - if !strings.Contains(image.ImageTag, IntegrationTestImageTag) { + + foundIntegrationTestNamespace := false + for _, item := range report.Namespaces { + if item.Name != IntegrationTestNamespace { continue } - foundIntegrationTestImage = true - if image.ImageDigest == "" { - t.Logf("Image Found, but no digest located: %v", image) + foundIntegrationTestNamespace = true + foundIntegrationTestImage := false + for _, image := range report.Containers { + if !strings.Contains(image.ImageTag, IntegrationTestImageTag) { + continue + } + foundIntegrationTestImage = true + if image.ImageDigest == "" { + t.Logf("Image Found, but no digest located: %v", image) + } + } + if !foundIntegrationTestImage { + t.Errorf("failed to locate integration test image") } } - if !foundIntegrationTestImage { - t.Errorf("failed to locate integration test image") + if !foundIntegrationTestNamespace { + t.Errorf("failed to locate integration test namespace") } } - if !foundIntegrationTestNamespace { - t.Errorf("failed to locate integration test namespace") - } } }