Skip to content

Commit

Permalink
Merge pull request #223 from anchore/batch-posting
Browse files Browse the repository at this point in the history
feat: add batch sending of inventory reports based on limits
  • Loading branch information
bradleyjones authored May 24, 2024
2 parents 6d3ea4b + 600fd01 commit 7655b22
Show file tree
Hide file tree
Showing 11 changed files with 202 additions and 65 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ RESET := $(shell tput -T linux sgr0)
TITLE := $(BOLD)$(PURPLE)
SUCCESS := $(BOLD)$(GREEN)
# the quality gate lower threshold for unit test total % coverage (by function statements)
COVERAGE_THRESHOLD := 50
COVERAGE_THRESHOLD := 45

CLUSTER_NAME=anchore-k8s-inventory-testing

Expand Down
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,17 @@ missing-tag-policy:
ignore-not-running: true
```

### Batching Inventory Report Posting

Set upper limits for the content that can be contained in a single inventory report POST
to Anchore Enterprise. If the inventory data is greater than the limit then the inventory
report will be broken into smaller requests up to the limit size specified.

```yaml
inventory-report-limits:
namespaces: 0 # default of 0 means no limit
```

### Anchore API configuration

Use this section to configure the Anchore Enterprise API endpoint
Expand Down
5 changes: 5 additions & 0 deletions anchore-k8s-inventory.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ ignore-not-running: true
# Only respected if mode is periodic
polling-interval-seconds: 300

# Batch Request configuration
inventory-report-limits:
namespaces: 0 # default of 0 means no limit per report

# Anchore API Configuration
anchore:
# url: $ANCHORE_K8S_INVENTORY_ANCHORE_URL
# user: $ANCHORE_K8S_INVENTORY_ANCHORE_USER
Expand Down
29 changes: 16 additions & 13 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,20 +58,23 @@ var rootCmd = &cobra.Command{
os.Exit(1)
}
anErrorOccurred := false
for account, report := range reports {
err = pkg.HandleReport(report, appConfig, account)
if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) {
// Retry with default account
retryAccount := appConfig.AnchoreDetails.Account
if appConfig.AccountRouteByNamespaceLabel.DefaultAccount != "" {
retryAccount = appConfig.AccountRouteByNamespaceLabel.DefaultAccount
for account, reportsForAccount := range reports {
for count, report := range reportsForAccount {
log.Infof("Sending Inventory Report to Anchore Account %s, %d of %d", account, count+1, len(reportsForAccount))
err = pkg.HandleReport(report, appConfig, account)
if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) {
// Retry with default account
retryAccount := appConfig.AnchoreDetails.Account
if appConfig.AccountRouteByNamespaceLabel.DefaultAccount != "" {
retryAccount = appConfig.AccountRouteByNamespaceLabel.DefaultAccount
}
log.Warnf("Error sending to Anchore Account %s, sending to default account", account)
err = pkg.HandleReport(report, appConfig, retryAccount)
}
if err != nil {
log.Errorf("Failed to handle Image Results: %+v", err)
anErrorOccurred = true
}
log.Warnf("Error sending to Anchore Account %s, sending to default account", account)
err = pkg.HandleReport(report, appConfig, retryAccount)
}
if err != nil {
log.Errorf("Failed to handle Image Results: %+v", err)
anErrorOccurred = true
}
}
if anErrorOccurred {
Expand Down
16 changes: 11 additions & 5 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,12 @@ type Application struct {
MissingRegistryOverride string `mapstructure:"missing-registry-override"`
MissingTagPolicy MissingTagConf `mapstructure:"missing-tag-policy"`
RunMode mode.Mode
Mode string `mapstructure:"mode"`
IgnoreNotRunning bool `mapstructure:"ignore-not-running"`
PollingIntervalSeconds int `mapstructure:"polling-interval-seconds"`
AnchoreDetails AnchoreInfo `mapstructure:"anchore"`
VerboseInventoryReports bool `mapstructure:"verbose-inventory-reports"`
Mode string `mapstructure:"mode"`
IgnoreNotRunning bool `mapstructure:"ignore-not-running"`
PollingIntervalSeconds int `mapstructure:"polling-interval-seconds"`
InventoryReportLimits InventoryReportLimits `mapstructure:"inventory-report-limits"`
AnchoreDetails AnchoreInfo `mapstructure:"anchore"`
VerboseInventoryReports bool `mapstructure:"verbose-inventory-reports"`
}

// MissingTagConf details the policy for handling missing tags when reporting images
Expand Down Expand Up @@ -92,6 +93,11 @@ type KubernetesAPI struct {
WorkerPoolSize int `mapstructure:"worker-pool-size"`
}

// Details upper limits for the inventory report contents before splitting into batches
type InventoryReportLimits struct {
Namespaces int `mapstructure:"namespaces"`
}

// Information for posting in-use image details to Anchore (or any URL for that matter)
type AnchoreInfo struct {
URL string `mapstructure:"url"`
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ runmode: 0
mode: adhoc
ignorenotrunning: true
pollingintervalseconds: 300
inventoryreportlimits:
namespaces: 0
anchoredetails:
url: ""
user: ""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ runmode: 0
mode: ""
ignorenotrunning: false
pollingintervalseconds: 0
inventoryreportlimits:
namespaces: 0
anchoredetails:
url: ""
user: ""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ runmode: 0
mode: adhoc
ignorenotrunning: true
pollingintervalseconds: 300
inventoryreportlimits:
namespaces: 0
anchoredetails:
url: ""
user: ""
Expand Down
75 changes: 54 additions & 21 deletions pkg/lib.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"encoding/json"
"errors"
"fmt"
"math"
"os"
"regexp"
"time"
Expand Down Expand Up @@ -35,7 +36,7 @@ type channels struct {
stopper chan struct{}
}

type AccountRoutedReports map[string]inventory.Report
type AccountRoutedReports map[string][]inventory.Report

func reportToStdout(report inventory.Report) error {
enc := json.NewEncoder(os.Stdout)
Expand Down Expand Up @@ -99,19 +100,22 @@ func PeriodicallyGetInventoryReport(cfg *config.Application) {
if err != nil {
log.Errorf("Failed to get Inventory Report: %w", err)
} else {
for account, report := range reports {
err := HandleReport(report, cfg, account)
if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) {
// Retry with default account
retryAccount := cfg.AnchoreDetails.Account
if cfg.AccountRouteByNamespaceLabel.DefaultAccount != "" {
retryAccount = cfg.AccountRouteByNamespaceLabel.DefaultAccount
for account, reportsForAccount := range reports {
for count, report := range reportsForAccount {
log.Infof("Sending Inventory Report to Anchore Account %s, %d of %d", account, count+1, len(reportsForAccount))
err := HandleReport(report, cfg, account)
if errors.Is(err, reporter.ErrAnchoreAccountDoesNotExist) {
// Retry with default account
retryAccount := cfg.AnchoreDetails.Account
if cfg.AccountRouteByNamespaceLabel.DefaultAccount != "" {
retryAccount = cfg.AccountRouteByNamespaceLabel.DefaultAccount
}
log.Warnf("Error sending to Anchore Account %s, sending to default account", account)
err = HandleReport(report, cfg, retryAccount)
}
if err != nil {
log.Errorf("Failed to handle Inventory Report: %w", err)
}
log.Warnf("Error sending to Anchore Account %s, sending to default account", account)
err = HandleReport(report, cfg, retryAccount)
}
if err != nil {
log.Errorf("Failed to handle Inventory Report: %w", err)
}
}
}
Expand Down Expand Up @@ -322,19 +326,43 @@ func GetAccountRoutedNamespaces(defaultAccount string, namespaces []inventory.Na
return accountRoutesForAllNamespaces
}

func GetNamespacesBatches(namespaces []inventory.Namespace, batchSize int) [][]inventory.Namespace {
batches := make([][]inventory.Namespace, 0)
if batchSize <= 0 {
return append(batches, namespaces)
}
for i := 0; i < len(namespaces); i += batchSize {
end := i + batchSize
if end > len(namespaces) {
end = len(namespaces)
}
batches = append(batches, namespaces[i:end])
}
return batches
}

func GetInventoryReports(cfg *config.Application) (AccountRoutedReports, error) {
log.Info("Starting image inventory collection")

reports := AccountRoutedReports{}
batchSize := cfg.InventoryReportLimits.Namespaces
if batchSize > 0 {
log.Infof("Batching namespaces into groups of %d", batchSize)
}

namespaces, _ := GetAllNamespaces(cfg)

if len(cfg.AccountRoutes) == 0 && cfg.AccountRouteByNamespaceLabel.LabelKey == "" {
allNamespacesReport, err := GetInventoryReportForNamespaces(cfg, namespaces)
if err != nil {
return AccountRoutedReports{}, err
for batchCount, batch := range GetNamespacesBatches(namespaces, batchSize) {
if batchSize > 0 {
log.Infof("Collecting batch %d of %d for account %s", batchCount+1, int(math.Ceil(float64(len(namespaces))/float64(batchSize))), cfg.AnchoreDetails.Account)
}
batchNamespacesReport, err := GetInventoryReportForNamespaces(cfg, batch)
if err != nil {
return AccountRoutedReports{}, err
}
reports[cfg.AnchoreDetails.Account] = append(reports[cfg.AnchoreDetails.Account], batchNamespacesReport)
}
reports[cfg.AnchoreDetails.Account] = allNamespacesReport
} else {
accountRoutesForAllNamespaces := GetAccountRoutedNamespaces(cfg.AnchoreDetails.Account, namespaces, cfg.AccountRoutes, cfg.AccountRouteByNamespaceLabel)

Expand All @@ -348,11 +376,16 @@ func GetInventoryReports(cfg *config.Application) (AccountRoutedReports, error)

// Get inventory reports for each account
for account, namespaces := range accountRoutesForAllNamespaces {
accountReport, err := GetInventoryReportForNamespaces(cfg, namespaces)
if err != nil {
return AccountRoutedReports{}, err
for batchCount, batch := range GetNamespacesBatches(namespaces, batchSize) {
if batchSize > 0 {
log.Infof("Collecting inventory batch %d of %d for account %s", batchCount+1, int(math.Ceil(float64(len(namespaces))/float64(batchSize))), account)
}
accountReport, err := GetInventoryReportForNamespaces(cfg, batch)
if err != nil {
return AccountRoutedReports{}, err
}
reports[account] = append(reports[account], accountReport)
}
reports[account] = accountReport
}
}

Expand Down
71 changes: 70 additions & 1 deletion pkg/lib_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (

"github.com/anchore/k8s-inventory/internal/config"
"github.com/anchore/k8s-inventory/pkg/inventory"

"github.com/stretchr/testify/assert"
)

Expand Down Expand Up @@ -223,3 +222,73 @@ func TestGetAccountRoutedNamespaces(t *testing.T) {
})
}
}

func TestGetNamespacesBatches(t *testing.T) {
type args struct {
namespaces []inventory.Namespace
batchSize int
}
tests := []struct {
name string
args args
want [][]inventory.Namespace
}{
{
name: "empty namespaces",
args: args{
namespaces: []inventory.Namespace{},
batchSize: 10,
},
want: [][]inventory.Namespace{},
},
{
name: "single batch",
args: args{
namespaces: TestNamespaces,
batchSize: 10,
},
want: [][]inventory.Namespace{
TestNamespaces,
},
},
{
name: "multiple batches",
args: args{
namespaces: TestNamespaces,
batchSize: 2,
},
want: [][]inventory.Namespace{
{TestNamespace1, TestNamespace2},
{TestNamespace3, TestNamespace4},
},
},
{
name: "multiple batches with remainder",
args: args{
namespaces: append(TestNamespaces, TestNamespace5),
batchSize: 2,
},
want: [][]inventory.Namespace{
{TestNamespace1, TestNamespace2},
{TestNamespace3, TestNamespace4},
{TestNamespace5},
},
},
{
name: "no batches configured (batch size 0)",
args: args{
namespaces: TestNamespaces,
batchSize: 0,
},
want: [][]inventory.Namespace{
TestNamespaces,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := GetNamespacesBatches(tt.args.namespaces, tt.args.batchSize)
assert.Equal(t, tt.want, got)
})
}
}
Loading

0 comments on commit 7655b22

Please sign in to comment.