diff --git a/README.md b/README.md index d56d825..5da15af 100644 --- a/README.md +++ b/README.md @@ -419,16 +419,16 @@ Include only a subset of annotations/labels for each resource type or disable me ```yaml metadata-collection: nodes: - include-annotations: [] # List of annotations to include (explicit or regex) - include-labels: [] # List of labels to include (explicit or regex) + annotations: [] # List of annotations to include (explicit or regex) + labels: [] # List of labels to include (explicit or regex) disable: false # Remove all optional node metadata from the inventory report namespaces: - include-annotations: [] # List of annotations to include (explicit or regex) - include-labels: [] # List of labels to include (explicit or regex) + annotations: [] # List of annotations to include (explicit or regex) + labels: [] # List of labels to include (explicit or regex) disable: false # Remove all optional namespace metadata from the inventory report pods: - include-annotations: [] # List of annotations to include (explicit or regex) - include-labels: [] # List of labels to include (explicit or regex) + annotations: [] # List of annotations to include (explicit or regex) + labels: [] # List of labels to include (explicit or regex) disable: false # Remove all optional pod metadata from the inventory report ``` diff --git a/pkg/inventory/namespace.go b/pkg/inventory/namespace.go index 4878b1a..d846b5c 100644 --- a/pkg/inventory/namespace.go +++ b/pkg/inventory/namespace.go @@ -75,7 +75,7 @@ func FetchNamespaces( c client.Client, batchSize, timeout int64, excludes, includes []string, - // annotations, labels []string, + includeAnnotations, includeLabels []string, disableMetadata bool, ) ([]Namespace, error) { defer tracker.TrackFunctionTime(time.Now(), "Fetching namespaces") @@ -98,11 +98,14 @@ func FetchNamespaces( for _, n := range list.Items { if !excludeNamespace(exclusionChecklist, n.ObjectMeta.Name) { if !disableMetadata { + annotations := processAnnotationsOrLabels(n.Annotations, includeAnnotations) + labels := processAnnotationsOrLabels(n.Labels, includeLabels) + nsMap[n.ObjectMeta.Name] = Namespace{ Name: n.ObjectMeta.Name, UID: string(n.UID), - Annotations: n.Annotations, - Labels: n.Labels, + Annotations: annotations, + Labels: labels, } } else { nsMap[n.ObjectMeta.Name] = Namespace{ diff --git a/pkg/inventory/namespace_test.go b/pkg/inventory/namespace_test.go index b574f8b..d0362ef 100644 --- a/pkg/inventory/namespace_test.go +++ b/pkg/inventory/namespace_test.go @@ -13,12 +13,14 @@ import ( func Test_fetchNamespaces(t *testing.T) { type args struct { - c client.Client - batchSize int64 - timeout int64 - excludes []string - includes []string - disableMetadata bool + c client.Client + batchSize int64 + timeout int64 + excludes []string + includes []string + includeAnnotations []string + includeLabels []string + disableMetadata bool } tests := []struct { name string @@ -43,11 +45,13 @@ func Test_fetchNamespaces(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 10, - excludes: []string{}, - includes: []string{}, - disableMetadata: false, + batchSize: 100, + timeout: 10, + excludes: []string{}, + includes: []string{}, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: []Namespace{ { @@ -64,11 +68,13 @@ func Test_fetchNamespaces(t *testing.T) { c: client.Client{ Clientset: fake.NewSimpleClientset(), }, - batchSize: 100, - timeout: 10, - excludes: []string{}, - includes: []string{}, - disableMetadata: false, + batchSize: 100, + timeout: 10, + excludes: []string{}, + includes: []string{}, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: nil, }, @@ -102,11 +108,13 @@ func Test_fetchNamespaces(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 10, - excludes: []string{"excluded-namespace"}, - includes: []string{}, - disableMetadata: false, + batchSize: 100, + timeout: 10, + excludes: []string{"excluded-namespace"}, + includes: []string{}, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: []Namespace{ { @@ -159,11 +167,13 @@ func Test_fetchNamespaces(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 10, - excludes: []string{"excluded.*"}, - includes: []string{}, - disableMetadata: false, + batchSize: 100, + timeout: 10, + excludes: []string{"excluded.*"}, + includes: []string{}, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: []Namespace{ { @@ -216,11 +226,13 @@ func Test_fetchNamespaces(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 10, - excludes: []string{"exclude.*"}, - includes: []string{"test-namespace"}, - disableMetadata: false, + batchSize: 100, + timeout: 10, + excludes: []string{"exclude.*"}, + includes: []string{"test-namespace"}, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: []Namespace{ { @@ -273,11 +285,13 @@ func Test_fetchNamespaces(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 10, - excludes: []string{}, - includes: []string{"test-namespace"}, - disableMetadata: false, + batchSize: 100, + timeout: 10, + excludes: []string{}, + includes: []string{"test-namespace"}, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: []Namespace{ { @@ -330,11 +344,13 @@ func Test_fetchNamespaces(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 10, - excludes: []string{}, - includes: []string{"test-namespace"}, - disableMetadata: true, + batchSize: 100, + timeout: 10, + excludes: []string{}, + includes: []string{"test-namespace"}, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: true, }, want: []Namespace{ { @@ -343,6 +359,91 @@ func Test_fetchNamespaces(t *testing.T) { }, }, }, + { + name: "only includes specified namespace annotations and labels", + args: args{ + c: client.Client{ + Clientset: fake.NewSimpleClientset( + &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + UID: "test-uid", + Annotations: map[string]string{ + "test-annotation": "test-value", + "test-annotation2": "test-value2", + "do-not-include": "do-not-include", + }, + Labels: map[string]string{ + "test-label": "test-value", + "do-not-include": "do-not-include", + }, + }, + }), + }, + batchSize: 100, + timeout: 10, + excludes: []string{}, + includes: []string{"test-namespace"}, + includeAnnotations: []string{"test-annotation", "test-annotation2"}, + includeLabels: []string{"test-label"}, + disableMetadata: false, + }, + want: []Namespace{ + { + Name: "test-namespace", + UID: "test-uid", + Annotations: map[string]string{ + "test-annotation": "test-value", + "test-annotation2": "test-value2", + }, + Labels: map[string]string{ + "test-label": "test-value", + }, + }, + }, + }, + { + name: "only includes specified namespace annotations and labels (regex)", + args: args{ + c: client.Client{ + Clientset: fake.NewSimpleClientset( + &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-namespace", + UID: "test-uid", + Annotations: map[string]string{ + "test-annotation": "test-value", + "test-annotation2": "test-value2", + "do-not-include": "do-not-include", + }, + Labels: map[string]string{ + "test-label": "test-value", + "do-not-include": "do-not-include", + }, + }, + }), + }, + batchSize: 100, + timeout: 10, + excludes: []string{}, + includes: []string{}, + includeAnnotations: []string{".*-not-.*"}, + includeLabels: []string{".*-not-.*"}, + disableMetadata: false, + }, + want: []Namespace{ + { + Name: "test-namespace", + UID: "test-uid", + Annotations: map[string]string{ + "do-not-include": "do-not-include", + }, + Labels: map[string]string{ + "do-not-include": "do-not-include", + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -352,6 +453,8 @@ func Test_fetchNamespaces(t *testing.T) { tt.args.timeout, tt.args.excludes, tt.args.includes, + tt.args.includeAnnotations, + tt.args.includeLabels, tt.args.disableMetadata, ) if (err != nil) != tt.wantErr { diff --git a/pkg/inventory/nodes.go b/pkg/inventory/nodes.go index 5772ec2..8074afb 100644 --- a/pkg/inventory/nodes.go +++ b/pkg/inventory/nodes.go @@ -10,7 +10,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func FetchNodes(c client.Client, batchSize, timeout int64, disableMetadata bool) (map[string]Node, error) { +func FetchNodes(c client.Client, batchSize, timeout int64, includeAnnotations, includeLabels []string, disableMetadata bool) (map[string]Node, error) { nodes := make(map[string]Node) cont := "" @@ -32,16 +32,18 @@ func FetchNodes(c client.Client, batchSize, timeout int64, disableMetadata bool) for _, n := range list.Items { if !disableMetadata { + annotations := processAnnotationsOrLabels(n.Annotations, includeAnnotations) + labels := processAnnotationsOrLabels(n.Labels, includeLabels) nodes[n.ObjectMeta.Name] = Node{ Name: n.ObjectMeta.Name, UID: string(n.UID), - Annotations: n.Annotations, + Annotations: annotations, Arch: n.Status.NodeInfo.Architecture, ContainerRuntimeVersion: n.Status.NodeInfo.ContainerRuntimeVersion, KernelVersion: n.Status.NodeInfo.KernelVersion, KubeProxyVersion: n.Status.NodeInfo.KubeProxyVersion, KubeletVersion: n.Status.NodeInfo.KubeletVersion, - Labels: n.Labels, + Labels: labels, OperatingSystem: n.Status.NodeInfo.OperatingSystem, } } else { diff --git a/pkg/inventory/nodes_test.go b/pkg/inventory/nodes_test.go index 77eeb7a..14cea0e 100644 --- a/pkg/inventory/nodes_test.go +++ b/pkg/inventory/nodes_test.go @@ -12,10 +12,12 @@ import ( func TestFetchNodes(t *testing.T) { type args struct { - c client.Client - batchSize int64 - timeout int64 - disableMetadata bool + c client.Client + batchSize int64 + timeout int64 + includeAnnotations []string + includeLabels []string + disableMetadata bool } tests := []struct { name string @@ -50,9 +52,11 @@ func TestFetchNodes(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 100, - disableMetadata: false, + batchSize: 100, + timeout: 100, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: map[string]Node{ "test-node": { @@ -100,9 +104,11 @@ func TestFetchNodes(t *testing.T) { }, }), }, - batchSize: 100, - timeout: 100, - disableMetadata: true, + batchSize: 100, + timeout: 100, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: true, }, want: map[string]Node{ "test-node": { @@ -117,10 +123,64 @@ func TestFetchNodes(t *testing.T) { }, }, }, + { + name: "successfully returns nodes with filtered annotation/label metadata", + args: args{ + c: client.Client{ + Clientset: fake.NewSimpleClientset(&v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + UID: "test-uid", + Annotations: map[string]string{ + "test-annotation": "test-value", + "test-annotation-2": "test-value-2", + }, + Labels: map[string]string{ + "test-label": "test-value", + "test-label-2": "test-value-2", + }, + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + Architecture: "arm64", + ContainerRuntimeVersion: "docker://20.10.23", + KernelVersion: "5.15.49-linuxkit", + KubeProxyVersion: "v1.26.1", + KubeletVersion: "v1.26.1", + OperatingSystem: "linux", + }, + }, + }), + }, + batchSize: 100, + timeout: 100, + includeAnnotations: []string{".*-2$"}, + includeLabels: []string{".*-2$"}, + disableMetadata: false, + }, + want: map[string]Node{ + "test-node": { + Name: "test-node", + UID: "test-uid", + Arch: "arm64", + ContainerRuntimeVersion: "docker://20.10.23", + KernelVersion: "5.15.49-linuxkit", + KubeProxyVersion: "v1.26.1", + KubeletVersion: "v1.26.1", + OperatingSystem: "linux", + Annotations: map[string]string{ + "test-annotation-2": "test-value-2", + }, + Labels: map[string]string{ + "test-label-2": "test-value-2", + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := FetchNodes(tt.args.c, tt.args.batchSize, tt.args.timeout, tt.args.disableMetadata) + got, err := FetchNodes(tt.args.c, tt.args.batchSize, tt.args.timeout, tt.args.includeAnnotations, tt.args.includeLabels, tt.args.disableMetadata) if (err != nil) != tt.wantErr { assert.Error(t, err) } diff --git a/pkg/inventory/pods.go b/pkg/inventory/pods.go index 732f542..8c3c2d1 100644 --- a/pkg/inventory/pods.go +++ b/pkg/inventory/pods.go @@ -40,7 +40,7 @@ func FetchPodsInNamespace(c client.Client, batchSize, timeout int64, namespace s return podList, nil } -func ProcessPods(pods []v1.Pod, namespaceUID string, nodes map[string]Node, disableMetadata bool) []Pod { +func ProcessPods(pods []v1.Pod, namespaceUID string, nodes map[string]Node, includeAnnotations, includeLabels []string, disableMetadata bool) []Pod { var podList []Pod for _, p := range pods { @@ -50,8 +50,8 @@ func ProcessPods(pods []v1.Pod, namespaceUID string, nodes map[string]Node, disa NamespaceUID: namespaceUID, } if !disableMetadata { - pod.Labels = p.Labels - pod.Annotations = p.Annotations + pod.Labels = processAnnotationsOrLabels(p.Labels, includeLabels) + pod.Annotations = processAnnotationsOrLabels(p.Annotations, includeAnnotations) } node, ok := nodes[p.Spec.NodeName] if ok { diff --git a/pkg/inventory/pods_test.go b/pkg/inventory/pods_test.go index 4ceb0a5..78cd3f7 100644 --- a/pkg/inventory/pods_test.go +++ b/pkg/inventory/pods_test.go @@ -73,10 +73,12 @@ func TestFetchPodsInNamespace(t *testing.T) { func TestProcessPods(t *testing.T) { type args struct { - pods []v1.Pod - namespaceUID string - nodes map[string]Node - disableMetadata bool + pods []v1.Pod + namespaceUID string + nodes map[string]Node + includeAnnotations []string + includeLabels []string + disableMetadata bool } tests := []struct { name string @@ -111,7 +113,9 @@ func TestProcessPods(t *testing.T) { UID: "test-node-uid", }, }, - disableMetadata: false, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: false, }, want: []Pod{ { @@ -156,7 +160,9 @@ func TestProcessPods(t *testing.T) { UID: "test-node-uid", }, }, - disableMetadata: true, + includeAnnotations: []string{}, + includeLabels: []string{}, + disableMetadata: true, }, want: []Pod{ { @@ -167,10 +173,59 @@ func TestProcessPods(t *testing.T) { }, }, }, + { + name: "successfully return pods with filtered annotations/labels", + args: args{ + pods: []v1.Pod{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + UID: "test-uid", + Annotations: map[string]string{ + "test-annotation": "test-value", + "do-not-include": "do-not-include", + }, + Labels: map[string]string{ + "test-label": "test-value", + "do-not-include": "do-not-include", + }, + Namespace: "test-namespace", + }, + Spec: v1.PodSpec{ + NodeName: "test-node", + }, + }, + }, + namespaceUID: "namespace-uid-0000", + nodes: map[string]Node{ + "test-node": { + Name: "test-node", + UID: "test-node-uid", + }, + }, + includeAnnotations: []string{"test-.*"}, + includeLabels: []string{"test-.*"}, + disableMetadata: false, + }, + want: []Pod{ + { + Name: "test-pod", + UID: "test-uid", + NamespaceUID: "namespace-uid-0000", + NodeUID: "test-node-uid", + Annotations: map[string]string{ + "test-annotation": "test-value", + }, + Labels: map[string]string{ + "test-label": "test-value", + }, + }, + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := ProcessPods(tt.args.pods, tt.args.namespaceUID, tt.args.nodes, tt.args.disableMetadata) + got := ProcessPods(tt.args.pods, tt.args.namespaceUID, tt.args.nodes, tt.args.includeAnnotations, tt.args.includeLabels, tt.args.disableMetadata) assert.Equal(t, tt.want, got) }) } diff --git a/pkg/inventory/util.go b/pkg/inventory/util.go new file mode 100644 index 0000000..1cef5e5 --- /dev/null +++ b/pkg/inventory/util.go @@ -0,0 +1,22 @@ +package inventory + +import "regexp" + +func checkForRegexMatch(regex string, value string) bool { + return regexp.MustCompile(regex).MatchString(value) +} + +func processAnnotationsOrLabels(annotationsOrLabels map[string]string, include []string) map[string]string { + if len(include) == 0 { + return annotationsOrLabels + } + toReturn := make(map[string]string) + for key, val := range annotationsOrLabels { + for _, includeKey := range include { + if checkForRegexMatch(includeKey, key) { + toReturn[key] = val + } + } + } + return toReturn +} diff --git a/pkg/inventory/util_test.go b/pkg/inventory/util_test.go new file mode 100644 index 0000000..05d1d0f --- /dev/null +++ b/pkg/inventory/util_test.go @@ -0,0 +1,137 @@ +package inventory + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func Test_processAnnotations(t *testing.T) { + type args struct { + annotationsOrLabels map[string]string + include []string + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "Empty include", + args: args{ + annotationsOrLabels: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + include: []string{}, + }, + want: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + { + name: "Include explicit keys", + args: args{ + annotationsOrLabels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value2", + }, + include: []string{"key1", "key3"}, + }, + want: map[string]string{ + "key1": "value1", + "key3": "value2", + }, + }, + { + name: "Include non-existent keys", + args: args{ + annotationsOrLabels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value2", + }, + include: []string{"key1", "key4"}, + }, + want: map[string]string{ + "key1": "value1", + }, + }, + { + name: "Empty annotationsOrLabels", + args: args{ + annotationsOrLabels: map[string]string{}, + include: []string{"key1", "key4"}, + }, + want: map[string]string{}, + }, + { + name: "Include keys by regex pattern", + args: args{ + annotationsOrLabels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value2", + }, + include: []string{"key[1-2]"}, + }, + want: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + { + name: "Include keys by regex pattern (all)", + args: args{ + annotationsOrLabels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value2", + }, + include: []string{".*"}, + }, + want: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value2", + }, + }, + { + name: "Include keys by regex pattern (non-existent)", + args: args{ + annotationsOrLabels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value2", + }, + include: []string{"key[4-5]"}, + }, + want: map[string]string{}, + }, + { + name: "Include keys by regex pattern and explicit", + args: args{ + annotationsOrLabels: map[string]string{ + "key1": "value1", + "key2": "value2", + "key3": "value2", + "explicit": "value2", + }, + include: []string{"key[1-2]", "explicit"}, + }, + want: map[string]string{ + "key1": "value1", + "key2": "value2", + "explicit": "value2", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := processAnnotationsOrLabels(tt.args.annotationsOrLabels, tt.args.include) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/pkg/lib.go b/pkg/lib.go index 4825a81..dfb8b53 100644 --- a/pkg/lib.go +++ b/pkg/lib.go @@ -200,6 +200,8 @@ func GetInventoryReportForNamespaces( client, cfg.Kubernetes.RequestBatchSize, cfg.Kubernetes.RequestTimeoutSeconds, + cfg.MetadataCollection.Nodes.Annotations, + cfg.MetadataCollection.Nodes.Labels, cfg.MetadataCollection.Nodes.Disable, ) if err != nil { @@ -272,7 +274,9 @@ func GetAllNamespaces(cfg *config.Application) ([]inventory.Namespace, error) { namespaces, err := inventory.FetchNamespaces(client, cfg.Kubernetes.RequestBatchSize, cfg.Kubernetes.RequestTimeoutSeconds, - cfg.NamespaceSelectors.Exclude, cfg.NamespaceSelectors.Include, cfg.MetadataCollection.Namespace.Disable) + cfg.NamespaceSelectors.Exclude, cfg.NamespaceSelectors.Include, + cfg.MetadataCollection.Namespace.Annotations, cfg.MetadataCollection.Namespace.Labels, + cfg.MetadataCollection.Namespace.Disable) if err != nil { return []inventory.Namespace{}, err } @@ -484,7 +488,7 @@ func processNamespace( return } - pods := inventory.ProcessPods(v1pods, ns.UID, nodes, cfg.MetadataCollection.Pods.Disable) + pods := inventory.ProcessPods(v1pods, ns.UID, nodes, cfg.MetadataCollection.Pods.Annotations, cfg.MetadataCollection.Pods.Labels, cfg.MetadataCollection.Pods.Disable) containers := inventory.GetContainersFromPods( v1pods, cfg.IgnoreNotRunning,