Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add items to parse functionallity to the mappings #44

Merged
merged 5 commits into from
May 6, 2024
Merged

Add items to parse functionallity to the mappings #44

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b0c81a9
add items to parse functionallity to the mappings
Apr 11, 2024
b6fcd86
add the item to the upper context instead of switch the context
Apr 14, 2024
55567a8
Merge branch 'main' of https://github.com/port-labs/port-k8s-exporter…
itamar-smirra-port Apr 25, 2024
b217def
revert last changes
itamar-smirra-port Apr 25, 2024
02877a2
insert the objects to map to array to remove the duplicate mapping code
itamar-smirra-port May 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 19 additions & 3 deletions pkg/jq/parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,12 @@ package jq

import (
"fmt"
"github.com/itchyny/gojq"
"k8s.io/klog/v2"
"os"
"strings"
"sync"

"github.com/itchyny/gojq"
"k8s.io/klog/v2"
)

var mutex = &sync.Mutex{}
Expand Down Expand Up @@ -66,7 +67,7 @@ func ParseString(jqQuery string, obj interface{}) (string, error) {

str, ok := queryRes.(string)
if !ok {
return "", fmt.Errorf("failed to parse string: %#v", queryRes)
return "", fmt.Errorf("failed to parse string with jq '%#v': %#v", jqQuery, queryRes)
}

return strings.Trim(str, "\""), nil
Expand All @@ -81,6 +82,21 @@ func ParseInterface(jqQuery string, obj interface{}) (interface{}, error) {
return queryRes, nil
}

func ParseArray(jqQuery string, obj interface{}) ([]interface{}, error) {
queryRes, err := runJQQuery(jqQuery, obj)

if err != nil {
return nil, err
}

items, ok := queryRes.([]interface{})
if !ok {
return nil, fmt.Errorf("failed to parse array with jq '%#v': %#v", jqQuery, queryRes)
}

return items, nil
}

func ParseMapInterface(jqQueries map[string]string, obj interface{}) (map[string]interface{}, error) {
mapInterface := make(map[string]interface{}, len(jqQueries))

Expand Down
90 changes: 71 additions & 19 deletions pkg/k8s/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package k8s
import (
"context"
"fmt"
"time"

"github.com/port-labs/port-k8s-exporter/pkg/config"
"github.com/port-labs/port-k8s-exporter/pkg/jq"
"github.com/port-labs/port-k8s-exporter/pkg/port"
Expand All @@ -12,7 +14,6 @@ import (
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/informers"
"time"

utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -184,7 +185,7 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error {

errors := make([]error, 0)
for _, kindConfig := range c.resource.KindConfigs {
portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings)
portEntities, err := c.getObjectEntities(obj, kindConfig.Selector, kindConfig.Port.Entity.Mappings, kindConfig.Port.ItemsToParse)
if err != nil {
utilruntime.HandleError(fmt.Errorf("error getting entities for object key '%s': %v", item.Key, err))
continue
Expand All @@ -205,7 +206,33 @@ func (c *Controller) objectHandler(obj interface{}, item EventItem) error {
return nil
}

func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping) ([]port.Entity, error) {
func isPassSelector(obj interface{}, selector port.Selector) (bool, error) {
if selector.Query == "" {
return true, nil
}

selectorResult, err := jq.ParseBool(selector.Query, obj)
if err != nil {
return false, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err)
}

return selectorResult, err
}

func mapEntities(obj interface{}, mappings []port.EntityMapping) ([]port.Entity, error) {
entities := make([]port.Entity, 0, len(mappings))
for _, entityMapping := range mappings {
portEntity, err := mapping.NewEntity(obj, entityMapping)
if err != nil {
return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err)
}
entities = append(entities, *portEntity)
}

return entities, nil
}

func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector, mappings []port.EntityMapping, itemsToParse string) ([]port.Entity, error) {
unstructuredObj, ok := obj.(*unstructured.Unstructured)
if !ok {
return nil, fmt.Errorf("error casting to unstructured")
Expand All @@ -216,25 +243,50 @@ func (c *Controller) getObjectEntities(obj interface{}, selector port.Selector,
return nil, fmt.Errorf("error converting from unstructured: %v", err)
}

var selectorResult = true
if selector.Query != "" {
selectorResult, err = jq.ParseBool(selector.Query, structuredObj)
if err != nil {
return nil, fmt.Errorf("invalid selector query '%s': %v", selector.Query, err)
entities := make([]port.Entity, 0, len(mappings))
if itemsToParse != "" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this if is redundant
instead always runs the loop and set the single value to an array in case there is no items to parse

Copy link
Contributor Author

@itamar-smirra-port itamar-smirra-port Apr 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I need the if so I can check wether the items array parsing is needed, but I had fix it to use only one loop as you suggest with the nice idea of insert the single object into array.
Let me know if that what you meant.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Apparently I had a problem with this solution, that because the items were overriding each other so only the last item was mapped to entity in the later for loop.
I revert it back to map the item to an entity in each item so it would not happened.
Another solution will be to clone each item to the edited object before insert it to the objectsToMap array but there is no build in clone method for Go so I figured it is a bit overkill.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we talked about this

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

items, parseItemsError := jq.ParseArray(itemsToParse, structuredObj)
if parseItemsError != nil {
return nil, parseItemsError
} else {
itamar-smirra-port marked this conversation as resolved.
Show resolved Hide resolved
editedObject, ok := structuredObj.(map[string]interface{})
if !ok {
return nil, fmt.Errorf("error parsing object '%#v'", structuredObj)
}

for _, item := range items {
editedObject["item"] = item
selectorResult, err := isPassSelector(editedObject, selector)

if err != nil {
return nil, err
}

if selectorResult {
currentEntities, err := mapEntities(editedObject, mappings)
if err != nil {
return nil, err
}

entities = append(entities, currentEntities...)
}
}
}
}
if !selectorResult {
return nil, nil
}
} else {
selectorResult, err := isPassSelector(structuredObj, selector)

entities := make([]port.Entity, 0, len(mappings))
for _, entityMapping := range mappings {
var portEntity *port.Entity
portEntity, err = mapping.NewEntity(structuredObj, entityMapping)
if err != nil {
return nil, fmt.Errorf("invalid entity mapping '%#v': %v", entityMapping, err)
return nil, err
}

if selectorResult {
currentEntities, err := mapEntities(structuredObj, mappings)
if err != nil {
return nil, err
}

entities = append(entities, currentEntities...)
}
entities = append(entities, *portEntity)
}

return entities, nil
Expand Down Expand Up @@ -323,7 +375,7 @@ func (c *Controller) GetEntitiesSet() (map[string]interface{}, error) {
Blueprint: m.Blueprint,
})
}
entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings)
entities, err := c.getObjectEntities(obj, kindConfig.Selector, mappings, kindConfig.Port.ItemsToParse)
if err != nil {
return nil, fmt.Errorf("error getting entities of object: %v", err)
}
Expand Down
9 changes: 5 additions & 4 deletions pkg/k8s/controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,11 @@ package k8s
import (
"context"
"fmt"
"reflect"
"strings"
"testing"
"time"

"github.com/port-labs/port-k8s-exporter/pkg/config"
"github.com/port-labs/port-k8s-exporter/pkg/port"
"github.com/port-labs/port-k8s-exporter/pkg/port/cli"
Expand All @@ -11,10 +16,6 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
k8sfake "k8s.io/client-go/dynamic/fake"
"reflect"
"strings"
"testing"
"time"

appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
Expand Down
3 changes: 2 additions & 1 deletion pkg/port/models.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,8 @@ type EntityMappings struct {
}

type Port struct {
Entity EntityMappings `json:"entity"`
Entity EntityMappings `json:"entity"`
ItemsToParse string `json:"ItemsToParse"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
ItemsToParse string `json:"ItemsToParse"`
ItemsToParse string `json:"itemsToParse"`

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i think we need to add support for yaml in here

Copy link
Contributor Author

@itamar-smirra-port itamar-smirra-port Apr 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In general or just for the itemsToParse?
If in general it maybe be better to create separate task for it.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

its not that big of a deal
just add yaml:"" and the name should be same as json

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it, done.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we talked about it
yaml

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

yairsimantov20 marked this conversation as resolved.
Show resolved Hide resolved
}

type Selector struct {
Expand Down
Loading