From d520a6651e11bcfbc10df1f568fb4e1d681e27c4 Mon Sep 17 00:00:00 2001 From: Rohit Jangid Date: Thu, 25 May 2023 10:40:44 +0530 Subject: [PATCH] feat: Add std.parseCsv and std.manifestCsv --- builtins.go | 143 +++++++++++++++++++++ linter/internal/types/stdlib.go | 2 + testdata/builtinManifestCsv.golden | 1 + testdata/builtinManifestCsv.jsonnet | 1 + testdata/builtinManifestCsv.linter.golden | 0 testdata/builtinManifestCsv2.golden | 1 + testdata/builtinManifestCsv2.jsonnet | 1 + testdata/builtinManifestCsv2.linter.golden | 0 testdata/builtinParseCsv.golden | 8 ++ testdata/builtinParseCsv.jsonnet | 1 + testdata/builtinParseCsv.linter.golden | 0 11 files changed, 158 insertions(+) create mode 100644 testdata/builtinManifestCsv.golden create mode 100644 testdata/builtinManifestCsv.jsonnet create mode 100644 testdata/builtinManifestCsv.linter.golden create mode 100644 testdata/builtinManifestCsv2.golden create mode 100644 testdata/builtinManifestCsv2.jsonnet create mode 100644 testdata/builtinManifestCsv2.linter.golden create mode 100644 testdata/builtinParseCsv.golden create mode 100644 testdata/builtinParseCsv.jsonnet create mode 100644 testdata/builtinParseCsv.linter.golden diff --git a/builtins.go b/builtins.go index 875fd5f24..04852dd08 100644 --- a/builtins.go +++ b/builtins.go @@ -20,6 +20,7 @@ import ( "bytes" "crypto/md5" "encoding/base64" + "encoding/csv" "encoding/hex" "encoding/json" "fmt" @@ -1425,6 +1426,146 @@ func builtinParseYAML(i *interpreter, str value) (value, error) { return jsonToValue(i, elems[0]) } +func builtinParseCSV(i *interpreter, str value) (value, error) { + sval, err := i.getString(str) + if err != nil { + return nil, err + } + s := sval.getGoString() + + json := make(map[string]interface{}) + var keys []string + reader := csv.NewReader(strings.NewReader(s)) + for { + record, err := reader.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, i.Error(fmt.Sprintf("failed to parse JSON: %s", err.Error())) + } + + if len(keys) == 0 { + keys = record + for _, r := range record { + json[r] = []interface{}{} + } + } else { + for i, r := range record { + k := keys[i] + v := json[k].([]interface{}) + json[k] = append(v, r) + } + } + } + return jsonToValue(i, json) +} + +func builtinManifestCsv(i *interpreter, arguments []value) (value, error) { + objv := arguments[0] + hv := arguments[1] + + obj, err := i.getObject(objv) + if err != nil { + return nil, err + } + + var headers []string + if hv.getType() == nullType { + // default to all headers + simpleObj := obj.uncached.(*simpleObject) + for fieldName := range simpleObj.fields { + headers = append(headers, fieldName) + } + } else { + // headers are provided + ha, err := i.getArray(hv) + if err != nil { + return nil, err + } + + for _, elem := range ha.elements { + header, err := i.evaluateString(elem) + if err != nil { + return nil, err + } + headers = append(headers, header.getGoString()) + } + } + + var buf bytes.Buffer + w := csv.NewWriter(&buf) + w.Write(headers) + + for r := 0; ; r++ { + record := make([]string, len(headers)) + elems := 0 + for c, h := range headers { + arrv, err := obj.index(i, h) + if err != nil { // no corresponding column + // skip to next column + continue + } + + v, err := i.getArray(arrv) + if err != nil { + return nil, i.Error("invalid JSON: not a valid json for CSV") + } + + if r >= len(v.elements) { // if less elements in record + continue + } + val, err := v.elements[r].getValue(i) + if err != nil { + return nil, err + } + + s, err := stringFromValue(i, val) + if err != nil { + return nil, err + } + record[c] = s + elems++ + } + if elems == 0 { // No elements in record + break + } + w.Write(record) + } + + w.Flush() + + return makeValueString(buf.String()), nil +} + +func stringFromValue(i *interpreter, v value) (string, error) { + switch v.getType() { + case stringType: + s, err := i.getString(v) + if err != nil { + return "", err + } + return s.getGoString(), nil + case numberType: + n, err := i.getNumber(v) + if err != nil { + return "", err + } + return fmt.Sprint(n.value), nil + case booleanType: + b, err := i.getBoolean(v) + if err != nil { + return "", err + } + return fmt.Sprint(b.value), nil + case nullType: + return "", nil + default: + // for functionType, objectType and arrayType + return "", i.Error("invalid string conversion") + } +} + func jsonEncode(v interface{}) (string, error) { buf := new(bytes.Buffer) enc := json.NewEncoder(buf) @@ -2290,6 +2431,8 @@ var funcBuiltins = buildBuiltinMap([]builtin{ &unaryBuiltin{name: "parseInt", function: builtinParseInt, params: ast.Identifiers{"str"}}, &unaryBuiltin{name: "parseJson", function: builtinParseJSON, params: ast.Identifiers{"str"}}, &unaryBuiltin{name: "parseYaml", function: builtinParseYAML, params: ast.Identifiers{"str"}}, + &unaryBuiltin{name: "parseCsv", function: builtinParseCSV, params: ast.Identifiers{"str"}}, + &generalBuiltin{name: "manifestCsv", function: builtinManifestCsv, params: []generalBuiltinParameter{{name: "obj"}, {name: "arr", defaultValue: &nullValue}}}, &generalBuiltin{name: "manifestJsonEx", function: builtinManifestJSONEx, params: []generalBuiltinParameter{{name: "value"}, {name: "indent"}, {name: "newline", defaultValue: &valueFlatString{value: []rune("\n")}}, {name: "key_val_sep", defaultValue: &valueFlatString{value: []rune(": ")}}}}, diff --git a/linter/internal/types/stdlib.go b/linter/internal/types/stdlib.go index dee12b27c..d0e4b72ac 100644 --- a/linter/internal/types/stdlib.go +++ b/linter/internal/types/stdlib.go @@ -103,6 +103,7 @@ func prepareStdlib(g *typeGraph) { "parseHex": g.newSimpleFuncType(numberType, "str"), "parseJson": g.newSimpleFuncType(jsonType, "str"), "parseYaml": g.newSimpleFuncType(jsonType, "str"), + "parseCsv": g.newSimpleFuncType(jsonType, "str"), "encodeUTF8": g.newSimpleFuncType(numberArrayType, "str"), "decodeUTF8": g.newSimpleFuncType(stringType, "arr"), @@ -116,6 +117,7 @@ func prepareStdlib(g *typeGraph) { "manifestJsonMinified": g.newSimpleFuncType(stringType, "value"), "manifestYamlDoc": g.newSimpleFuncType(stringType, "value"), "manifestYamlStream": g.newSimpleFuncType(stringType, "value"), + "manifestCsv": g.newSimpleFuncType(stringType, "obj", "arr"), "manifestXmlJsonml": g.newSimpleFuncType(stringType, "value"), // Arrays diff --git a/testdata/builtinManifestCsv.golden b/testdata/builtinManifestCsv.golden new file mode 100644 index 000000000..b87af8a56 --- /dev/null +++ b/testdata/builtinManifestCsv.golden @@ -0,0 +1 @@ +"head1,head2\nval1,val2\n,1\nval3,\n" diff --git a/testdata/builtinManifestCsv.jsonnet b/testdata/builtinManifestCsv.jsonnet new file mode 100644 index 000000000..d494f540a --- /dev/null +++ b/testdata/builtinManifestCsv.jsonnet @@ -0,0 +1 @@ +std.manifestCsv({ "head1": ["val1", "", "val3"], "head2": ["val2", 1], "head3": ["foo", "bar"] }, ["head1", "head2"]) \ No newline at end of file diff --git a/testdata/builtinManifestCsv.linter.golden b/testdata/builtinManifestCsv.linter.golden new file mode 100644 index 000000000..e69de29bb diff --git a/testdata/builtinManifestCsv2.golden b/testdata/builtinManifestCsv2.golden new file mode 100644 index 000000000..17dbafb14 --- /dev/null +++ b/testdata/builtinManifestCsv2.golden @@ -0,0 +1 @@ +"head1\nval1\nval2\n" diff --git a/testdata/builtinManifestCsv2.jsonnet b/testdata/builtinManifestCsv2.jsonnet new file mode 100644 index 000000000..ad12a7d0b --- /dev/null +++ b/testdata/builtinManifestCsv2.jsonnet @@ -0,0 +1 @@ +std.manifestCsv({ "head1": ["val1", "val2"] }) \ No newline at end of file diff --git a/testdata/builtinManifestCsv2.linter.golden b/testdata/builtinManifestCsv2.linter.golden new file mode 100644 index 000000000..e69de29bb diff --git a/testdata/builtinParseCsv.golden b/testdata/builtinParseCsv.golden new file mode 100644 index 000000000..a36eaad75 --- /dev/null +++ b/testdata/builtinParseCsv.golden @@ -0,0 +1,8 @@ +{ + "head1": [ + "val1" + ], + "head2": [ + "val2" + ] +} diff --git a/testdata/builtinParseCsv.jsonnet b/testdata/builtinParseCsv.jsonnet new file mode 100644 index 000000000..986f9fdc6 --- /dev/null +++ b/testdata/builtinParseCsv.jsonnet @@ -0,0 +1 @@ +std.parseCsv("head1,head2\nval1,val2") \ No newline at end of file diff --git a/testdata/builtinParseCsv.linter.golden b/testdata/builtinParseCsv.linter.golden new file mode 100644 index 000000000..e69de29bb