Skip to content

Commit

Permalink
More missing methods implemented in sjsonnet (#227)
Browse files Browse the repository at this point in the history 
- Basically almost every missing methods
- Also imported the most recent version of the c-jsonnet stdlib test
file
  • Loading branch information
@stephenamar-db
stephenamar-db authored Dec 10, 2024
1 parent 2beaacc commit 6e03591
Show file tree
Hide file tree
Showing 14 changed files with 2,671 additions and 146 deletions.
7 changes: 5 additions & 2 deletions sjsonnet/src-jvm/sjsonnet/Platform.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,13 @@ object Platform {
def yamlToJson(yamlString: String): String = {
try {
val yaml = new Yaml(new SafeConstructor(new LoaderOptions())).loadAll(yamlString)
.asInstanceOf[java.lang.Iterable[java.util.Map[String, Object]]].asScala.toSeq
.asInstanceOf[java.lang.Iterable[java.util.Collection[Object]]].asScala.toSeq
yaml.size match {
case 0 => "{}"
case 1 => new JSONObject(yaml.head).toString()
case 1 if yaml.head.isInstanceOf[java.util.Map[String, Object]] =>
new JSONObject(yaml.head.asInstanceOf[java.util.Map[String, Object]]).toString()
case 1 if yaml.head.isInstanceOf[java.util.List[Object]] =>
new JSONArray(yaml.head.asInstanceOf[java.util.List[Object]]).toString()
case _ => new JSONArray(yaml.asJava).toString()
}
} catch {
Expand Down
8 changes: 8 additions & 0 deletions sjsonnet/src/sjsonnet/Evaluator.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -664,6 +664,14 @@ class Evaluator(resolver: CachedResolver,
case Nil => scopes
}

def compare(x: Val, y: Val): Int = (x, y) match {
case (_: Val.Null, _: Val.Null) => 0
case (x: Val.Num, y: Val.Num) => x.value.compareTo(y.value)
case (x: Val.Str, y: Val.Str) => x.value.compareTo(y.value)
case (x: Val.Bool, y: Val.Bool) => x.asBoolean.compareTo(y.asBoolean)
case _ => Error.fail("Cannot compare " + x.prettyName + " with " + y.prettyName, x.pos)
}

def equal(x: Val, y: Val): Boolean = (x eq y) || (x match {
case _: Val.True => y.isInstanceOf[Val.True]
case _: Val.False => y.isInstanceOf[Val.False]
Expand Down
206 changes: 158 additions & 48 deletions sjsonnet/src/sjsonnet/Std.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import sjsonnet.Expr.Member.Visibility

import scala.collection.Searching._
import scala.collection.mutable
import scala.util.control.Breaks.{break, breakable}
import scala.util.matching.Regex

/**
Expand All @@ -19,6 +20,8 @@ import scala.util.matching.Regex
class Std {
private val dummyPos: Position = new Position(null, 0)
private val emptyLazyArray = new Array[Lazy](0)
private val leadingWhiteSpacePattern = Pattern.compile("^[ \t\n\f\r\u0085\u00A0']+")
private val trailingWhiteSpacePattern = Pattern.compile("[ \t\n\f\r\u0085\u00A0']+$")

private object AssertEqual extends Val.Builtin2("assertEqual", "a", "b") {
def evalRhs(v1: Val, v2: Val, ev: EvalScope, pos: Position): Val = {
Expand Down Expand Up @@ -131,20 +134,58 @@ class Std {
}


private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hidden"), Array(null, null, Val.Null(dummyPos), Val.True(dummyPos))) {
override def evalRhs(args: Array[_ <: Lazy], ev: EvalScope, pos: Position): Val = {
val obj = args(0).force.asObj
val k = args(1).force.asString
val incHidden = args(3).force.asBoolean
if (incHidden && obj.containsKey(k)) {
obj.value(k, pos.noOffset, obj)(ev)
} else if (!incHidden && obj.containsVisibleKey(k)) {
obj.value(k, pos.noOffset, obj)(ev)
} else {
args(2).force
private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hidden"), Array(null, null, Val.Null(dummyPos), Val.True(dummyPos))) {
override def evalRhs(args: Array[_ <: Lazy], ev: EvalScope, pos: Position): Val = {
val obj = args(0).force.asObj
val k = args(1).force.asString
val incHidden = args(3).force.asBoolean
if (incHidden && obj.containsKey(k)) {
obj.value(k, pos.noOffset, obj)(ev)
} else if (!incHidden && obj.containsVisibleKey(k)) {
obj.value(k, pos.noOffset, obj)(ev)
} else {
args(2).force
}
}
}

private object MinArray extends Val.Builtin("minArray", Array("arr", "keyF", "onEmpty"), Array(null, Val.False(dummyPos), Val.False(dummyPos))) {
override def evalRhs(args: Array[_ <: Lazy], ev: EvalScope, pos: Position): Val = {
val arr = args(0).force.asArr
val keyF = args(1).force
val onEmpty = args(2)
if (arr.length == 0) {
if (onEmpty.isInstanceOf[Val.False]) {
Error.fail("Expected at least one element in array. Got none")
} else {
onEmpty.force
}
} else if (keyF.isInstanceOf[Val.False]) {
arr.asStrictArray.min(ev)
} else {
arr.asStrictArray.map(v => keyF.asInstanceOf[Val.Func].apply1(v, pos.fileScope.noOffsetPos)(ev)).min(ev)
}
}
}

private object MaxArray extends Val.Builtin("maxArray", Array("arr", "keyF", "onEmpty"), Array(null, Val.False(dummyPos), Val.False(dummyPos))) {
override def evalRhs(args: Array[_ <: Lazy], ev: EvalScope, pos: Position): Val = {
val arr = args(0).force.asArr
val keyF = args(1).force
val onEmpty = args(2)
if (arr.length == 0) {
if (onEmpty.isInstanceOf[Val.False]) {
Error.fail("Expected at least one element in array. Got none")
} else {
onEmpty.force
}
} else if (keyF.isInstanceOf[Val.False]) {
arr.asStrictArray.max(ev)
} else {
arr.asStrictArray.map(v => keyF.asInstanceOf[Val.Func].apply1(v, pos.fileScope.noOffsetPos)(ev)).max(ev)
}
}
}
}

private object Any extends Val.Builtin1("any", "arr") {
def evalRhs(arr: Val, ev: EvalScope, pos: Position): Val = {
Expand Down Expand Up @@ -362,12 +403,12 @@ private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hi
}
}

private object EncodeUTF8 extends Val.Builtin1("encodeUtf8", "s") {
private object EncodeUTF8 extends Val.Builtin1("encodeUTF8", "s") {
def evalRhs(s: Val, ev: EvalScope, pos: Position): Val =
new Val.Arr(pos, s.asString.getBytes(UTF_8).map(i => Val.Num(pos, i & 0xff)))
}

private object DecodeUTF8 extends Val.Builtin1("decodeUtf8", "arr") {
private object DecodeUTF8 extends Val.Builtin1("decodeUTF8", "arr") {
def evalRhs(arr: Val, ev: EvalScope, pos: Position): Val =
new Val.Str(pos, new String(arr.asArr.iterator.map(_.cast[Val.Num].value.toByte).toArray, UTF_8))
}
Expand Down Expand Up @@ -489,31 +530,46 @@ private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hi
}
}

private object Split extends Val.Builtin2("split", "str", "c") {
def evalRhs(_str: Val, _c: Val, ev: EvalScope, pos: Position): Val = {
val str = _str.asString
val cStr = _c.asString
if(cStr.length != 1) Error.fail("std.split second parameter should have length 1, got "+cStr.length)
val c = cStr.charAt(0)
val b = new mutable.ArrayBuilder.ofRef[Lazy]
var i = 0
var start = 0
while(i < str.length) {
if(str.charAt(i) == c) {
val finalStr = Val.Str(pos, str.substring(start, i))
private def splitLimit(pos: Position, str: String, cStr: String, maxSplits: Int): Array[Lazy] = {
val b = new mutable.ArrayBuilder.ofRef[Lazy]
var sz = 0
var i = 0
var start = 0
breakable {
while (i < str.length) {
if (maxSplits >= 0 && sz >= maxSplits) {
break
}
if (str.slice(i, i + cStr.length) == cStr) {
val finalStr = Val.Str(pos, str.slice(start, i))
b.+=(finalStr)
start = i+1
start = i + cStr.length
sz += 1
}
i += 1
}
b.+=(Val.Str(pos, str.substring(start, math.min(i, str.length))))
new Val.Arr(pos, b.result())
}
b.+=(Val.Str(pos, str.substring(start)))
sz += 1
b.result()
}

private object Split extends Val.Builtin2("split", "str", "c") {
def evalRhs(str: Val, c: Val, ev: EvalScope, pos: Position): Val = {
new Val.Arr(pos, splitLimit(pos, str.asString, c.asString, -1))
}
}

private object SplitLimit extends Val.Builtin3("splitLimit", "str", "c", "maxSplits") {
def evalRhs(str: Val, c: Val, maxSplits: Val, ev: EvalScope, pos: Position): Val = {
new Val.Arr(pos, str.asString.split(java.util.regex.Pattern.quote(c.asString), maxSplits.asInt + 1).map(s => Val.Str(pos, s)))
new Val.Arr(pos, splitLimit(pos, str.asString, c.asString, maxSplits.asInt))
}
}

private object SplitLimitR extends Val.Builtin3("splitLimitR", "str", "c", "maxSplits") {
def evalRhs(str: Val, c: Val, maxSplits: Val, ev: EvalScope, pos: Position): Val = {
new Val.Arr(pos, splitLimit(pos, str.asString.reverse, c.asString.reverse, maxSplits.asInt)
.map(s => Val.Str(pos, s.force.asString.reverse)).reverse)
}
}

Expand Down Expand Up @@ -1131,8 +1187,8 @@ private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hi
}
},

"encodeUTF8" -> EncodeUTF8,
"decodeUTF8" -> DecodeUTF8,
builtin(EncodeUTF8),
builtin(DecodeUTF8),

builtinWithDefaults("uniq", "arr" -> null, "keyF" -> Val.False(dummyPos)) { (args, pos, ev) =>
uniqArr(pos, ev, args(0), args(1))
Expand Down Expand Up @@ -1206,15 +1262,16 @@ private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hi
existsInSet(ev, pos, keyF, arr, args(0))
},

"split" -> Split,
"splitLimit" -> SplitLimit,
"stringChars" -> StringChars,
"parseInt" -> ParseInt,
"parseOctal" -> ParseOctal,
"parseHex" -> ParseHex,
"parseJson" -> ParseJson,
"parseYaml" -> ParseYaml,
"md5" -> MD5,
builtin(Split),
builtin(SplitLimit),
builtin(SplitLimitR),
builtin(StringChars),
builtin(ParseInt),
builtin(ParseOctal),
builtin(ParseHex),
builtin(ParseJson),
builtin(ParseYaml),
builtin(MD5),
builtin("prune", "x"){ (pos, ev, s: Val) =>
def filter(x: Val) = x match{
case c: Val.Arr if c.length == 0 => false
Expand Down Expand Up @@ -1248,7 +1305,7 @@ private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hi
str.isEmpty
},
builtin("trim", "str") { (_, _, str: String) =>
str.trim
trailingWhiteSpacePattern.matcher(leadingWhiteSpacePattern.matcher(str).replaceAll("")).replaceAll("")
},
builtin("equalsIgnoreCase", "str1", "str2") { (_, _, str1: String, str2: String) =>
str1.equalsIgnoreCase(str2)
Expand All @@ -1271,6 +1328,64 @@ private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hi
builtin("sha3", "str") { (_, _, str: String) =>
Platform.sha3(str)
},
builtin("sum", "arr") { (_, _, arr: Val.Arr) =>
if (!arr.forall(_.isInstanceOf[Val.Num])) {
Error.fail("Argument must be an array of numbers")
}
arr.asLazyArray.map(_.force.asDouble).sum
},
builtin("avg", "arr") { (_, _, arr: Val.Arr) =>
if (!arr.forall(_.isInstanceOf[Val.Num])) {
Error.fail("Argument must be an array of numbers")
}
if (arr.length == 0) {
Error.fail("Cannot calculate average of an empty array")
}
arr.asLazyArray.map(_.force.asDouble).sum/arr.length
},
builtin("contains", "arr", "elem") { (_, ev, arr: Val.Arr, elem: Val) =>
arr.asLazyArray.indexWhere(s => ev.equal(s.force, elem)) != -1
},
builtin("remove", "arr", "elem") { (_, ev, arr: Val.Arr, elem: Val) =>
val idx = arr.asLazyArray.indexWhere(s => ev.equal(s.force, elem))
if (idx == -1) {
arr
} else {
new Val.Arr(arr.pos, arr.asLazyArray.slice(0, idx) ++ arr.asLazyArray.slice(idx + 1, arr.length))
}
},
builtin("removeAt", "arr", "idx") { (_, _, arr: Val.Arr, idx: Int) =>
if (!(0 <= idx && idx < arr.length)) {
Error.fail("index out of bounds: 0 <= " + idx + " < " + arr.length)
}
new Val.Arr(arr.pos, arr.asLazyArray.slice(0, idx) ++ arr.asLazyArray.slice(idx + 1, arr.length))
},
builtin("objectKeysValues", "o") { (pos, ev, o: Val.Obj) =>
val keys = getVisibleKeys(ev, o)
new Val.Arr(pos, keys.map(k => Val.Obj.mk(
pos.fileScope.noOffsetPos,
"key" -> new Val.Obj.ConstMember(false, Visibility.Normal, Val.Str(pos.fileScope.noOffsetPos, k)),
"value" -> new Val.Obj.ConstMember(false, Visibility.Normal, o.value(k, pos.fileScope.noOffsetPos)(ev))
)))
},
builtin("objectKeysValuesAll", "o") { (pos, ev, o: Val.Obj) =>
val keys = getAllKeys(ev, o)
new Val.Arr(pos, keys.map(k => Val.Obj.mk(
pos.fileScope.noOffsetPos,
"key" -> new Val.Obj.ConstMember(false, Visibility.Normal, Val.Str(pos.fileScope.noOffsetPos, k)),
"value" -> new Val.Obj.ConstMember(false, Visibility.Normal, o.value(k, pos.fileScope.noOffsetPos)(ev))
)))
},
builtin("objectRemoveKey", "obj", "key") { (pos, ev, o: Val.Obj, key: String) =>
val bindings = for{
k <- o.visibleKeyNames
v = o.value(k, pos.fileScope.noOffsetPos)(ev)
if k != key
}yield (k, new Val.Obj.ConstMember(false, Visibility.Normal, v))
Val.Obj.mk(pos, bindings: _*)
},
builtin(MinArray),
builtin(MaxArray),
)

private def toSetArrOrString(args: Array[Val], idx: Int, pos: Position, ev: EvalScope) = {
Expand Down Expand Up @@ -1316,12 +1431,7 @@ private object Get extends Val.Builtin("get", Array("o", "f", "default", "inc_hi
case keyFFunc: Val.Func => keyFFunc.apply1(value, pos.noOffset)(ev)
case _ => value
}
toFind.force match {
case s: Val.Str if appliedValue.isInstanceOf[Val.Str] => Ordering.String.compare(s.asString, appliedValue.force.asString)
case n: Val.Num if appliedValue.isInstanceOf[Val.Num] => java.lang.Double.compare(n.asDouble, appliedValue.force.asDouble)
case t: Val.Bool if appliedValue.isInstanceOf[Val.Bool] => Ordering.Boolean.compare(t.asBoolean, appliedValue.force.asBoolean)
case _ => Error.fail("Cannot perform set operation on " + toFind.force.prettyName + " and " + appliedValue.force.prettyName)
}
ev.compare(toFind.force, appliedValue.force)
}).isInstanceOf[Found]
} else {
arr.exists(value => {
Expand Down
5 changes: 3 additions & 2 deletions sjsonnet/src/sjsonnet/Val.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package sjsonnet

import java.util
import java.util.Arrays

import sjsonnet.Expr.Member.Visibility
import sjsonnet.Expr.Params

Expand Down Expand Up @@ -563,14 +562,16 @@ object Val{
* [[EvalScope]] models the per-evaluator context that is propagated
* throughout the Jsonnet evaluation.
*/
abstract class EvalScope extends EvalErrorScope {
abstract class EvalScope extends EvalErrorScope with Ordering[Val] {
def visitExpr(expr: Expr)
(implicit scope: ValScope): Val

def materialize(v: Val): ujson.Value

def equal(x: Val, y: Val): Boolean

def compare(x: Val, y: Val): Int

val emptyMaterializeFileScope = new FileScope(wd / "(materialize)")
val emptyMaterializeFileScopePos = new Position(emptyMaterializeFileScope, -1)

Expand Down
4 changes: 3 additions & 1 deletion sjsonnet/test/resources/test_suite/arith_float.jsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,9 @@ std.assertEqual(4.5 / 3, 1.5) &&
std.assertEqual(4.5 % 2, 0.5) &&

std.assertEqual(4.5 << 2, 16) &&
std.assertEqual(4.5 << 66, 16) &&
std.assertEqual(4.5 >> 2, 1) &&
std.assertEqual(4.5 >> 66, 1) &&
std.assertEqual(4.5 ^ 3.6, 7) &&
std.assertEqual(5.5 & 3.3, 1) &&
std.assertEqual(4.5 | 1.9, 5) &&
Expand Down Expand Up @@ -134,4 +136,4 @@ local obj = {

std.assertEqual(obj.g, -1) &&

true
true
10 changes: 9 additions & 1 deletion sjsonnet/test/resources/test_suite/object.jsonnet
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,9 @@ std.assertEqual({ ["" + (k + 1)]: (k + 1) for k in [0, 1, 2] }, { ["" + k]: k fo
std.assertEqual({ ["" + k]: k for k in [1, 2, 3] }, { "1": 1, "2": 2, "3": 3 }) &&
std.assertEqual({ [x + ""]: x + foo, local foo = 3 for x in [1, 2, 3] }, { "1": 4, "2": 5, "3": 6 }) &&

// Test for #791
std.assertEqual({ [x]: true for x in ['\\k'] }, { '\\k': true }) &&


local obj = {
f14true: { x: 1, y: 4, z: true },
Expand All @@ -77,6 +80,11 @@ local obj = {

std.assertEqual(obj, { ["f" + x + y + z]: { x: x, y: y, z: z } for x in [1, 2, 3] for y in [1, 4, 6] if x + 2 < y for z in [true, false] }) &&

// Tests for #1111 - object comprehension fields should use "inherit" visibility.
std.assertEqual(std.objectFields({ x: 1 } + { [k]: 2 for k in ['x'] }), ['x']) &&
std.assertEqual(std.objectFields({ x:: 1 } + { [k]: 2 for k in ['x'] }), []) &&
std.assertEqual(std.objectFields({ x::: 1 } + { [k]: 2 for k in ['x'] }), ['x']) &&

std.assertEqual({ f: { foo: 7, bar: 1 } { [self.name]+: 3, name:: "foo" }, name:: "bar" },
{ f: { foo: 7, bar: 4 } }) &&

Expand Down Expand Up @@ -108,4 +116,4 @@ std.assertEqual({ x: 1, a: "x" in self, b: "y" in self }, { x: 1, a: true, b: fa
std.assertEqual({ x:: 1, a: "x" in self, b: "y" in self }, { a: true, b: false }) &&
std.assertEqual({ f: "f" in self }, { f: true }) &&

true
true
Loading

0 comments on commit 6e03591

Please sign in to comment.