From f03364237a2a52d85aa1a098e83ed46f84fb7274 Mon Sep 17 00:00:00 2001 From: Geoffrey Hendrey Date: Sat, 18 Nov 2023 19:30:20 -0800 Subject: [PATCH] Jit (#24) * v0.0.89 * +JIT function support * +DependencyFinder in test * fully show json pointer inference from jit method in test * test++ improve readability? * propagate 'op'- groundwork for supporting function --------- Co-authored-by: Geoffrey Hendrey --- example/animate.yaml | 38 +++++++++++++++++++ package.json | 2 +- src/DependencyFinder.ts | 31 ++++++++++++--- src/TemplateProcessor.ts | 61 ++++++++++++++++++++++-------- src/test/TemplateProcessor.test.js | 57 +++++++++++++++++++++++++++- 5 files changed, 167 insertions(+), 22 deletions(-) create mode 100644 example/animate.yaml diff --git a/example/animate.yaml b/example/animate.yaml new file mode 100644 index 00000000..e74a49d7 --- /dev/null +++ b/example/animate.yaml @@ -0,0 +1,38 @@ +svgson: "${$fetch('https://raw.githubusercontent.com/geoffhendrey/jsonataplay/main/animateLines.json').json()}" +data: { + bandwidth: { + data:{ + replica1: 2000, + replica2: 500 + } + } +} +computeDurationOfAnimation: "${function($bandwidth){$string(1000/$bandwidth) & 's' }}" #produces '2s' for instance +setLineAnimationDuration: | + ${ + function($svg, $lineId, $duration ){ + $svg ~> |$.[children[name='line' + and + attributes[id=$lineId]] + .children[name='animate']] + .attributes + |{'dur':$duration}| /*sets the svg animate tag's 'dur' attribute like dur="0.5s"*/ + } + } +setBandwidthBoxText: | + ${ + function($svg, $id, $bw ){ + $svg ~> |$.children[name='text' and attributes[id=$id]] + .children[type='text'].attributes|{'value':$string($bw)&' Mbps'}| /*sets the text inside the box to like 1000 Mps'*/ + } + } +dur1: "${ computeDurationOfAnimation($$.data.bandwidth.data.replica1)}" +dur2: "${ computeDurationOfAnimation($$.data.bandwidth.data.replica2)}" +content: | + ${ + svgson + ~> setLineAnimationDuration('line1', $$.dur1) + ~> setLineAnimationDuration('line2', $$.dur2) + ~> setBandwidthBoxText('text1', $$.data.bandwidth.data.replica1) + ~> setBandwidthBoxText('text2', $$.data.bandwidth.data.replica2) + } diff --git a/package.json b/package.json index 640e1af6..d3896503 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "stated-js", - "version": "0.0.87", + "version": "0.0.89", "license": "Apache-2.0", "description": "JSONata embedded in JSON", "main": "./dist/src/TemplateProcessor.js", diff --git a/src/DependencyFinder.ts b/src/DependencyFinder.ts index 10304c42..e05c3332 100644 --- a/src/DependencyFinder.ts +++ b/src/DependencyFinder.ts @@ -13,7 +13,7 @@ // limitations under the License. import last from 'lodash/last.js'; -import {default as jsonata} from "jsonata"; +import {default as jsonata, ExprNode} from "jsonata"; /* There are cases where we need to generate some AST nodes the JSONata does not generate. Instead of referring to @@ -39,16 +39,37 @@ export default class DependencyFinder { private readonly currentSteps: StepRecord[][]; //logically, [[a,b,c],[d,e,f]] private nodeStack: GeneratedExprNode[]; private readonly dependencies: string[][]; //during tree walking we collect these dependencies like [["a", "b", "c"], ["foo", "bar"]] which means the dependencies are a.b.c and foo.bar + /** + * program can be either a string to be compiled, or an already-compiled AST + * @param program + */ + constructor(program: string | ExprNode) { + if (typeof program === 'string') { + // Handle the case where program is a string + this.compiledExpression = jsonata(program); + this.ast = this.compiledExpression.ast(); + } else { + this.ast = program; + } - - constructor(program: string) { - this.compiledExpression = jsonata(program); - this.ast = this.compiledExpression.ast(); this.currentSteps = []; this.dependencies = []; this.nodeStack = []; } + /** + * If we are looking to analyze only a portion of the jsonata program we can provide another jsonata expression + * such as '**[procedure.value='serial']' which will filter the AST down to what is defined. In the case of + * '**[procedure.value='serial']' the expression will extract the AST for $serial(...) as it may exist in the + * original program. + * @param jsonatExpr + */ + async withAstFilterExpression(jsonatExpr:string):Promise{ + const filter = jsonata(jsonatExpr); + this.ast = await filter.evaluate(this.ast); + return this; + } + /* Walk the AST of the JSONata program */ diff --git a/src/TemplateProcessor.ts b/src/TemplateProcessor.ts index 45050c22..ee121b96 100644 --- a/src/TemplateProcessor.ts +++ b/src/TemplateProcessor.ts @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import JSONPointer, { default as jp } from './JsonPointer.js'; +import { default as jp } from './JsonPointer.js'; import isEqual from "lodash-es/isEqual.js"; import merge from 'lodash-es/merge.js'; import yaml from 'js-yaml'; @@ -113,6 +113,16 @@ export default class TemplateProcessor { /** Common callback function used within the template processor. */ commonCallback: any; + + /** function generators can be provided by a caller when functions need to be + * created in such a way that they are somehow 'responsive' or dependent on their + * location inside the template. $import is an example of this kind of behavior. + * When $import('http://mytemplate.com/foo.json') is called, the import function + * is actually genrated on the fly, using knowledge of the json path that it was + * called at, to replace the cotnent of the template at that path with the downloaded + * content.*/ + functionGenerators:Map(any)=>any > + private changeCallbacks:Mapvoid>; /** Flag indicating if the template processor is currently initializing. */ @@ -131,6 +141,8 @@ export default class TemplateProcessor { public postInitialize: ()=> Promise; + + public static fromString(template:string, context = {}, options={} ):TemplateProcessor{ let inferredType: "JSON" | "YAML" | "UNKNOWN" = "UNKNOWN"; @@ -177,6 +189,7 @@ export default class TemplateProcessor { this.isInitializing = false; this.tempVars = []; this.changeCallbacks = new Map(); + this.functionGenerators = new Map(); } private setupContext(context: {}) { @@ -754,7 +767,7 @@ export default class TemplateProcessor { this.isEnabled("debug") && this.logger.debug(`setData on ${jsonPtr} for TemplateProcessor uid=${this.uniqueId}`) //get all the jsonPtrs we need to update, including this one, to percolate the change const sortedJsonPtrs = [...this.from(jsonPtr)]; //defensive copy - const plan = {sortedJsonPtrs, data}; + const plan = {sortedJsonPtrs, data, op:'set'}; this.executionQueue.push(plan); if(this.isEnabled("debug")) { this.logger.debug(`execution plan (uid=${this.uniqueId}): ${JSON.stringify(plan)}`); @@ -766,8 +779,8 @@ export default class TemplateProcessor { async function drainQueue() { while (this.executionQueue.length > 0) { - const {sortedJsonPtrs, data} = this.executionQueue[0]; - await this.evaluateJsonPointersInOrder(sortedJsonPtrs, data); + const {sortedJsonPtrs, data, op} = this.executionQueue[0]; + await this.evaluateJsonPointersInOrder(sortedJsonPtrs, data, op); this.executionQueue.shift(); } } @@ -790,13 +803,13 @@ export default class TemplateProcessor { } } - private async evaluateJsonPointersInOrder(jsonPtrList, data = TemplateProcessor.NOOP) { + private async evaluateJsonPointersInOrder(jsonPtrList, data = TemplateProcessor.NOOP, op:"set"|"delete"="set") { const resp = []; let first; if (data !== TemplateProcessor.NOOP) { first = jsonPtrList.shift(); //first jsonPtr is the target of the change, the rest are dependents if (!jp.has(this.output, first)) { //node doesn't exist yet, so just create it - const didUpdate = await this.evaluateNode(first, data); + const didUpdate = await this.evaluateNode(first, data, op); jp.get(this.templateMeta, first).didUpdate__ = didUpdate; } else { // Check if the node contains an expression. If so, print a warning and return. @@ -805,7 +818,7 @@ export default class TemplateProcessor { this.logger.log('warn', `Attempted to replace expressions with data under ${first}. This operation is ignored.`); return false; //fixme - although not used, returning false here is inconsistent. we need to return [firstMeta] } - firstMeta.didUpdate__ = await this.evaluateNode(first, data); // Evaluate the node provided with the data provided + firstMeta.didUpdate__ = await this.evaluateNode(first, data, op); // Evaluate the node provided with the data provided if (!firstMeta.didUpdate__) { this.logger.verbose(`data did not change for ${first}, short circuiting dependents.`); return false; @@ -832,7 +845,7 @@ export default class TemplateProcessor { return thoseThatUpdated; } - private async evaluateNode(jsonPtr, data?) { + private async evaluateNode(jsonPtr, data=undefined, op:"set"|"delete"="set") { const {output, templateMeta} = this; //an untracked json pointer is one that we have no metadata about. It's just a request out of the blue to @@ -844,7 +857,7 @@ export default class TemplateProcessor { const hasDataToSet = data !== undefined && data !== TemplateProcessor.NOOP; if (hasDataToSet) { - return this.setDataIntoTrackedLocation(templateMeta, jsonPtr, data); + return this.setDataIntoTrackedLocation(templateMeta, jsonPtr, data, op); } return this._evaluateExpression(jsonPtr); @@ -908,13 +921,13 @@ export default class TemplateProcessor { } } - private setDataIntoTrackedLocation(templateMeta, jsonPtr, data) { + private setDataIntoTrackedLocation(templateMeta, jsonPtr, data=undefined,op:"set"|"delete"="set" ) { const {treeHasExpressions__} = jp.get(templateMeta, jsonPtr); if (treeHasExpressions__) { this.logger.log('warn', `nodes containing expressions cannot be overwritten: ${jsonPtr}`); return false; } - let didSet = this._setData(jsonPtr, data); + let didSet = this._setData(jsonPtr, data, op); if (didSet) { jp.set(templateMeta, jsonPtr + "/data__", data); //saving the data__ in the templateMeta is just for debugging jp.set(templateMeta, jsonPtr + "/materialized__", true); @@ -939,14 +952,25 @@ export default class TemplateProcessor { private async _evaluateExprNode(jsonPtr) { let evaluated; - const {compiledExpr__, exprTargetJsonPointer__, jsonPointer__, expr__} = jp.get(this.templateMeta, jsonPtr); + const metaInfo = jp.get(this.templateMeta, jsonPtr); + const {compiledExpr__, exprTargetJsonPointer__, jsonPointer__, expr__} = metaInfo; let target; try { target = jp.get(this.output, exprTargetJsonPointer__); //an expression is always relative to a target const safe = this.withErrorHandling.bind(this); + const jittedFunctions = {}; + for (const k of this.functionGenerators.keys()) { + const generator = this.functionGenerators.get(k); + jittedFunctions[k] = safe(generator(metaInfo, this)); + } + evaluated = await compiledExpr__.evaluate( target, - merge(this.context, {"import": safe(this.getImport(jsonPointer__))})); + {...this.context, + ...{"import": safe(this.getImport(jsonPointer__))}, + ...jittedFunctions + } + ); } catch (error) { this.logger.error(`Error evaluating expression at ${jsonPtr}`); this.logger.error(error); @@ -968,11 +992,18 @@ export default class TemplateProcessor { return Array.from(tagSetOnTheExpression).every(tag => this.tagSet.has(tag)); } - private _setData(jsonPtr, data) { + private _setData(jsonPtr:JsonPointerString, data:any=undefined, op:"set"|"delete" ="set"):boolean { if (data === TemplateProcessor.NOOP) { //a No-Op is used as the return from 'import' where we don't actually need to make the assignment as init has already dont it return false; } const {output} = this; + if(op === 'delete'){ + if(jp.has(output, jsonPtr)) { + jp.remove(output, jsonPtr); + return true; + } + return false; + } let existingData; if (jp.has(output, jsonPtr)) { existingData = jp.get(output, jsonPtr); @@ -989,7 +1020,7 @@ export default class TemplateProcessor { } } -// getDependentsTransitiveExecutionPlan(jsonPtr) { + from(jsonPtr) { //check execution plan cache if (this.executionPlans[jsonPtr] === undefined) { diff --git a/src/test/TemplateProcessor.test.js b/src/test/TemplateProcessor.test.js index 22b06b10..fb89a22d 100644 --- a/src/test/TemplateProcessor.test.js +++ b/src/test/TemplateProcessor.test.js @@ -19,8 +19,9 @@ import fs from 'fs'; import yaml from 'js-yaml'; import {fileURLToPath} from 'url'; import {dirname} from 'path'; -import MetaInfoProducer from "../../dist/src/MetaInfoProducer.js"; +import DependencyFinder from "../../dist/src/DependencyFinder.js"; import jsonata from "jsonata"; +import { default as jp } from "../../dist/src/JsonPointer.js"; test("test 1", async () => { @@ -1572,6 +1573,60 @@ test("parallel TemplateProcessors", () => { }); }); +test("function generators",async () => { + let template = { + a: "${ $jit() }", + b: "${ $jit() }", + d:{e:"${ $jit() }"}, + e: "${ (koink; $serial([foo.zing.zap,bar,baz]))}" + }; + + let tp = new TemplateProcessor(template); + const jit = (metaInf, tp)=>{ + return ()=>{ + return `path was: ${metaInf.jsonPointer__}`; + } + } + tp.functionGenerators.set("jit", jit); + let serialDeps; + const serial = (metaInf, tp)=>{ + return async (input, steps, context)=>{ + const ast = metaInf.compiledExpr__.ast(); + let depFinder = new DependencyFinder(ast); + depFinder = await depFinder.withAstFilterExpression("**[procedure.value='serial']"); + //this is just an example of how we can find the dependencies of $serial([foo, bar]) and cache them for later use + serialDeps = depFinder.findDependencies(); + return "nothing to see here" + } + } + tp.functionGenerators.set('serial', serial); + await tp.initialize(); + expect(tp.output).toStrictEqual({ + "a": "path was: /a", + "b": "path was: /b", + "d": { + "e": "path was: /d/e" + }, + "e": "nothing to see here" + }); + expect(serialDeps).toStrictEqual([ + [ + "foo", + "zing", + "zap" + ], + [ + "bar" + ], + [ + "baz" + ] + ]); + const jsonPointers = serialDeps.map(jp.compile); + expect(jsonPointers).toStrictEqual(["/foo/zing/zap", "/bar", "/baz"]); + +}); +