Jit (#24)

* v0.0.89 * +JIT function support * +DependencyFinder in test * fully show json pointer inference from jit method in test * test++ improve readability? * propagate 'op'- groundwork for supporting function --------- Co-authored-by: Geoffrey Hendrey <[email protected]>
cisco-open · Nov 19, 2023 · f033642 · f033642
1 parent 0136d9f
commit f033642
Show file tree

Hide file tree

Showing 5 changed files with 167 additions and 22 deletions.
diff --git a/example/animate.yaml b/example/animate.yaml
@@ -0,0 +1,38 @@
+svgson: "${$fetch('https://raw.githubusercontent.com/geoffhendrey/jsonataplay/main/animateLines.json').json()}"
+data: {
+  bandwidth: {
+    data:{
+      replica1: 2000,
+      replica2: 500
+    }
+  }
+}
+computeDurationOfAnimation: "${function($bandwidth){$string(1000/$bandwidth) & 's' }}" #produces '2s' for instance
+setLineAnimationDuration: |
+  ${ 
+    function($svg, $lineId, $duration ){ 
+      $svg ~> |$.[children[name='line' 
+                           and 
+                           attributes[id=$lineId]]
+              .children[name='animate']]
+              .attributes
+              |{'dur':$duration}|  /*sets the svg animate tag's 'dur' attribute like dur="0.5s"*/
+    } 
+  }
+setBandwidthBoxText: |
+  ${ 
+    function($svg, $id, $bw ){ 
+      $svg ~> |$.children[name='text' and attributes[id=$id]]
+               .children[type='text'].attributes|{'value':$string($bw)&' Mbps'}|  /*sets the text inside the box to like 1000 Mps'*/
+    } 
+  }
+dur1: "${ computeDurationOfAnimation($$.data.bandwidth.data.replica1)}"
+dur2: "${ computeDurationOfAnimation($$.data.bandwidth.data.replica2)}"
+content: |
+  ${
+     svgson
+     ~> setLineAnimationDuration('line1', $$.dur1)
+     ~> setLineAnimationDuration('line2', $$.dur2)
+     ~> setBandwidthBoxText('text1', $$.data.bandwidth.data.replica1)
+     ~> setBandwidthBoxText('text2', $$.data.bandwidth.data.replica2)
+  }
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "stated-js",
-  "version": "0.0.87",
+  "version": "0.0.89",
   "license": "Apache-2.0",
   "description": "JSONata embedded in JSON",
   "main": "./dist/src/TemplateProcessor.js",

diff --git a/src/DependencyFinder.ts b/src/DependencyFinder.ts
@@ -13,7 +13,7 @@
 // limitations under the License.
 
 import last from 'lodash/last.js';
-import {default as jsonata} from "jsonata";
+import {default as jsonata, ExprNode} from "jsonata";
 
 /*
   There are cases where we need to generate some AST nodes the JSONata does not generate. Instead of referring to
@@ -39,16 +39,37 @@ export default class DependencyFinder {
     private readonly currentSteps: StepRecord[][]; //logically, [[a,b,c],[d,e,f]]
     private nodeStack: GeneratedExprNode[];
     private readonly dependencies: string[][]; //during tree walking we collect these dependencies like [["a", "b", "c"], ["foo", "bar"]] which means the dependencies are a.b.c and foo.bar
+    /**
+     * program can be either a string to be compiled, or an already-compiled AST
+     * @param program
+     */
+    constructor(program: string | ExprNode) {
+        if (typeof program === 'string') {
+            // Handle the case where program is a string
+            this.compiledExpression = jsonata(program);
+            this.ast = this.compiledExpression.ast();
+        } else {
+            this.ast = program;
+        }
 
-
-    constructor(program: string) {
-        this.compiledExpression = jsonata(program);
-        this.ast = this.compiledExpression.ast();
         this.currentSteps = [];
         this.dependencies = [];
         this.nodeStack = [];
     }
 
+    /**
+     * If we are looking to analyze only a portion of the jsonata program we can provide another jsonata expression
+     * such as '**[procedure.value='serial']' which will filter the AST down to what is defined. In the case of
+     * '**[procedure.value='serial']' the expression will extract the AST for $serial(...) as it may exist in the
+     * original program.
+     * @param jsonatExpr
+     */
+    async withAstFilterExpression(jsonatExpr:string):Promise<DependencyFinder>{
+        const filter = jsonata(jsonatExpr);
+        this.ast = await filter.evaluate(this.ast);
+        return this;
+    }
+
     /*
         Walk the AST of the JSONata program
      */

diff --git a/src/TemplateProcessor.ts b/src/TemplateProcessor.ts
@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-import JSONPointer, { default as jp } from './JsonPointer.js';
+import { default as jp } from './JsonPointer.js';
 import isEqual from "lodash-es/isEqual.js";
 import merge from 'lodash-es/merge.js';
 import yaml from 'js-yaml';
@@ -113,6 +113,16 @@ export default class TemplateProcessor {
 
     /** Common callback function used within the template processor. */
     commonCallback: any;
+
+    /** function generators can be provided by a caller when functions need to be
+     *  created in such a way that they are somehow 'responsive' or dependent on their
+     *  location inside the template. $import is an example of this kind of behavior.
+     *  When $import('http://mytemplate.com/foo.json') is called, the import function
+     *  is actually genrated on the fly, using knowledge of the json path that it was
+     *  called at, to replace the cotnent of the template at that path with the downloaded
+     *  content.*/
+    functionGenerators:Map<string, (MetaInfo,TemplateProcessor)=>(any)=>any >
+
     private changeCallbacks:Map<JsonPointerString, (data:any, jsonPointer: JsonPointerString, removed:boolean)=>void>;
 
     /** Flag indicating if the template processor is currently initializing. */
@@ -131,6 +141,8 @@ export default class TemplateProcessor {
     public postInitialize: ()=> Promise<void>;
 
 
+
+
     public static fromString(template:string, context = {}, options={} ):TemplateProcessor{
             let inferredType: "JSON" | "YAML" | "UNKNOWN" = "UNKNOWN";
 
@@ -177,6 +189,7 @@ export default class TemplateProcessor {
         this.isInitializing = false;
         this.tempVars = [];
         this.changeCallbacks = new Map();
+        this.functionGenerators = new Map();
     }
 
     private setupContext(context: {}) {
@@ -754,7 +767,7 @@ export default class TemplateProcessor {
         this.isEnabled("debug") && this.logger.debug(`setData on ${jsonPtr} for TemplateProcessor uid=${this.uniqueId}`)
         //get all the jsonPtrs we need to update, including this one, to percolate the change
         const sortedJsonPtrs = [...this.from(jsonPtr)]; //defensive copy
-        const plan = {sortedJsonPtrs, data};
+        const plan = {sortedJsonPtrs, data, op:'set'};
         this.executionQueue.push(plan);
         if(this.isEnabled("debug")) {
             this.logger.debug(`execution plan (uid=${this.uniqueId}): ${JSON.stringify(plan)}`);
@@ -766,8 +779,8 @@ export default class TemplateProcessor {
 
         async function drainQueue() {
             while (this.executionQueue.length > 0) {
-                const {sortedJsonPtrs, data} = this.executionQueue[0];
-                await this.evaluateJsonPointersInOrder(sortedJsonPtrs, data);
+                const {sortedJsonPtrs, data, op} = this.executionQueue[0];
+                await this.evaluateJsonPointersInOrder(sortedJsonPtrs, data, op);
                 this.executionQueue.shift();
             }
         }
@@ -790,13 +803,13 @@ export default class TemplateProcessor {
         }
     }
 
-    private async evaluateJsonPointersInOrder(jsonPtrList, data = TemplateProcessor.NOOP) {
+    private async evaluateJsonPointersInOrder(jsonPtrList, data = TemplateProcessor.NOOP, op:"set"|"delete"="set") {
         const resp = [];
         let first;
         if (data !== TemplateProcessor.NOOP) {
             first = jsonPtrList.shift(); //first jsonPtr is the target of the change, the rest are dependents
             if (!jp.has(this.output, first)) { //node doesn't exist yet, so just create it
-                const didUpdate = await this.evaluateNode(first, data);
+                const didUpdate = await this.evaluateNode(first, data, op);
                 jp.get(this.templateMeta, first).didUpdate__ = didUpdate;
             } else {
                 // Check if the node contains an expression. If so, print a warning and return.
@@ -805,7 +818,7 @@ export default class TemplateProcessor {
                     this.logger.log('warn', `Attempted to replace expressions with data under ${first}. This operation is ignored.`);
                     return false; //fixme - although not used, returning false here is inconsistent. we need to return [firstMeta]
                 }
-                firstMeta.didUpdate__ = await this.evaluateNode(first, data); // Evaluate the node provided with the data provided
+                firstMeta.didUpdate__ = await this.evaluateNode(first, data, op); // Evaluate the node provided with the data provided
                 if (!firstMeta.didUpdate__) {
                     this.logger.verbose(`data did not change for ${first}, short circuiting dependents.`);
                     return false;
@@ -832,7 +845,7 @@ export default class TemplateProcessor {
         return thoseThatUpdated;
     }
 
-    private async evaluateNode(jsonPtr, data?) {
+    private async evaluateNode(jsonPtr, data=undefined, op:"set"|"delete"="set") {
         const {output, templateMeta} = this;
 
         //an untracked json pointer is one that we have no metadata about. It's just a request out of the blue to
@@ -844,7 +857,7 @@ export default class TemplateProcessor {
 
         const hasDataToSet = data !== undefined && data !== TemplateProcessor.NOOP;
         if (hasDataToSet) {
-            return this.setDataIntoTrackedLocation(templateMeta, jsonPtr, data);
+            return this.setDataIntoTrackedLocation(templateMeta, jsonPtr, data, op);
         }
 
         return this._evaluateExpression(jsonPtr);
@@ -908,13 +921,13 @@ export default class TemplateProcessor {
         }
     }
 
-    private setDataIntoTrackedLocation(templateMeta, jsonPtr, data) {
+    private setDataIntoTrackedLocation(templateMeta, jsonPtr, data=undefined,op:"set"|"delete"="set" ) {
         const {treeHasExpressions__} = jp.get(templateMeta, jsonPtr);
         if (treeHasExpressions__) {
             this.logger.log('warn', `nodes containing expressions cannot be overwritten: ${jsonPtr}`);
             return false;
         }
-        let didSet = this._setData(jsonPtr, data);
+        let didSet = this._setData(jsonPtr, data, op);
         if (didSet) {
             jp.set(templateMeta, jsonPtr + "/data__", data); //saving the data__ in the templateMeta is just for debugging
             jp.set(templateMeta, jsonPtr + "/materialized__", true);
@@ -939,14 +952,25 @@ export default class TemplateProcessor {
 
     private async _evaluateExprNode(jsonPtr) {
         let evaluated;
-        const {compiledExpr__, exprTargetJsonPointer__, jsonPointer__, expr__} = jp.get(this.templateMeta, jsonPtr);
+        const metaInfo = jp.get(this.templateMeta, jsonPtr);
+        const {compiledExpr__, exprTargetJsonPointer__, jsonPointer__, expr__} = metaInfo;
         let target;
         try {
             target = jp.get(this.output, exprTargetJsonPointer__); //an expression is always relative to a target
             const safe =  this.withErrorHandling.bind(this);
+            const jittedFunctions = {};
+            for (const k of this.functionGenerators.keys()) {
+                const generator = this.functionGenerators.get(k);
+                jittedFunctions[k] = safe(generator(metaInfo, this));
+            }
+
             evaluated = await compiledExpr__.evaluate(
                 target,
-                merge(this.context, {"import": safe(this.getImport(jsonPointer__))}));
+                {...this.context,
+                    ...{"import": safe(this.getImport(jsonPointer__))},
+                    ...jittedFunctions
+                }
+            );
         } catch (error) {
             this.logger.error(`Error evaluating expression at ${jsonPtr}`);
             this.logger.error(error);
@@ -968,11 +992,18 @@ export default class TemplateProcessor {
         return Array.from(tagSetOnTheExpression).every(tag => this.tagSet.has(tag));
     }
 
-    private _setData(jsonPtr, data) {
+    private _setData(jsonPtr:JsonPointerString, data:any=undefined, op:"set"|"delete" ="set"):boolean {
         if (data === TemplateProcessor.NOOP) { //a No-Op is used as the return from 'import' where we don't actually need to make the assignment as init has already dont it
             return false;
         }
         const {output} = this;
+        if(op === 'delete'){
+            if(jp.has(output, jsonPtr)) {
+                jp.remove(output, jsonPtr);
+                return true;
+            }
+            return false;
+        }
         let existingData;
         if (jp.has(output, jsonPtr)) {
             existingData = jp.get(output, jsonPtr);
@@ -989,7 +1020,7 @@ export default class TemplateProcessor {
         }
 
     }
-//    getDependentsTransitiveExecutionPlan(jsonPtr) {
+
     from(jsonPtr) {
         //check execution plan cache
         if (this.executionPlans[jsonPtr] === undefined) {

diff --git a/src/test/TemplateProcessor.test.js b/src/test/TemplateProcessor.test.js
@@ -19,8 +19,9 @@ import fs from 'fs';
 import yaml from 'js-yaml';
 import {fileURLToPath} from 'url';
 import {dirname} from 'path';
-import MetaInfoProducer from "../../dist/src/MetaInfoProducer.js";
+import DependencyFinder from "../../dist/src/DependencyFinder.js";
 import jsonata from "jsonata";
+import { default as jp } from "../../dist/src/JsonPointer.js";
 
 
 test("test 1", async () => {
@@ -1572,6 +1573,60 @@ test("parallel TemplateProcessors", () => {
     });
 });
 
+test("function generators",async () => {
+    let template = {
+        a: "${ $jit() }",
+        b: "${ $jit() }",
+        d:{e:"${ $jit() }"},
+        e: "${ (koink; $serial([foo.zing.zap,bar,baz]))}"
+    };
+
+    let tp = new TemplateProcessor(template);
+    const jit = (metaInf, tp)=>{
+        return ()=>{
+            return `path was: ${metaInf.jsonPointer__}`;
+        }
+    }
+    tp.functionGenerators.set("jit", jit);
+    let serialDeps;
+    const serial = (metaInf, tp)=>{
+        return async (input, steps, context)=>{
+            const ast = metaInf.compiledExpr__.ast();
+            let depFinder = new DependencyFinder(ast);
+            depFinder = await depFinder.withAstFilterExpression("**[procedure.value='serial']");
+            //this is just an example of how we can find the dependencies of $serial([foo, bar]) and cache them for later use
+            serialDeps =  depFinder.findDependencies();
+            return "nothing to see here"
+        }
+    }
+    tp.functionGenerators.set('serial', serial);
+    await tp.initialize();
+    expect(tp.output).toStrictEqual({
+        "a": "path was: /a",
+        "b": "path was: /b",
+        "d": {
+            "e": "path was: /d/e"
+        },
+        "e": "nothing to see here"
+    });
+    expect(serialDeps).toStrictEqual([
+        [
+            "foo",
+            "zing",
+            "zap"
+        ],
+        [
+            "bar"
+        ],
+        [
+            "baz"
+        ]
+    ]);
+    const jsonPointers = serialDeps.map(jp.compile);
+    expect(jsonPointers).toStrictEqual(["/foo/zing/zap", "/bar", "/baz"]);
+
+});
+