From 8085971dbdb0575a3e858dcaa04cd51e761b387d Mon Sep 17 00:00:00 2001 From: Kamil Figiela Date: Tue, 28 Jan 2014 13:28:13 +0100 Subject: [PATCH 01/73] Add AMQP executor configuration in separate file --- functions/amqpCommand.config.js | 26 ++++++++++++++++++++++++++ functions/amqpCommand.js | 12 +++--------- 2 files changed, 29 insertions(+), 9 deletions(-) create mode 100644 functions/amqpCommand.config.js diff --git a/functions/amqpCommand.config.js b/functions/amqpCommand.config.js new file mode 100644 index 0000000..bf9abad --- /dev/null +++ b/functions/amqpCommand.config.js @@ -0,0 +1,26 @@ +var AMQP_URL = process.env.AMQP_URL ? process.env.AMQP_URL : "amqp://localhost:5672"; +var S3_BUCKET = process.env.S3_BUCKET; +var S3_PATH = process.env.S3_PATH; + +exports.amqp_url = AMQP_URL; + +// S3 storage +exports.options = { + "storage": "s3", + "bucket": S3_BUCKET, + "prefix": S3_PATH +}; + + +// NFS storage +// exports.options = { +// "storage": "nfs", +// "workdir": "/path/where/workflow/data/is", +// } + + +// Local storage +// exports.options = { +// "storage": "local" +// "workdir": "/path/where/workflow/data/is", +// } \ No newline at end of file diff --git a/functions/amqpCommand.js b/functions/amqpCommand.js index 3f533bf..7f55302 100644 --- a/functions/amqpCommand.js +++ b/functions/amqpCommand.js @@ -3,13 +3,10 @@ var uuid = require('uuid'); var when = require('when'); var defer = when.defer; var util = require('util'); - -var AMQP_URL = process.env.AMQP_URL ? process.env.AMQP_URL : "amqp://localhost:5672"; -var S3_BUCKET = process.env.S3_BUCKET; -var S3_PATH = process.env.S3_PATH; +var executor_config = require('./amqpCommand.config.js'); console.log("[AMQP] Starting connection!"); -var connection = require('amqplib').connect(AMQP_URL); +var connection = require('amqplib').connect(executor_config.amqp_url); var connectionReady = false; connection.then(function(conn) { @@ -24,10 +21,7 @@ function amqpCommand(ins, outs, config, cb) { "args": config.executor.args, "inputs": ins, "outputs": outs, - "options": { - "bucket": S3_BUCKET, - "prefix": S3_PATH, - } + "options": executor_config.options }; var answer = defer(); var corrId = uuid.v4(); From ed98b05bc08253e4a55611b80afd77f57f026ff8 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 29 Jan 2014 09:54:23 +0100 Subject: [PATCH 02/73] Minor cleanup. --- app.js | 21 --------------------- wflib/index.js | 4 ++-- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/app.js b/app.js index eb7a2a4..2f58174 100644 --- a/app.js +++ b/app.js @@ -19,27 +19,6 @@ var redis = require('redis'), rcl = redis.createClient(); var server = http.createServer(app); - -// for couch -var cradle = require('cradle'); -var host = 'http://beboj.iriscouch.com'; -var port = 5984; -var credentials = { - username: 'balis', - password: 'ala123' -}; -var local = false; -var db; -if (local === true) { - db = new(cradle.Connection)().database('hyperwf'); -} -else { - db = new(cradle.Connection)(host, port, { - auth: credentials - }).database('hyperwf'); -} - -var executor = require('./executor_simple').init(); var wflib = require('./wflib').init(rcl); var Engine = require('./engine'); var engine = {}; // engine.i contains the engine object for workflow instance 'i' diff --git a/wflib/index.js b/wflib/index.js index 24e941b..c3a5507 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -860,9 +860,9 @@ exports.init = function(redisClient) { // result: true if 'count' instances of each signal 'sigId' are present in the queues // sigValues (optional) format: [ [ spec[0] signal values ], [ spec[1] signal values ], ... ] // example: [ [ { name: 'filename', - // uri: '/workflow/Wf_continuous_file_splitter/instances/1/data-1', + // uri: '/apps/1/sigs/1', // _id: '1', - // _ts: 8 + // _ts: 415334, // data: [ { path: 'tmp1/asynctest.js' } ] // } ] ] // From 17a0dfff593f071e1bec8b7bdf76ad4b441194d4 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 11 Feb 2014 12:21:16 +0100 Subject: [PATCH 03/73] Remove old amqp dependency. --- functions/amqpCommand.js | 4 ++-- package.json | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/functions/amqpCommand.js b/functions/amqpCommand.js index 3f533bf..320d5ea 100644 --- a/functions/amqpCommand.js +++ b/functions/amqpCommand.js @@ -1,7 +1,7 @@ -var amqp = require('amqp'); var uuid = require('uuid'); var when = require('when'); var defer = when.defer; +var amqplib = require('amqplib'); var util = require('util'); var AMQP_URL = process.env.AMQP_URL ? process.env.AMQP_URL : "amqp://localhost:5672"; @@ -9,7 +9,7 @@ var S3_BUCKET = process.env.S3_BUCKET; var S3_PATH = process.env.S3_PATH; console.log("[AMQP] Starting connection!"); -var connection = require('amqplib').connect(AMQP_URL); +var connection = amqplib.connect(AMQP_URL); var connectionReady = false; connection.then(function(conn) { diff --git a/package.json b/package.json index 0ee6468..6dd2388 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,6 @@ "underscore":"1.x", "uuid":"", "amqplib":"", - "amqp":"", "when":"", "optimist":"0.6.0", "walkdir":"0.0.7", From 54334fcb5c29de4bd97d9075e5618ac7ff777dce Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 11 Feb 2014 18:34:23 +0100 Subject: [PATCH 04/73] Added base for command with event notifications. --- engine/index.js | 4 +- functions/command.js | 11 + functions/index.js | 1 + package.json | 1 + workflows/Montage_143_events.json | 4539 +++++++++++++++++++++++++++++ 5 files changed, 4555 insertions(+), 1 deletion(-) create mode 100644 workflows/Montage_143_events.json diff --git a/engine/index.js b/engine/index.js index 14e29c1..f2117f9 100644 --- a/engine/index.js +++ b/engine/index.js @@ -15,7 +15,8 @@ var fs = require('fs'), xml2js = require('xml2js'), fsm = require('./automata.js'), - async = require('async'); + async = require('async'), + EventEmitter2 = require('eventemitter2').EventEmitter2; var TaskDataflowFSM = require('./taskDataflowFSM.js'); var TaskForeachFSM = require('./taskForeachFSM.js'); @@ -58,6 +59,7 @@ fsm.registerFSM(TaskChoiceFSM); // - config.emulate (true/false) = should engine work in the emulation mode? var Engine = function(config, wflib, wfId, cb) { this.wflib = wflib; + this.eventServer = new EventEmitter2({wildcard:true}); this.wfId = wfId; this.tasks = []; // array of task FSMs this.ins = []; diff --git a/functions/command.js b/functions/command.js index 28276ff..bfd54a5 100644 --- a/functions/command.js +++ b/functions/command.js @@ -35,5 +35,16 @@ function command_print(ins, outs, config, cb) { cb(null, outs); } +function command_notifyevents(ins, outs, config, cb) { + var exec = config.executor.executable, + args = config.executor.args; + + console.log(exec, args); + + cb(null, outs); +} + + exports.command = command; exports.command_print = command_print; +exports.command_notifyevents = command_notifyevents; diff --git a/functions/index.js b/functions/index.js index ddc51ca..d5376ee 100644 --- a/functions/index.js +++ b/functions/index.js @@ -166,6 +166,7 @@ exports.command = cmd.command; exports.amqpCommand = amqpCmd.amqpCommand; exports.exit = exit; exports.command_print = cmd.command_print; +exports.command_notifyevents = cmd.command_notifyevents; exports.scanDirForJs = scanDirForJs; exports.grepFile = grepFile; exports.chooseEvenOdd = chooseEvenOdd; diff --git a/package.json b/package.json index 6dd2388..a6d22ea 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "uuid":"", "amqplib":"", "when":"", + "eventemitter2":"", "optimist":"0.6.0", "walkdir":"0.0.7", "path":"0.4.9", diff --git a/workflows/Montage_143_events.json b/workflows/Montage_143_events.json new file mode 100644 index 0000000..c29d0ad --- /dev/null +++ b/workflows/Montage_143_events.json @@ -0,0 +1,4539 @@ +{ + "functions": [ + { + "name": "command_notifyevents", + "module": "functions" + } + ], + "tasks": [ + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99385 2mass-atlas-990502s-j1450080.fits p2mass-atlas-990502s-j1450080.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 0, + 3 + ], + "outs": [ + 1, + 2 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99366 2mass-atlas-990502s-j1440198.fits p2mass-atlas-990502s-j1440198.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 4, + 3 + ], + "outs": [ + 5, + 6 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99431 2mass-atlas-990502s-j1460198.fits p2mass-atlas-990502s-j1460198.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 7, + 3 + ], + "outs": [ + 8, + 9 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99440 2mass-atlas-990502s-j1460186.fits p2mass-atlas-990502s-j1460186.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 10, + 3 + ], + "outs": [ + 11, + 12 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99321 2mass-atlas-990502s-j1420209.fits p2mass-atlas-990502s-j1420209.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 13, + 3 + ], + "outs": [ + 14, + 15 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99302 2mass-atlas-990502s-j1430068.fits p2mass-atlas-990502s-j1430068.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 16, + 3 + ], + "outs": [ + 17, + 18 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99421 2mass-atlas-990502s-j1460209.fits p2mass-atlas-990502s-j1460209.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 19, + 3 + ], + "outs": [ + 20, + 21 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99366 2mass-atlas-990502s-j1450068.fits p2mass-atlas-990502s-j1450068.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 22, + 3 + ], + "outs": [ + 23, + 24 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99357 2mass-atlas-990502s-j1440209.fits p2mass-atlas-990502s-j1440209.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 25, + 3 + ], + "outs": [ + 26, + 27 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98664 2mass-atlas-990502s-j1330103.fits p2mass-atlas-990502s-j1330103.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 28, + 3 + ], + "outs": [ + 29, + 30 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99074 2mass-atlas-990502s-j1350103.fits p2mass-atlas-990502s-j1350103.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 31, + 3 + ], + "outs": [ + 32, + 33 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98873 2mass-atlas-990502s-j1340174.fits p2mass-atlas-990502s-j1340174.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 34, + 3 + ], + "outs": [ + 35, + 36 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98646 2mass-atlas-990502s-j1330092.fits p2mass-atlas-990502s-j1330092.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 37, + 3 + ], + "outs": [ + 38, + 39 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98846 2mass-atlas-990502s-j1340209.fits p2mass-atlas-990502s-j1340209.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 40, + 3 + ], + "outs": [ + 41, + 42 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99028 2mass-atlas-990502s-j1350068.fits p2mass-atlas-990502s-j1350068.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 43, + 3 + ], + "outs": [ + 44, + 45 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98619 2mass-atlas-990502s-j1330068.fits p2mass-atlas-990502s-j1330068.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 46, + 3 + ], + "outs": [ + 47, + 48 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99458 2mass-atlas-990502s-j1460174.fits p2mass-atlas-990502s-j1460174.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 49, + 3 + ], + "outs": [ + 50, + 51 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99339 2mass-atlas-990502s-j1430103.fits p2mass-atlas-990502s-j1430103.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 52, + 3 + ], + "outs": [ + 53, + 54 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99376 2mass-atlas-990502s-j1440186.fits p2mass-atlas-990502s-j1440186.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 55, + 3 + ], + "outs": [ + 56, + 57 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99421 2mass-atlas-990502s-j1450103.fits p2mass-atlas-990502s-j1450103.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 58, + 3 + ], + "outs": [ + 59, + 60 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99385 2mass-atlas-990502s-j1440174.fits p2mass-atlas-990502s-j1440174.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 61, + 3 + ], + "outs": [ + 62, + 63 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99330 2mass-atlas-990502s-j1430092.fits p2mass-atlas-990502s-j1430092.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 64, + 3 + ], + "outs": [ + 65, + 66 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99403 2mass-atlas-990502s-j1450092.fits p2mass-atlas-990502s-j1450092.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 67, + 3 + ], + "outs": [ + 68, + 69 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98637 2mass-atlas-990502s-j1330080.fits p2mass-atlas-990502s-j1330080.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 70, + 3 + ], + "outs": [ + 71, + 72 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99330 2mass-atlas-990502s-j1420198.fits p2mass-atlas-990502s-j1420198.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 73, + 3 + ], + "outs": [ + 74, + 75 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98864 2mass-atlas-990502s-j1340186.fits p2mass-atlas-990502s-j1340186.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 76, + 3 + ], + "outs": [ + 77, + 78 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99339 2mass-atlas-990502s-j1420186.fits p2mass-atlas-990502s-j1420186.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 79, + 3 + ], + "outs": [ + 80, + 81 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99056 2mass-atlas-990502s-j1350092.fits p2mass-atlas-990502s-j1350092.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 82, + 3 + ], + "outs": [ + 83, + 84 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99321 2mass-atlas-990502s-j1430080.fits p2mass-atlas-990502s-j1430080.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 85, + 3 + ], + "outs": [ + 86, + 87 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99047 2mass-atlas-990502s-j1350080.fits p2mass-atlas-990502s-j1350080.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 88, + 3 + ], + "outs": [ + 89, + 90 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.99348 2mass-atlas-990502s-j1420174.fits p2mass-atlas-990502s-j1420174.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 91, + 3 + ], + "outs": [ + 92, + 93 + ] + }, + { + "name": "mProjectPP", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mProjectPP", + "args": "-X -x 0.98855 2mass-atlas-990502s-j1340198.fits p2mass-atlas-990502s-j1340198.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 94, + 3 + ], + "outs": [ + 95, + 96 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000001.000002.txt p2mass-atlas-990502s-j1450080.fits p2mass-atlas-990502s-j1440198.fits diff.000001.000002.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 1, + 2, + 5, + 6, + 3 + ], + "outs": [ + 99, + 100 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000001.000003.txt p2mass-atlas-990502s-j1450080.fits p2mass-atlas-990502s-j1460198.fits diff.000001.000003.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 1, + 2, + 8, + 9, + 3 + ], + "outs": [ + 101, + 102 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000001.000004.txt p2mass-atlas-990502s-j1450080.fits p2mass-atlas-990502s-j1460186.fits diff.000001.000004.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 1, + 2, + 11, + 12, + 3 + ], + "outs": [ + 103, + 104 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000001.000008.txt p2mass-atlas-990502s-j1450080.fits p2mass-atlas-990502s-j1450068.fits diff.000001.000008.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 1, + 2, + 23, + 24, + 3 + ], + "outs": [ + 105, + 106 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000001.000019.txt p2mass-atlas-990502s-j1450080.fits p2mass-atlas-990502s-j1440186.fits diff.000001.000019.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 1, + 2, + 56, + 57, + 3 + ], + "outs": [ + 107, + 108 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000001.000023.txt p2mass-atlas-990502s-j1450080.fits p2mass-atlas-990502s-j1450092.fits diff.000001.000023.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 1, + 2, + 68, + 69, + 3 + ], + "outs": [ + 109, + 110 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000002.000006.txt p2mass-atlas-990502s-j1440198.fits p2mass-atlas-990502s-j1430068.fits diff.000002.000006.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 5, + 6, + 17, + 18, + 3 + ], + "outs": [ + 111, + 112 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000002.000008.txt p2mass-atlas-990502s-j1440198.fits p2mass-atlas-990502s-j1450068.fits diff.000002.000008.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 5, + 6, + 23, + 24, + 3 + ], + "outs": [ + 113, + 114 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000002.000009.txt p2mass-atlas-990502s-j1440198.fits p2mass-atlas-990502s-j1440209.fits diff.000002.000009.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 5, + 6, + 26, + 27, + 3 + ], + "outs": [ + 115, + 116 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000002.000019.txt p2mass-atlas-990502s-j1440198.fits p2mass-atlas-990502s-j1440186.fits diff.000002.000019.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 5, + 6, + 56, + 57, + 3 + ], + "outs": [ + 117, + 118 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000002.000029.txt p2mass-atlas-990502s-j1440198.fits p2mass-atlas-990502s-j1430080.fits diff.000002.000029.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 5, + 6, + 86, + 87, + 3 + ], + "outs": [ + 119, + 120 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000003.000004.txt p2mass-atlas-990502s-j1460198.fits p2mass-atlas-990502s-j1460186.fits diff.000003.000004.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 8, + 9, + 11, + 12, + 3 + ], + "outs": [ + 121, + 122 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000003.000007.txt p2mass-atlas-990502s-j1460198.fits p2mass-atlas-990502s-j1460209.fits diff.000003.000007.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 8, + 9, + 20, + 21, + 3 + ], + "outs": [ + 123, + 124 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000003.000008.txt p2mass-atlas-990502s-j1460198.fits p2mass-atlas-990502s-j1450068.fits diff.000003.000008.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 8, + 9, + 23, + 24, + 3 + ], + "outs": [ + 125, + 126 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000004.000017.txt p2mass-atlas-990502s-j1460186.fits p2mass-atlas-990502s-j1460174.fits diff.000004.000017.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 11, + 12, + 50, + 51, + 3 + ], + "outs": [ + 127, + 128 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000004.000023.txt p2mass-atlas-990502s-j1460186.fits p2mass-atlas-990502s-j1450092.fits diff.000004.000023.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 11, + 12, + 68, + 69, + 3 + ], + "outs": [ + 129, + 130 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000005.000006.txt p2mass-atlas-990502s-j1420209.fits p2mass-atlas-990502s-j1430068.fits diff.000005.000006.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 14, + 15, + 17, + 18, + 3 + ], + "outs": [ + 131, + 132 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000005.000015.txt p2mass-atlas-990502s-j1420209.fits p2mass-atlas-990502s-j1350068.fits diff.000005.000015.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 14, + 15, + 44, + 45, + 3 + ], + "outs": [ + 133, + 134 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000005.000025.txt p2mass-atlas-990502s-j1420209.fits p2mass-atlas-990502s-j1420198.fits diff.000005.000025.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 14, + 15, + 74, + 75, + 3 + ], + "outs": [ + 135, + 136 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000006.000009.txt p2mass-atlas-990502s-j1430068.fits p2mass-atlas-990502s-j1440209.fits diff.000006.000009.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 17, + 18, + 26, + 27, + 3 + ], + "outs": [ + 137, + 138 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000006.000025.txt p2mass-atlas-990502s-j1430068.fits p2mass-atlas-990502s-j1420198.fits diff.000006.000025.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 17, + 18, + 74, + 75, + 3 + ], + "outs": [ + 139, + 140 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000006.000029.txt p2mass-atlas-990502s-j1430068.fits p2mass-atlas-990502s-j1430080.fits diff.000006.000029.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 17, + 18, + 86, + 87, + 3 + ], + "outs": [ + 141, + 142 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000007.000008.txt p2mass-atlas-990502s-j1460209.fits p2mass-atlas-990502s-j1450068.fits diff.000007.000008.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 20, + 21, + 23, + 24, + 3 + ], + "outs": [ + 143, + 144 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000008.000009.txt p2mass-atlas-990502s-j1450068.fits p2mass-atlas-990502s-j1440209.fits diff.000008.000009.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 23, + 24, + 26, + 27, + 3 + ], + "outs": [ + 145, + 146 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000010.000012.txt p2mass-atlas-990502s-j1330103.fits p2mass-atlas-990502s-j1340174.fits diff.000010.000012.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 29, + 30, + 35, + 36, + 3 + ], + "outs": [ + 147, + 148 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000010.000013.txt p2mass-atlas-990502s-j1330103.fits p2mass-atlas-990502s-j1330092.fits diff.000010.000013.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 29, + 30, + 38, + 39, + 3 + ], + "outs": [ + 149, + 150 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000011.000012.txt p2mass-atlas-990502s-j1350103.fits p2mass-atlas-990502s-j1340174.fits diff.000011.000012.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 32, + 33, + 35, + 36, + 3 + ], + "outs": [ + 151, + 152 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000011.000028.txt p2mass-atlas-990502s-j1350103.fits p2mass-atlas-990502s-j1350092.fits diff.000011.000028.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 32, + 33, + 83, + 84, + 3 + ], + "outs": [ + 153, + 154 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000011.000031.txt p2mass-atlas-990502s-j1350103.fits p2mass-atlas-990502s-j1420174.fits diff.000011.000031.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 32, + 33, + 92, + 93, + 3 + ], + "outs": [ + 155, + 156 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000012.000013.txt p2mass-atlas-990502s-j1340174.fits p2mass-atlas-990502s-j1330092.fits diff.000012.000013.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 35, + 36, + 38, + 39, + 3 + ], + "outs": [ + 157, + 158 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000012.000026.txt p2mass-atlas-990502s-j1340174.fits p2mass-atlas-990502s-j1340186.fits diff.000012.000026.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 35, + 36, + 77, + 78, + 3 + ], + "outs": [ + 159, + 160 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000012.000028.txt p2mass-atlas-990502s-j1340174.fits p2mass-atlas-990502s-j1350092.fits diff.000012.000028.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 35, + 36, + 83, + 84, + 3 + ], + "outs": [ + 161, + 162 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000013.000024.txt p2mass-atlas-990502s-j1330092.fits p2mass-atlas-990502s-j1330080.fits diff.000013.000024.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 38, + 39, + 71, + 72, + 3 + ], + "outs": [ + 163, + 164 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000013.000026.txt p2mass-atlas-990502s-j1330092.fits p2mass-atlas-990502s-j1340186.fits diff.000013.000026.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 38, + 39, + 77, + 78, + 3 + ], + "outs": [ + 165, + 166 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000014.000015.txt p2mass-atlas-990502s-j1340209.fits p2mass-atlas-990502s-j1350068.fits diff.000014.000015.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 41, + 42, + 44, + 45, + 3 + ], + "outs": [ + 167, + 168 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000014.000016.txt p2mass-atlas-990502s-j1340209.fits p2mass-atlas-990502s-j1330068.fits diff.000014.000016.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 41, + 42, + 47, + 48, + 3 + ], + "outs": [ + 169, + 170 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000014.000032.txt p2mass-atlas-990502s-j1340209.fits p2mass-atlas-990502s-j1340198.fits diff.000014.000032.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 41, + 42, + 95, + 96, + 3 + ], + "outs": [ + 171, + 172 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000015.000025.txt p2mass-atlas-990502s-j1350068.fits p2mass-atlas-990502s-j1420198.fits diff.000015.000025.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 44, + 45, + 74, + 75, + 3 + ], + "outs": [ + 173, + 174 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000015.000030.txt p2mass-atlas-990502s-j1350068.fits p2mass-atlas-990502s-j1350080.fits diff.000015.000030.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 44, + 45, + 89, + 90, + 3 + ], + "outs": [ + 175, + 176 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000015.000032.txt p2mass-atlas-990502s-j1350068.fits p2mass-atlas-990502s-j1340198.fits diff.000015.000032.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 44, + 45, + 95, + 96, + 3 + ], + "outs": [ + 177, + 178 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000016.000024.txt p2mass-atlas-990502s-j1330068.fits p2mass-atlas-990502s-j1330080.fits diff.000016.000024.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 47, + 48, + 71, + 72, + 3 + ], + "outs": [ + 179, + 180 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000016.000032.txt p2mass-atlas-990502s-j1330068.fits p2mass-atlas-990502s-j1340198.fits diff.000016.000032.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 47, + 48, + 95, + 96, + 3 + ], + "outs": [ + 181, + 182 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000017.000020.txt p2mass-atlas-990502s-j1460174.fits p2mass-atlas-990502s-j1450103.fits diff.000017.000020.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 50, + 51, + 59, + 60, + 3 + ], + "outs": [ + 183, + 184 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000017.000023.txt p2mass-atlas-990502s-j1460174.fits p2mass-atlas-990502s-j1450092.fits diff.000017.000023.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 50, + 51, + 68, + 69, + 3 + ], + "outs": [ + 185, + 186 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000018.000021.txt p2mass-atlas-990502s-j1430103.fits p2mass-atlas-990502s-j1440174.fits diff.000018.000021.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 53, + 54, + 62, + 63, + 3 + ], + "outs": [ + 187, + 188 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000018.000022.txt p2mass-atlas-990502s-j1430103.fits p2mass-atlas-990502s-j1430092.fits diff.000018.000022.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 53, + 54, + 65, + 66, + 3 + ], + "outs": [ + 189, + 190 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000018.000031.txt p2mass-atlas-990502s-j1430103.fits p2mass-atlas-990502s-j1420174.fits diff.000018.000031.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 53, + 54, + 92, + 93, + 3 + ], + "outs": [ + 191, + 192 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000019.000021.txt p2mass-atlas-990502s-j1440186.fits p2mass-atlas-990502s-j1440174.fits diff.000019.000021.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 56, + 57, + 62, + 63, + 3 + ], + "outs": [ + 193, + 194 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000019.000022.txt p2mass-atlas-990502s-j1440186.fits p2mass-atlas-990502s-j1430092.fits diff.000019.000022.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 56, + 57, + 65, + 66, + 3 + ], + "outs": [ + 195, + 196 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000019.000023.txt p2mass-atlas-990502s-j1440186.fits p2mass-atlas-990502s-j1450092.fits diff.000019.000023.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 56, + 57, + 68, + 69, + 3 + ], + "outs": [ + 197, + 198 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000019.000029.txt p2mass-atlas-990502s-j1440186.fits p2mass-atlas-990502s-j1430080.fits diff.000019.000029.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 56, + 57, + 86, + 87, + 3 + ], + "outs": [ + 199, + 200 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000020.000021.txt p2mass-atlas-990502s-j1450103.fits p2mass-atlas-990502s-j1440174.fits diff.000020.000021.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 59, + 60, + 62, + 63, + 3 + ], + "outs": [ + 201, + 202 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000020.000023.txt p2mass-atlas-990502s-j1450103.fits p2mass-atlas-990502s-j1450092.fits diff.000020.000023.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 59, + 60, + 68, + 69, + 3 + ], + "outs": [ + 203, + 204 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000021.000022.txt p2mass-atlas-990502s-j1440174.fits p2mass-atlas-990502s-j1430092.fits diff.000021.000022.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 62, + 63, + 65, + 66, + 3 + ], + "outs": [ + 205, + 206 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000021.000023.txt p2mass-atlas-990502s-j1440174.fits p2mass-atlas-990502s-j1450092.fits diff.000021.000023.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 62, + 63, + 68, + 69, + 3 + ], + "outs": [ + 207, + 208 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000022.000027.txt p2mass-atlas-990502s-j1430092.fits p2mass-atlas-990502s-j1420186.fits diff.000022.000027.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 65, + 66, + 80, + 81, + 3 + ], + "outs": [ + 209, + 210 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000022.000029.txt p2mass-atlas-990502s-j1430092.fits p2mass-atlas-990502s-j1430080.fits diff.000022.000029.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 65, + 66, + 86, + 87, + 3 + ], + "outs": [ + 211, + 212 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000022.000031.txt p2mass-atlas-990502s-j1430092.fits p2mass-atlas-990502s-j1420174.fits diff.000022.000031.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 65, + 66, + 92, + 93, + 3 + ], + "outs": [ + 213, + 214 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000024.000026.txt p2mass-atlas-990502s-j1330080.fits p2mass-atlas-990502s-j1340186.fits diff.000024.000026.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 71, + 72, + 77, + 78, + 3 + ], + "outs": [ + 215, + 216 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000024.000032.txt p2mass-atlas-990502s-j1330080.fits p2mass-atlas-990502s-j1340198.fits diff.000024.000032.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 71, + 72, + 95, + 96, + 3 + ], + "outs": [ + 217, + 218 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000025.000027.txt p2mass-atlas-990502s-j1420198.fits p2mass-atlas-990502s-j1420186.fits diff.000025.000027.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 74, + 75, + 80, + 81, + 3 + ], + "outs": [ + 219, + 220 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000025.000029.txt p2mass-atlas-990502s-j1420198.fits p2mass-atlas-990502s-j1430080.fits diff.000025.000029.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 74, + 75, + 86, + 87, + 3 + ], + "outs": [ + 221, + 222 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000025.000030.txt p2mass-atlas-990502s-j1420198.fits p2mass-atlas-990502s-j1350080.fits diff.000025.000030.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 74, + 75, + 89, + 90, + 3 + ], + "outs": [ + 223, + 224 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000026.000028.txt p2mass-atlas-990502s-j1340186.fits p2mass-atlas-990502s-j1350092.fits diff.000026.000028.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 77, + 78, + 83, + 84, + 3 + ], + "outs": [ + 225, + 226 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000026.000030.txt p2mass-atlas-990502s-j1340186.fits p2mass-atlas-990502s-j1350080.fits diff.000026.000030.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 77, + 78, + 89, + 90, + 3 + ], + "outs": [ + 227, + 228 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000026.000032.txt p2mass-atlas-990502s-j1340186.fits p2mass-atlas-990502s-j1340198.fits diff.000026.000032.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 77, + 78, + 95, + 96, + 3 + ], + "outs": [ + 229, + 230 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000027.000028.txt p2mass-atlas-990502s-j1420186.fits p2mass-atlas-990502s-j1350092.fits diff.000027.000028.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 80, + 81, + 83, + 84, + 3 + ], + "outs": [ + 231, + 232 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000027.000029.txt p2mass-atlas-990502s-j1420186.fits p2mass-atlas-990502s-j1430080.fits diff.000027.000029.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 80, + 81, + 86, + 87, + 3 + ], + "outs": [ + 233, + 234 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000027.000030.txt p2mass-atlas-990502s-j1420186.fits p2mass-atlas-990502s-j1350080.fits diff.000027.000030.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 80, + 81, + 89, + 90, + 3 + ], + "outs": [ + 235, + 236 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000027.000031.txt p2mass-atlas-990502s-j1420186.fits p2mass-atlas-990502s-j1420174.fits diff.000027.000031.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 80, + 81, + 92, + 93, + 3 + ], + "outs": [ + 237, + 238 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000028.000030.txt p2mass-atlas-990502s-j1350092.fits p2mass-atlas-990502s-j1350080.fits diff.000028.000030.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 83, + 84, + 89, + 90, + 3 + ], + "outs": [ + 239, + 240 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000028.000031.txt p2mass-atlas-990502s-j1350092.fits p2mass-atlas-990502s-j1420174.fits diff.000028.000031.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 83, + 84, + 92, + 93, + 3 + ], + "outs": [ + 241, + 242 + ] + }, + { + "name": "mDiffFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mDiffFit", + "args": "-s fit.000030.000032.txt p2mass-atlas-990502s-j1350080.fits p2mass-atlas-990502s-j1340198.fits diff.000030.000032.fits big_region_20130212_013030_8881.hdr" + } + }, + "ins": [ + 97, + 98, + 89, + 90, + 95, + 96, + 3 + ], + "outs": [ + 243, + 244 + ] + }, + { + "name": "mConcatFit", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mConcatFit", + "args": "statfile_20130212_013030_8881.tbl fits.tbl ." + } + }, + "ins": [ + 245, + 99, + 101, + 103, + 105, + 107, + 109, + 111, + 113, + 115, + 117, + 119, + 121, + 123, + 125, + 127, + 129, + 131, + 133, + 135, + 137, + 139, + 141, + 143, + 145, + 147, + 149, + 151, + 153, + 155, + 157, + 159, + 161, + 163, + 165, + 167, + 169, + 171, + 173, + 175, + 177, + 179, + 181, + 183, + 185, + 187, + 189, + 191, + 193, + 195, + 197, + 199, + 201, + 203, + 205, + 207, + 209, + 211, + 213, + 215, + 217, + 219, + 221, + 223, + 225, + 227, + 229, + 231, + 233, + 235, + 237, + 239, + 241, + 243 + ], + "outs": [ + 246 + ] + }, + { + "name": "mBgModel", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBgModel", + "args": "-i 100000 pimages_20130212_013030_8881.tbl fits.tbl corrections.tbl" + } + }, + "ins": [ + 247, + 246 + ], + "outs": [ + 248 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1450080.fits c2mass-atlas-990502s-j1450080.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 1, + 2, + 247, + 248 + ], + "outs": [ + 249, + 250 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1440198.fits c2mass-atlas-990502s-j1440198.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 5, + 6, + 247, + 248 + ], + "outs": [ + 251, + 252 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1460198.fits c2mass-atlas-990502s-j1460198.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 8, + 9, + 247, + 248 + ], + "outs": [ + 253, + 254 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1460186.fits c2mass-atlas-990502s-j1460186.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 11, + 12, + 247, + 248 + ], + "outs": [ + 255, + 256 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1420209.fits c2mass-atlas-990502s-j1420209.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 14, + 15, + 247, + 248 + ], + "outs": [ + 257, + 258 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1430068.fits c2mass-atlas-990502s-j1430068.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 17, + 18, + 247, + 248 + ], + "outs": [ + 259, + 260 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1460209.fits c2mass-atlas-990502s-j1460209.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 20, + 21, + 247, + 248 + ], + "outs": [ + 261, + 262 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1450068.fits c2mass-atlas-990502s-j1450068.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 23, + 24, + 247, + 248 + ], + "outs": [ + 263, + 264 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1440209.fits c2mass-atlas-990502s-j1440209.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 26, + 27, + 247, + 248 + ], + "outs": [ + 265, + 266 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1330103.fits c2mass-atlas-990502s-j1330103.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 29, + 30, + 247, + 248 + ], + "outs": [ + 267, + 268 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1350103.fits c2mass-atlas-990502s-j1350103.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 32, + 33, + 247, + 248 + ], + "outs": [ + 269, + 270 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1340174.fits c2mass-atlas-990502s-j1340174.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 35, + 36, + 247, + 248 + ], + "outs": [ + 271, + 272 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1330092.fits c2mass-atlas-990502s-j1330092.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 38, + 39, + 247, + 248 + ], + "outs": [ + 273, + 274 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1340209.fits c2mass-atlas-990502s-j1340209.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 41, + 42, + 247, + 248 + ], + "outs": [ + 275, + 276 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1350068.fits c2mass-atlas-990502s-j1350068.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 44, + 45, + 247, + 248 + ], + "outs": [ + 277, + 278 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1330068.fits c2mass-atlas-990502s-j1330068.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 47, + 48, + 247, + 248 + ], + "outs": [ + 279, + 280 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1460174.fits c2mass-atlas-990502s-j1460174.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 50, + 51, + 247, + 248 + ], + "outs": [ + 281, + 282 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1430103.fits c2mass-atlas-990502s-j1430103.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 53, + 54, + 247, + 248 + ], + "outs": [ + 283, + 284 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1440186.fits c2mass-atlas-990502s-j1440186.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 56, + 57, + 247, + 248 + ], + "outs": [ + 285, + 286 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1450103.fits c2mass-atlas-990502s-j1450103.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 59, + 60, + 247, + 248 + ], + "outs": [ + 287, + 288 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1440174.fits c2mass-atlas-990502s-j1440174.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 62, + 63, + 247, + 248 + ], + "outs": [ + 289, + 290 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1430092.fits c2mass-atlas-990502s-j1430092.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 65, + 66, + 247, + 248 + ], + "outs": [ + 291, + 292 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1450092.fits c2mass-atlas-990502s-j1450092.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 68, + 69, + 247, + 248 + ], + "outs": [ + 293, + 294 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1330080.fits c2mass-atlas-990502s-j1330080.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 71, + 72, + 247, + 248 + ], + "outs": [ + 295, + 296 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1420198.fits c2mass-atlas-990502s-j1420198.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 74, + 75, + 247, + 248 + ], + "outs": [ + 297, + 298 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1340186.fits c2mass-atlas-990502s-j1340186.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 77, + 78, + 247, + 248 + ], + "outs": [ + 299, + 300 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1420186.fits c2mass-atlas-990502s-j1420186.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 80, + 81, + 247, + 248 + ], + "outs": [ + 301, + 302 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1350092.fits c2mass-atlas-990502s-j1350092.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 83, + 84, + 247, + 248 + ], + "outs": [ + 303, + 304 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1430080.fits c2mass-atlas-990502s-j1430080.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 86, + 87, + 247, + 248 + ], + "outs": [ + 305, + 306 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1350080.fits c2mass-atlas-990502s-j1350080.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 89, + 90, + 247, + 248 + ], + "outs": [ + 307, + 308 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1420174.fits c2mass-atlas-990502s-j1420174.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 92, + 93, + 247, + 248 + ], + "outs": [ + 309, + 310 + ] + }, + { + "name": "mBackground", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mBackground", + "args": "-t p2mass-atlas-990502s-j1340198.fits c2mass-atlas-990502s-j1340198.fits pimages_20130212_013030_8881.tbl corrections.tbl" + } + }, + "ins": [ + 95, + 96, + 247, + 248 + ], + "outs": [ + 311, + 312 + ] + }, + { + "name": "mImgtbl", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mImgtbl", + "args": "-t cimages_20130212_013030_8881.tbl . newcimages.tbl" + } + }, + "ins": [ + 313, + 249, + 251, + 253, + 255, + 257, + 259, + 261, + 263, + 265, + 267, + 269, + 271, + 273, + 275, + 277, + 279, + 281, + 283, + 285, + 287, + 289, + 291, + 293, + 295, + 297, + 299, + 301, + 303, + 305, + 307, + 309, + 311 + ], + "outs": [ + 314 + ] + }, + { + "name": "mAdd", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mAdd", + "args": "-e newcimages.tbl region_20130212_013030_8881.hdr mosaic_20130212_013030_8881.fits" + } + }, + "ins": [ + 314, + 315, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312 + ], + "outs": [ + 316, + 317 + ] + }, + { + "name": "mShrink", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mShrink", + "args": "mosaic_20130212_013030_8881.fits shrunken_20130212_013030_8881.fits 2" + } + }, + "ins": [ + 316 + ], + "outs": [ + 318 + ] + }, + { + "name": "mJPEG", + "function": "command_notifyevents", + "type": "dataflow", + "firingLimit": 1, + "config": { + "executor": { + "executable": "mJPEG", + "args": "-ct 1 -gray shrunken_20130212_013030_8881.fits min max gaussianlog -out shrunken_20130212_013030_8881.jpg" + } + }, + "ins": [ + 318 + ], + "outs": [ + 319 + ] + } + ], + "data": [ + { + "name": "2mass-atlas-990502s-j1450080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450080_area.fits" + }, + { + "name": "big_region_20130212_013030_8881.hdr" + }, + { + "name": "2mass-atlas-990502s-j1440198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440198_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1460198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460198_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1460186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460186_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1420209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420209_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1430068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430068_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1460209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460209_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1450068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450068_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1440209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440209_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1330103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330103_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1350103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350103_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1340174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340174_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1330092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330092_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1340209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340209.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340209_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1350068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350068_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1330068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330068.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330068_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1460174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1460174_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1430103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430103_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1440186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440186_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1450103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450103.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450103_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1440174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1440174_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1430092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430092_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1450092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1450092_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1330080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1330080_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1420198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420198_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1340186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340186_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1420186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420186.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420186_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1350092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350092.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350092_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1430080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1430080_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1350080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350080.fits" + }, + { + "name": "p2mass-atlas-990502s-j1350080_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1420174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420174.fits" + }, + { + "name": "p2mass-atlas-990502s-j1420174_area.fits" + }, + { + "name": "2mass-atlas-990502s-j1340198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340198.fits" + }, + { + "name": "p2mass-atlas-990502s-j1340198_area.fits" + }, + { + "name": "mDiff" + }, + { + "name": "mFitplane" + }, + { + "name": "fit.000001.000002.txt" + }, + { + "name": "diff.000001.000002.fits" + }, + { + "name": "fit.000001.000003.txt" + }, + { + "name": "diff.000001.000003.fits" + }, + { + "name": "fit.000001.000004.txt" + }, + { + "name": "diff.000001.000004.fits" + }, + { + "name": "fit.000001.000008.txt" + }, + { + "name": "diff.000001.000008.fits" + }, + { + "name": "fit.000001.000019.txt" + }, + { + "name": "diff.000001.000019.fits" + }, + { + "name": "fit.000001.000023.txt" + }, + { + "name": "diff.000001.000023.fits" + }, + { + "name": "fit.000002.000006.txt" + }, + { + "name": "diff.000002.000006.fits" + }, + { + "name": "fit.000002.000008.txt" + }, + { + "name": "diff.000002.000008.fits" + }, + { + "name": "fit.000002.000009.txt" + }, + { + "name": "diff.000002.000009.fits" + }, + { + "name": "fit.000002.000019.txt" + }, + { + "name": "diff.000002.000019.fits" + }, + { + "name": "fit.000002.000029.txt" + }, + { + "name": "diff.000002.000029.fits" + }, + { + "name": "fit.000003.000004.txt" + }, + { + "name": "diff.000003.000004.fits" + }, + { + "name": "fit.000003.000007.txt" + }, + { + "name": "diff.000003.000007.fits" + }, + { + "name": "fit.000003.000008.txt" + }, + { + "name": "diff.000003.000008.fits" + }, + { + "name": "fit.000004.000017.txt" + }, + { + "name": "diff.000004.000017.fits" + }, + { + "name": "fit.000004.000023.txt" + }, + { + "name": "diff.000004.000023.fits" + }, + { + "name": "fit.000005.000006.txt" + }, + { + "name": "diff.000005.000006.fits" + }, + { + "name": "fit.000005.000015.txt" + }, + { + "name": "diff.000005.000015.fits" + }, + { + "name": "fit.000005.000025.txt" + }, + { + "name": "diff.000005.000025.fits" + }, + { + "name": "fit.000006.000009.txt" + }, + { + "name": "diff.000006.000009.fits" + }, + { + "name": "fit.000006.000025.txt" + }, + { + "name": "diff.000006.000025.fits" + }, + { + "name": "fit.000006.000029.txt" + }, + { + "name": "diff.000006.000029.fits" + }, + { + "name": "fit.000007.000008.txt" + }, + { + "name": "diff.000007.000008.fits" + }, + { + "name": "fit.000008.000009.txt" + }, + { + "name": "diff.000008.000009.fits" + }, + { + "name": "fit.000010.000012.txt" + }, + { + "name": "diff.000010.000012.fits" + }, + { + "name": "fit.000010.000013.txt" + }, + { + "name": "diff.000010.000013.fits" + }, + { + "name": "fit.000011.000012.txt" + }, + { + "name": "diff.000011.000012.fits" + }, + { + "name": "fit.000011.000028.txt" + }, + { + "name": "diff.000011.000028.fits" + }, + { + "name": "fit.000011.000031.txt" + }, + { + "name": "diff.000011.000031.fits" + }, + { + "name": "fit.000012.000013.txt" + }, + { + "name": "diff.000012.000013.fits" + }, + { + "name": "fit.000012.000026.txt" + }, + { + "name": "diff.000012.000026.fits" + }, + { + "name": "fit.000012.000028.txt" + }, + { + "name": "diff.000012.000028.fits" + }, + { + "name": "fit.000013.000024.txt" + }, + { + "name": "diff.000013.000024.fits" + }, + { + "name": "fit.000013.000026.txt" + }, + { + "name": "diff.000013.000026.fits" + }, + { + "name": "fit.000014.000015.txt" + }, + { + "name": "diff.000014.000015.fits" + }, + { + "name": "fit.000014.000016.txt" + }, + { + "name": "diff.000014.000016.fits" + }, + { + "name": "fit.000014.000032.txt" + }, + { + "name": "diff.000014.000032.fits" + }, + { + "name": "fit.000015.000025.txt" + }, + { + "name": "diff.000015.000025.fits" + }, + { + "name": "fit.000015.000030.txt" + }, + { + "name": "diff.000015.000030.fits" + }, + { + "name": "fit.000015.000032.txt" + }, + { + "name": "diff.000015.000032.fits" + }, + { + "name": "fit.000016.000024.txt" + }, + { + "name": "diff.000016.000024.fits" + }, + { + "name": "fit.000016.000032.txt" + }, + { + "name": "diff.000016.000032.fits" + }, + { + "name": "fit.000017.000020.txt" + }, + { + "name": "diff.000017.000020.fits" + }, + { + "name": "fit.000017.000023.txt" + }, + { + "name": "diff.000017.000023.fits" + }, + { + "name": "fit.000018.000021.txt" + }, + { + "name": "diff.000018.000021.fits" + }, + { + "name": "fit.000018.000022.txt" + }, + { + "name": "diff.000018.000022.fits" + }, + { + "name": "fit.000018.000031.txt" + }, + { + "name": "diff.000018.000031.fits" + }, + { + "name": "fit.000019.000021.txt" + }, + { + "name": "diff.000019.000021.fits" + }, + { + "name": "fit.000019.000022.txt" + }, + { + "name": "diff.000019.000022.fits" + }, + { + "name": "fit.000019.000023.txt" + }, + { + "name": "diff.000019.000023.fits" + }, + { + "name": "fit.000019.000029.txt" + }, + { + "name": "diff.000019.000029.fits" + }, + { + "name": "fit.000020.000021.txt" + }, + { + "name": "diff.000020.000021.fits" + }, + { + "name": "fit.000020.000023.txt" + }, + { + "name": "diff.000020.000023.fits" + }, + { + "name": "fit.000021.000022.txt" + }, + { + "name": "diff.000021.000022.fits" + }, + { + "name": "fit.000021.000023.txt" + }, + { + "name": "diff.000021.000023.fits" + }, + { + "name": "fit.000022.000027.txt" + }, + { + "name": "diff.000022.000027.fits" + }, + { + "name": "fit.000022.000029.txt" + }, + { + "name": "diff.000022.000029.fits" + }, + { + "name": "fit.000022.000031.txt" + }, + { + "name": "diff.000022.000031.fits" + }, + { + "name": "fit.000024.000026.txt" + }, + { + "name": "diff.000024.000026.fits" + }, + { + "name": "fit.000024.000032.txt" + }, + { + "name": "diff.000024.000032.fits" + }, + { + "name": "fit.000025.000027.txt" + }, + { + "name": "diff.000025.000027.fits" + }, + { + "name": "fit.000025.000029.txt" + }, + { + "name": "diff.000025.000029.fits" + }, + { + "name": "fit.000025.000030.txt" + }, + { + "name": "diff.000025.000030.fits" + }, + { + "name": "fit.000026.000028.txt" + }, + { + "name": "diff.000026.000028.fits" + }, + { + "name": "fit.000026.000030.txt" + }, + { + "name": "diff.000026.000030.fits" + }, + { + "name": "fit.000026.000032.txt" + }, + { + "name": "diff.000026.000032.fits" + }, + { + "name": "fit.000027.000028.txt" + }, + { + "name": "diff.000027.000028.fits" + }, + { + "name": "fit.000027.000029.txt" + }, + { + "name": "diff.000027.000029.fits" + }, + { + "name": "fit.000027.000030.txt" + }, + { + "name": "diff.000027.000030.fits" + }, + { + "name": "fit.000027.000031.txt" + }, + { + "name": "diff.000027.000031.fits" + }, + { + "name": "fit.000028.000030.txt" + }, + { + "name": "diff.000028.000030.fits" + }, + { + "name": "fit.000028.000031.txt" + }, + { + "name": "diff.000028.000031.fits" + }, + { + "name": "fit.000030.000032.txt" + }, + { + "name": "diff.000030.000032.fits" + }, + { + "name": "statfile_20130212_013030_8881.tbl" + }, + { + "name": "fits.tbl" + }, + { + "name": "pimages_20130212_013030_8881.tbl" + }, + { + "name": "corrections.tbl" + }, + { + "name": "c2mass-atlas-990502s-j1450080.fits" + }, + { + "name": "c2mass-atlas-990502s-j1450080_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440198.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440198_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460198.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460198_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460186.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460186_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420209.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420209_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430068.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430068_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460209.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460209_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1450068.fits" + }, + { + "name": "c2mass-atlas-990502s-j1450068_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440209.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440209_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330103.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330103_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350103.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350103_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340174.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340174_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330092.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330092_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340209.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340209_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350068.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350068_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330068.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330068_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460174.fits" + }, + { + "name": "c2mass-atlas-990502s-j1460174_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430103.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430103_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440186.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440186_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1450103.fits" + }, + { + "name": "c2mass-atlas-990502s-j1450103_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440174.fits" + }, + { + "name": "c2mass-atlas-990502s-j1440174_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430092.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430092_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1450092.fits" + }, + { + "name": "c2mass-atlas-990502s-j1450092_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330080.fits" + }, + { + "name": "c2mass-atlas-990502s-j1330080_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420198.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420198_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340186.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340186_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420186.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420186_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350092.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350092_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430080.fits" + }, + { + "name": "c2mass-atlas-990502s-j1430080_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350080.fits" + }, + { + "name": "c2mass-atlas-990502s-j1350080_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420174.fits" + }, + { + "name": "c2mass-atlas-990502s-j1420174_area.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340198.fits" + }, + { + "name": "c2mass-atlas-990502s-j1340198_area.fits" + }, + { + "name": "cimages_20130212_013030_8881.tbl" + }, + { + "name": "newcimages.tbl" + }, + { + "name": "region_20130212_013030_8881.hdr" + }, + { + "name": "mosaic_20130212_013030_8881.fits" + }, + { + "name": "mosaic_20130212_013030_8881_area.fits" + }, + { + "name": "shrunken_20130212_013030_8881.fits" + }, + { + "name": "shrunken_20130212_013030_8881.jpg" + } + ], + "ins": [ + 0, + 3, + 4, + 7, + 10, + 13, + 16, + 19, + 22, + 25, + 28, + 31, + 34, + 37, + 40, + 43, + 46, + 49, + 52, + 55, + 58, + 61, + 64, + 67, + 70, + 73, + 76, + 79, + 82, + 85, + 88, + 91, + 94, + 97, + 98, + 245, + 247, + 313, + 315 + ], + "outs": [ + 100, + 102, + 104, + 106, + 108, + 110, + 112, + 114, + 116, + 118, + 120, + 122, + 124, + 126, + 128, + 130, + 132, + 134, + 136, + 138, + 140, + 142, + 144, + 146, + 148, + 150, + 152, + 154, + 156, + 158, + 160, + 162, + 164, + 166, + 168, + 170, + 172, + 174, + 176, + 178, + 180, + 182, + 184, + 186, + 188, + 190, + 192, + 194, + 196, + 198, + 200, + 202, + 204, + 206, + 208, + 210, + 212, + 214, + 216, + 218, + 220, + 222, + 224, + 226, + 228, + 230, + 232, + 234, + 236, + 238, + 240, + 242, + 244, + 317, + 319 + ] +} From b74b06958414f43fca153b9fc508c4c407365937 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 11 Feb 2014 19:07:42 +0100 Subject: [PATCH 05/73] Added prototype of sending event to (probably) available eventServer. --- engine/taskDataflowFSM.js | 3 ++- functions/command.js | 6 +++++- scripts/runwf.js | 4 ++++ wflib/index.js | 5 ++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/engine/taskDataflowFSM.js b/engine/taskDataflowFSM.js index cbe6b89..5f1af2f 100644 --- a/engine/taskDataflowFSM.js +++ b/engine/taskDataflowFSM.js @@ -296,7 +296,8 @@ function TaskLogic() { task.id, funcIns, task.sigValues, - funcOuts, emul, + funcOuts, emul, + task.engine.eventServer, function(err, outs) { err ? cb(err): cb(null, outs, returned2Ready); } diff --git a/functions/command.js b/functions/command.js index bfd54a5..6c31412 100644 --- a/functions/command.js +++ b/functions/command.js @@ -39,8 +39,12 @@ function command_notifyevents(ins, outs, config, cb) { var exec = config.executor.executable, args = config.executor.args; - console.log(exec, args); +// console.log(exec, args); + var eventserver = config['eventserver']; + if(eventserver !== 'undefined') { + eventserver.emit("job.done", exec, args); + } cb(null, outs); } diff --git a/scripts/runwf.js b/scripts/runwf.js index 301eb5b..bacc0cf 100644 --- a/scripts/runwf.js +++ b/scripts/runwf.js @@ -44,6 +44,10 @@ if (argv.d) { var runWf = function(wfId) { engine = new Engine({"emulate":"false"}, wflib, wfId, function(err) { + //This represent custom plugin listening on event from available eventServer + engine.eventServer.on('job.done', function(exec, args) { + console.log('Event captured: ' + exec + ' ' + args + ' job done'); + }); engine.runInstance(function(err) { console.log("Wf id="+wfId); if (argv.s) { diff --git a/wflib/index.js b/wflib/index.js index c3a5507..4456089 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -1287,7 +1287,7 @@ function public_invokeTaskFunction1(wfId, taskId, insIds_, outsIds_, emulate, cb /* * @insValues - array of input signal values as returned by fetchInputs */ -function public_invokeTaskFunction2(wfId, taskId, insIds_, insValues, outsIds_, emulate, cb) { +function public_invokeTaskFunction2(wfId, taskId, insIds_, insValues, outsIds_, emulate, eventServer, cb) { function isArray(what) { return Object.prototype.toString.call(what) === '[object Array]'; } @@ -1361,6 +1361,9 @@ function public_invokeTaskFunction2(wfId, taskId, insIds_, insValues, outsIds_, //var executor = taskInfo.executor ? taskInfo.executor: null; //onsole.log("INS VALUES", insValues); + if (eventServer !== 'undefined') { + conf['eventserver'] = eventServer; + } f(ins, outs, conf, function(err, outs) { //if (outs) { onsole.log("VALUE="+outs[0].value); } // DEBUG From b7bf610aff812b812626cbab1a69b4ace5d7fb25 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 12 Feb 2014 14:35:31 +0100 Subject: [PATCH 06/73] Added basic tests for functions. --- package.json | 3 ++- tests/functions.js | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tests/functions.js diff --git a/package.json b/package.json index a6d22ea..9f4fc1b 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "nconf": "0.6.8", "file": "0.2.x", "value": "0.3.0", - "form-data": "" + "form-data": "", + "nodetest": "" } } diff --git a/tests/functions.js b/tests/functions.js new file mode 100644 index 0000000..399156f --- /dev/null +++ b/tests/functions.js @@ -0,0 +1,18 @@ +var functions = require('../functions') + +exports.test_function_notifyevents = function(test) { + var ins = [], + outs = []; + var config = { + executor: { + executable: "ls", + args: "-la" + } + }; + var cb = function(exceptions, outs) { + + }; + functions.command_print(ins, outs, config, cb); + + test.done() +} From e2c8687a139f46bdfc562d9c3aa77c41e96da0a7 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 12 Feb 2014 14:35:45 +0100 Subject: [PATCH 07/73] Added logging of executable and args in command_notifyevents in case no eventServer is available. --- functions/command.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/command.js b/functions/command.js index 6c31412..acfae45 100644 --- a/functions/command.js +++ b/functions/command.js @@ -39,11 +39,11 @@ function command_notifyevents(ins, outs, config, cb) { var exec = config.executor.executable, args = config.executor.args; -// console.log(exec, args); - var eventserver = config['eventserver']; if(eventserver !== 'undefined') { eventserver.emit("job.done", exec, args); + } else { + console.log("log: " + exec, args); } cb(null, outs); } @@ -51,4 +51,4 @@ function command_notifyevents(ins, outs, config, cb) { exports.command = command; exports.command_print = command_print; -exports.command_notifyevents = command_notifyevents; +exports.command_notifyevents = command_notifyevents; \ No newline at end of file From b906aae6bf72a52400b3ff35f6a93a045188f2fc Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 12 Feb 2014 14:58:38 +0100 Subject: [PATCH 08/73] Added test_function_notifyevents_with_eventserver. --- tests/functions.js | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/tests/functions.js b/tests/functions.js index 399156f..3fbed7e 100644 --- a/tests/functions.js +++ b/tests/functions.js @@ -1,18 +1,33 @@ var functions = require('../functions') -exports.test_function_notifyevents = function(test) { +exports.setUp = function(callback) { + this.config = { + executor: { + executable: "ls", + args: "-la" + } + }; + callback(); +} + +exports.test_function_notifyevents_with_eventserver = function(test) { + var notified = false; var ins = [], outs = []; - var config = { - executor: { - executable: "ls", - args: "-la" - } + + this.config.eventserver = { + emit: function(type, arg1, arg2) { + notified = true; + } }; - var cb = function(exceptions, outs) { + var cb = function(exceptions, outs) { + test.equal(exceptions, null); }; - functions.command_print(ins, outs, config, cb); + + + functions.command_notifyevents(ins, outs, this.config, cb); + test.ok(notified); test.done() } From caf0465c596f8298f6a919f098ba35ff9be27a41 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 12 Feb 2014 15:01:59 +0100 Subject: [PATCH 09/73] Fix detecting presence of eventServer in command_notifyevents. --- functions/command.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/command.js b/functions/command.js index acfae45..7c376e5 100644 --- a/functions/command.js +++ b/functions/command.js @@ -40,10 +40,10 @@ function command_notifyevents(ins, outs, config, cb) { args = config.executor.args; var eventserver = config['eventserver']; - if(eventserver !== 'undefined') { + if(typeof eventserver !== 'undefined' && eventserver) { eventserver.emit("job.done", exec, args); } else { - console.log("log: " + exec, args); + console.log("loged: " + exec, args); } cb(null, outs); } From 49c19ab497e40a0b4a94f128436c0dc6a6816f33 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 12 Feb 2014 15:18:02 +0100 Subject: [PATCH 10/73] Added test_function_notifyevents_without_eventserver. --- tests/functions.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/functions.js b/tests/functions.js index 3fbed7e..4961656 100644 --- a/tests/functions.js +++ b/tests/functions.js @@ -31,3 +31,16 @@ exports.test_function_notifyevents_with_eventserver = function(test) { test.done() } + +exports.test_function_notifyevents_without_eventserver = function(test) { + var ins = [], + outs = []; + + var cb = function(exceptions, outs) { + test.equal(exceptions, null); + }; + + functions.command_notifyevents(ins, outs, this.config, cb); + + test.done() +} From 2111109049a15164228755e30b6d22e456135fd9 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 12 Feb 2014 15:23:58 +0100 Subject: [PATCH 11/73] Remve arguments from mocked eventserver. --- tests/functions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functions.js b/tests/functions.js index 4961656..3b47c46 100644 --- a/tests/functions.js +++ b/tests/functions.js @@ -16,7 +16,7 @@ exports.test_function_notifyevents_with_eventserver = function(test) { outs = []; this.config.eventserver = { - emit: function(type, arg1, arg2) { + emit: function() { notified = true; } }; From 4bb1ec653c5c0639d002aa6a9f46b14cfbbbf586 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Thu, 13 Feb 2014 10:37:29 +0100 Subject: [PATCH 12/73] Add TODO about amqpCommand initialization. --- functions/amqpCommand.js | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/amqpCommand.js b/functions/amqpCommand.js index 320d5ea..5f6b51e 100644 --- a/functions/amqpCommand.js +++ b/functions/amqpCommand.js @@ -8,6 +8,7 @@ var AMQP_URL = process.env.AMQP_URL ? process.env.AMQP_URL : "amqp://localhost: var S3_BUCKET = process.env.S3_BUCKET; var S3_PATH = process.env.S3_PATH; +//TODO: initialize @ first use, or module.init() console.log("[AMQP] Starting connection!"); var connection = amqplib.connect(AMQP_URL); var connectionReady = false; From 1fbd724094d20011ec0a98c57a82e0cd0a1320d4 Mon Sep 17 00:00:00 2001 From: Kamil Figiela Date: Tue, 18 Feb 2014 19:04:52 +0100 Subject: [PATCH 13/73] Cleanup garbage code --- functions/amqpCommand.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/functions/amqpCommand.js b/functions/amqpCommand.js index 4955e74..faf1f6f 100644 --- a/functions/amqpCommand.js +++ b/functions/amqpCommand.js @@ -2,12 +2,10 @@ var uuid = require('uuid'); var when = require('when'); var defer = when.defer; var amqplib = require('amqplib'); -var util = require('util'); var executor_config = require('./amqpCommand.config.js'); console.log("[AMQP] Starting connection!"); -var connection = require('amqplib').connect(executor_config.amqp_url); -var connectionReady = false; +var connection = amqplib.connect(executor_config.amqp_url); connection.then(function(conn) { connection.once('SIGINT', function() { connection.close(); }); From 39a6097801f7b458ccead7417607c5e2630ad926 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 24 Feb 2014 22:02:49 +0100 Subject: [PATCH 14/73] Incorporate proposed event server. --- mediator/index.js | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 mediator/index.js diff --git a/mediator/index.js b/mediator/index.js new file mode 100644 index 0000000..a715526 --- /dev/null +++ b/mediator/index.js @@ -0,0 +1,43 @@ +var EventEmitter2 = require('eventemitter2').EventEmitter2, + server = new EventEmitter2({ + wildcard: true, // should the event emitter use wildcards. + delimiter: '.', // the delimiter used to segment namespaces, defaults to `.`. + newListener: false, // if you want to emit the newListener event set to true. + maxListeners: 20 // the max number of listeners that can be assigned to an event, defaults to 10. + }); + +function emit(ev, data) { + data.time = new Date().toISOString(); // automatically add a timestamp to event data + server.emit(ev, data); +} + +function on(ev, listener) { + server.on(ev, listener); +} + +// this object is exported as the API of the event logger (hides implementation) +var eventlog = { + emit: emit, + on: on +} + +// Here's how to subscribe to events: +server.on('trace.*', function(data) { + console.log(arguments); + // "this.event" contains the full event name + console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); +}); + +exports.eventlog = eventlog; + +// Example use of the event logger from another module: +// eventlog = require('../eventlog').eventlog, +// ... +// eventlog.emit('trace.invokingFunction', {"appId": wfId, "procId": procId }); +// +// We should agree on a set of standard fields that allow one to "correlate" the event, such as: +// - appId - unique id of the workflow instance +// - procId - process id (if the event is related to a process) +// - firingId - number of firing of the process (if the event is related to a particular firing) +// - sigId - signal id (if the event is related to a signal) +// - time - time stamp From 504f314a17c440435c4e9448ff8ed236e4e580a7 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 24 Feb 2014 22:03:40 +0100 Subject: [PATCH 15/73] Rename mediator to eventlog --- eventlog/index.js | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 eventlog/index.js diff --git a/eventlog/index.js b/eventlog/index.js new file mode 100644 index 0000000..a715526 --- /dev/null +++ b/eventlog/index.js @@ -0,0 +1,43 @@ +var EventEmitter2 = require('eventemitter2').EventEmitter2, + server = new EventEmitter2({ + wildcard: true, // should the event emitter use wildcards. + delimiter: '.', // the delimiter used to segment namespaces, defaults to `.`. + newListener: false, // if you want to emit the newListener event set to true. + maxListeners: 20 // the max number of listeners that can be assigned to an event, defaults to 10. + }); + +function emit(ev, data) { + data.time = new Date().toISOString(); // automatically add a timestamp to event data + server.emit(ev, data); +} + +function on(ev, listener) { + server.on(ev, listener); +} + +// this object is exported as the API of the event logger (hides implementation) +var eventlog = { + emit: emit, + on: on +} + +// Here's how to subscribe to events: +server.on('trace.*', function(data) { + console.log(arguments); + // "this.event" contains the full event name + console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); +}); + +exports.eventlog = eventlog; + +// Example use of the event logger from another module: +// eventlog = require('../eventlog').eventlog, +// ... +// eventlog.emit('trace.invokingFunction', {"appId": wfId, "procId": procId }); +// +// We should agree on a set of standard fields that allow one to "correlate" the event, such as: +// - appId - unique id of the workflow instance +// - procId - process id (if the event is related to a process) +// - firingId - number of firing of the process (if the event is related to a particular firing) +// - sigId - signal id (if the event is related to a signal) +// - time - time stamp From 5e2356fc4d1cc415860d95ee7796fa9411a9928b Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 24 Feb 2014 22:04:02 +0100 Subject: [PATCH 16/73] Delete mediator. --- mediator/index.js | 43 ------------------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 mediator/index.js diff --git a/mediator/index.js b/mediator/index.js deleted file mode 100644 index a715526..0000000 --- a/mediator/index.js +++ /dev/null @@ -1,43 +0,0 @@ -var EventEmitter2 = require('eventemitter2').EventEmitter2, - server = new EventEmitter2({ - wildcard: true, // should the event emitter use wildcards. - delimiter: '.', // the delimiter used to segment namespaces, defaults to `.`. - newListener: false, // if you want to emit the newListener event set to true. - maxListeners: 20 // the max number of listeners that can be assigned to an event, defaults to 10. - }); - -function emit(ev, data) { - data.time = new Date().toISOString(); // automatically add a timestamp to event data - server.emit(ev, data); -} - -function on(ev, listener) { - server.on(ev, listener); -} - -// this object is exported as the API of the event logger (hides implementation) -var eventlog = { - emit: emit, - on: on -} - -// Here's how to subscribe to events: -server.on('trace.*', function(data) { - console.log(arguments); - // "this.event" contains the full event name - console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); -}); - -exports.eventlog = eventlog; - -// Example use of the event logger from another module: -// eventlog = require('../eventlog').eventlog, -// ... -// eventlog.emit('trace.invokingFunction', {"appId": wfId, "procId": procId }); -// -// We should agree on a set of standard fields that allow one to "correlate" the event, such as: -// - appId - unique id of the workflow instance -// - procId - process id (if the event is related to a process) -// - firingId - number of firing of the process (if the event is related to a particular firing) -// - sigId - signal id (if the event is related to a signal) -// - time - time stamp From e168ea2f12c7050eb67d85d01cfed9be1aab0319 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 24 Feb 2014 22:26:42 +0100 Subject: [PATCH 17/73] Change eventlog to return new eventserver instance on request. --- eventlog/index.js | 52 +++++++++++++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/eventlog/index.js b/eventlog/index.js index a715526..1320e32 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -1,34 +1,42 @@ -var EventEmitter2 = require('eventemitter2').EventEmitter2, - server = new EventEmitter2({ - wildcard: true, // should the event emitter use wildcards. - delimiter: '.', // the delimiter used to segment namespaces, defaults to `.`. - newListener: false, // if you want to emit the newListener event set to true. - maxListeners: 20 // the max number of listeners that can be assigned to an event, defaults to 10. +var EventEmitter2 = require('eventemitter2').EventEmitter2 + +var EventServer = function () { + var server = new EventEmitter2({ + wildcard: true, // should the event emitter use wildcards. + delimiter: '.', // the delimiter used to segment namespaces, defaults to `.`. + newListener: false, // if you want to emit the newListener event set to true. + maxListeners: 20 // the max number of listeners that can be assigned to an event, defaults to 10. }); +}; -function emit(ev, data) { +EventServer.prototype.emit = function (ev, data) { data.time = new Date().toISOString(); // automatically add a timestamp to event data - server.emit(ev, data); -} + this.emit(ev, data); +}; + +EventServer.prototype.on = function (ev, listener) { + this.on(ev, listener); +}; -function on(ev, listener) { - server.on(ev, listener); -} +function createEventServer() { + var eventServer = new EventServer(); // this object is exported as the API of the event logger (hides implementation) -var eventlog = { - emit: emit, - on: on + var eventlog = { + emit: eventServer.emit, + on: eventServer.on + } +// Here's how to subscribe to events: + eventlog.on('trace.*', function (data) { + console.log(arguments); + // "this.event" contains the full event name + console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); + }); + return eventlog; } -// Here's how to subscribe to events: -server.on('trace.*', function(data) { - console.log(arguments); - // "this.event" contains the full event name - console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); -}); -exports.eventlog = eventlog; +exports.createEventlog = createEventServer(); // Example use of the event logger from another module: // eventlog = require('../eventlog').eventlog, From fbfcedf81fa131879d0c9b7ccc4479a5c788b7ef Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 24 Feb 2014 22:27:46 +0100 Subject: [PATCH 18/73] Update usage instructions for eventlog. --- eventlog/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eventlog/index.js b/eventlog/index.js index 1320e32..ffdcbf2 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -39,8 +39,10 @@ function createEventServer() { exports.createEventlog = createEventServer(); // Example use of the event logger from another module: -// eventlog = require('../eventlog').eventlog, -// ... +// eventlog = require('../eventlog').createEventLog +// +// //embedd eventlog in well known place, then call +// // eventlog.emit('trace.invokingFunction', {"appId": wfId, "procId": procId }); // // We should agree on a set of standard fields that allow one to "correlate" the event, such as: From 1893d00e0ca3e14aaf2c867a48f280f9a1868c2a Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 24 Feb 2014 22:59:36 +0100 Subject: [PATCH 19/73] Added test for new EventServer. --- eventlog/index.js | 2 +- tests/eventlog.js | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 tests/eventlog.js diff --git a/eventlog/index.js b/eventlog/index.js index ffdcbf2..63a9065 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -25,7 +25,7 @@ function createEventServer() { var eventlog = { emit: eventServer.emit, on: eventServer.on - } + }; // Here's how to subscribe to events: eventlog.on('trace.*', function (data) { console.log(arguments); diff --git a/tests/eventlog.js b/tests/eventlog.js new file mode 100644 index 0000000..5ee439c --- /dev/null +++ b/tests/eventlog.js @@ -0,0 +1,14 @@ +var eventServer = require('../eventlog') + +exports.setUp = function(callback) { + //get one instance of log server + this.logServer = eventServer.createEventlog(); + + callback(); +} + +exports.emit_simple_event = function(test) { + this.eventServer.emit("trace.data", {'key': "value"}); + + test.done(); +} \ No newline at end of file From 4a03b0a7feb6098c7602d550508d2d762615a4c7 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Feb 2014 16:30:23 +0100 Subject: [PATCH 20/73] Update .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 70acd70..182ae20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ *~ *.swp +.idea From ee09c6b469233c2d639bf16a9cdb8a1064649bdf Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Feb 2014 16:43:24 +0100 Subject: [PATCH 21/73] Add nodeunit dependency. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9f4fc1b..8356a56 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,6 @@ "file": "0.2.x", "value": "0.3.0", "form-data": "", - "nodetest": "" + "nodeunit": "" } } From 7efcd517fc612798ebf57586aadcfebd73896c1b Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Feb 2014 17:19:27 +0100 Subject: [PATCH 22/73] New implementation of eventLog --- eventlog/index.js | 28 ++++++++++++++++------------ tests/eventlog.js | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/eventlog/index.js b/eventlog/index.js index 63a9065..798e014 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -1,7 +1,7 @@ var EventEmitter2 = require('eventemitter2').EventEmitter2 var EventServer = function () { - var server = new EventEmitter2({ + this.server = new EventEmitter2({ wildcard: true, // should the event emitter use wildcards. delimiter: '.', // the delimiter used to segment namespaces, defaults to `.`. newListener: false, // if you want to emit the newListener event set to true. @@ -11,32 +11,36 @@ var EventServer = function () { EventServer.prototype.emit = function (ev, data) { data.time = new Date().toISOString(); // automatically add a timestamp to event data - this.emit(ev, data); + this.server.emit(ev, data); }; EventServer.prototype.on = function (ev, listener) { - this.on(ev, listener); + this.server.on(ev, listener); }; function createEventServer() { var eventServer = new EventServer(); -// this object is exported as the API of the event logger (hides implementation) - var eventlog = { - emit: eventServer.emit, - on: eventServer.on +// this object is exported as the API of the event logger (hides implementation) is it really needed? + var eventLog = { + emit: function(key, data) { + eventServer.emit(key, data) + }, + on: function(key, cb) { + eventServer.on(key, cb); + } }; + // Here's how to subscribe to events: - eventlog.on('trace.*', function (data) { + eventServer.on('trace.*', function (data) { console.log(arguments); // "this.event" contains the full event name console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); }); - return eventlog; + return eventLog; } - -exports.createEventlog = createEventServer(); +exports.createEventServer = createEventServer; // Example use of the event logger from another module: // eventlog = require('../eventlog').createEventLog @@ -50,4 +54,4 @@ exports.createEventlog = createEventServer(); // - procId - process id (if the event is related to a process) // - firingId - number of firing of the process (if the event is related to a particular firing) // - sigId - signal id (if the event is related to a signal) -// - time - time stamp +// - time - time stamp \ No newline at end of file diff --git a/tests/eventlog.js b/tests/eventlog.js index 5ee439c..e26a9e0 100644 --- a/tests/eventlog.js +++ b/tests/eventlog.js @@ -2,7 +2,7 @@ var eventServer = require('../eventlog') exports.setUp = function(callback) { //get one instance of log server - this.logServer = eventServer.createEventlog(); + this.eventServer = eventServer.createEventServer(); callback(); } From 2fddce9e7bbc3d0ef9467429723a5d005abb13f1 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Feb 2014 17:21:33 +0100 Subject: [PATCH 23/73] Fix bad rename. --- eventlog/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eventlog/index.js b/eventlog/index.js index 798e014..fc60d11 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -23,16 +23,16 @@ function createEventServer() { // this object is exported as the API of the event logger (hides implementation) is it really needed? var eventLog = { - emit: function(key, data) { + emit: function (key, data) { eventServer.emit(key, data) }, - on: function(key, cb) { + on: function (key, cb) { eventServer.on(key, cb); } }; // Here's how to subscribe to events: - eventServer.on('trace.*', function (data) { + eventLog.on('trace.*', function (data) { console.log(arguments); // "this.event" contains the full event name console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); From e4ac2829ab7bac0a010e658649728381529d836a Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Feb 2014 17:26:55 +0100 Subject: [PATCH 24/73] Wrapped methods now accept variable list of arguments. --- eventlog/index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eventlog/index.js b/eventlog/index.js index fc60d11..872856a 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -23,11 +23,11 @@ function createEventServer() { // this object is exported as the API of the event logger (hides implementation) is it really needed? var eventLog = { - emit: function (key, data) { - eventServer.emit(key, data) + emit: function () { + eventServer.emit.apply(eventServer, arguments) }, - on: function (key, cb) { - eventServer.on(key, cb); + on: function () { + eventServer.on.apply(eventServer, arguments) } }; @@ -43,7 +43,7 @@ function createEventServer() { exports.createEventServer = createEventServer; // Example use of the event logger from another module: -// eventlog = require('../eventlog').createEventLog +// eventlog = require('../eventlog').createEventLog() // // //embedd eventlog in well known place, then call // From 75d284ff0a2d46b94a548873fd374dfa4c33f1b6 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Tue, 25 Feb 2014 17:42:24 +0100 Subject: [PATCH 25/73] Implementation of a HyperFlow server providing REST API for workflows. Implemented API operations in this version: - creation of a new workflow instance - sending signals to a workflow instance --- app.js | 315 +++++++++++++++++++++++------------- scripts/restapi_test.sh | 28 ++++ scripts/runwf.js | 2 + wflib/index.js | 31 ++-- workflows/Wf_Rest_test.json | 27 ++++ 5 files changed, 283 insertions(+), 120 deletions(-) create mode 100644 scripts/restapi_test.sh create mode 100644 workflows/Wf_Rest_test.json diff --git a/app.js b/app.js index 2f58174..90ea295 100644 --- a/app.js +++ b/app.js @@ -1,6 +1,8 @@ /* -** Hypermedia workflow -** Author: Bartosz Balis (2012) +** HyperFlow engine +** Author: Bartosz Balis (2012-2014) +** +** HyperFlow server implementing the REST API for HyperFlow workflows. */ 'use strict'; @@ -42,7 +44,7 @@ app.configure(function() { app.engine('ejs', cons.ejs); app.set('views', __dirname + '/views'); app.set('view engine', 'ejs'); - app.use(express.bodyParser()); + app.use(express.bodyParser({strict: false})); app.use(express.methodOverride()); app.use(app.router); app.use(express.static(__dirname + '/public')); @@ -60,6 +62,194 @@ app.configure('production', function() { app.use(express.errorHandler()); }); +///////////////////////////////////////////////////////////////// +//// REST API for HyperFlow workflows //// +///////////////////////////////////////////////////////////////// + +// returns a list of all workflow instances (aka 'apps') +app.get('/apps', function(req, res) { + var renderHTML = function() { + var ctype = acceptsXml(req); + res.header('content-type', ctype); + res.send('GET /apps'); + //res.render ... TODO + } + var renderJSON = function() { + res.header('content-type', 'text/plain'); + res.send('GET /apps'); + // res.render ... TODO + } + res.format({ + 'text/html': renderHTML, + 'application/json': renderJSON + }); +}); + +// creates a new workflow instance (app) +// body must be a valid workflow description in JSON +app.post('/apps', function(req, res) { + var wfJson = req.body; + var baseUrl = ''; + //onsole.log(wfJson); + + // FIXME: validate workflow description + // FIXME: add proper/more detailed error info instead of "badRequest(res)" + wflib.createInstance(wfJson, baseUrl, function(err, appId) { + if (err) return badRequest(res); + engine[appId] = new Engine({"emulate": "false"}, wflib, appId, function(err) { + if (err) return badRequest(res); + engine[appId].runInstance(function(err) { + if (err) return badRequest(res); + res.header('Location', req.url + '/' + appId); + res.send(201, null); + //res.redirect(req.url + '/' + appId, 302); + // TODO: implement sending all input signals (just like -s flag in runwf.js) + }); + }); + }); +}); + +// returns workflow instance (app) info +app.get('/apps/:i', function(req, res) { + var renderHTML = function() { + var ctype = acceptsXml(req); + res.header('content-type', ctype); + res.send('GET /apps/{appId}'); + //res.render ... TODO + } + var renderJSON = function() { + res.header('content-type', 'text/plain'); + res.send('GET /apps/{appId}'); + // res.render ... TODO + } + res.format({ + 'text/html': renderHTML, + 'application/json': renderJSON + }); +}); + +// emits a signal to a workflow +// body must be a valid signal representation, such as: +// { +// "name": +// : +// "data": [ sig(s) data ] +// } +// - attribute 'name' is mandatory and must be equal to a signal name in the target wf +// - other attributes (including actual signal data) are optional +// - if the 'data' array contains multiple elements, multiple signals will be emitted +app.post('/apps/:i', function(req, res) { + var appId = req.params.i; + if (!(appId in engine)) return notfound(res); // 404 + + var ctype = req.headers["content-type"]; + var sigValue; + if (ctype == "application/json") { + sigValue = req.body; + } else if (ctype == "application/x-www-form-urlencoded") { + sigValue = req.body; + } + console.log(ctype); + console.log(sigValue); + console.log(sigValue.name); + console.log(req.headers); + if (!("name" in sigValue)) return badrequest(res); + + var sigName = sigValue.name; + wflib.getSigByName(appId, sigName, function(err, sigId) { + if (err) return badrequest(res); // FIXME: add detailed error info + sigValue._id = sigId; + console.log(sigValue); + engine[appId].emitSignals([ sigValue ], function(err) { + if (err) return badrequest(res); // FIXME: add detailed error info + res.header('content-type', 'text/plain'); + res.send('Emit signal OK!'); + }); + }); + +}); + + +// returns a list of signals consumed/emitted by the workflow +app.get('/apps/:i/sigs', function(req, res) { + var renderHTML = function() { + var ctype = acceptsXml(req); + res.header('content-type', ctype); + //res.render ... TODO + } + var renderJSON = function() { + // TODO + } + res.format({ + 'text/html': renderHTML, + 'application/json': renderJSON + }); +}); + + +// returns a list of input signals for the workflow +app.get('/apps/:i/ins', function(req, res) { + var wfId = req.params.i; + var wfInsInfo; + + var renderHTML = function() { + var ctype = acceptsXml(req); + res.header('content-type', ctype); + //res.send(wfInsInfo); + res.render('workflow-inputs', { + title: 'workflow inputs', + wfname: 'Workflow', + wfins: wfInsInfo, + submit_ins_uri: req.url + }); + } + + var renderJSON = function() { + res.header('content-type', 'application/json'); + res.send(wfInsInfo); + } + + wflib.getWfIns(wfId, false, function(err, wfIns) { + wflib.getSignalInfo(wfId, wfIns, function(err, sigsInfo) { + wfInsInfo = sigsInfo; + res.format({ + 'text/html': renderHTML, + 'application/json': renderJSON + }); + }); + }); +}); + +// returns info about a signal exchanged within the workflow +app.get('/apps/:i/sigs/:j', function(req, res) { + var wfId = req.params.i, dataId = req.params.j; + wflib.getDataInfoFull(wfId, dataId, function(err, wfData, dSource, dSinks) { + if (err) { + res.statusCode = 404; + res.send(inst.toString()); + } else { + var ctype = acceptsXml(req); + res.header('content-type', ctype); + res.render('workflow-data', { + title: 'workflow data', + wfname: req.params.w, + data: wfData, + source: dSource, + data_id: dataId, + sinks: dSinks + }); + } + }); +}); + + +//////////////////////////////////////////////////////////////////////// +//// REST API (END) ///// +//////////////////////////////////////////////////////////////////////// + + + + /* validate user (from db) via HTTP Basic Auth */ function validateUser(req, res, next) { @@ -402,114 +592,6 @@ app.post('/workflow/:w/instances/:i/data-:j', function(req, res) { } }); -//////////////////////////////////////////////////////////////////////// -//// NEW HTTP API for workflows (experimental) -- in JSON and HTML ///// -//////////////////////////////////////////////////////////////////////// - - -// returns a list of input signals for the workflow -app.get('/workflows/:i/inputs', function(req, res) { - var wfId = req.params.i; - var wfInsInfo; - - var renderHTML = function() { - var ctype = acceptsXml(req); - res.header('content-type', ctype); - //res.send(wfInsInfo); - res.render('workflow-inputs', { - title: 'workflow inputs', - wfname: 'Workflow', - wfins: wfInsInfo, - submit_ins_uri: req.url - }); - } - - var renderJSON = function() { - res.header('content-type', 'application/json'); - res.send(wfInsInfo); - } - - wflib.getWfIns(wfId, false, function(err, wfIns) { - wflib.getSignalInfo(wfId, wfIns, function(err, sigsInfo) { - wfInsInfo = sigsInfo; - res.format({ - 'text/html': renderHTML, - 'application/json': renderJSON - }); - }); - }); -}); - -// returns info about a signal exchanged within the workflow -app.get('/workflows/:i/sig-:j', function(req, res) { - var wfId = req.params.i, dataId = req.params.j; - wflib.getDataInfoFull(wfId, dataId, function(err, wfData, dSource, dSinks) { - if (err) { - res.statusCode = 404; - res.send(inst.toString()); - } else { - var ctype = acceptsXml(req); - res.header('content-type', ctype); - res.render('workflow-data', { - title: 'workflow data', - wfname: req.params.w, - data: wfData, - source: dSource, - data_id: dataId, - sinks: dSinks - }); - } - }); -}); - -// posts a signal to the workflow -app.post('/workflows/:i/sig-:j', function(req, res) { - var wfId = req.params.i, dataId = req.params.j; - var ctype = req.headers["Content-Type"]; - var sigValue; - if (ctype == "application/json") { - sigValue = req.body; - - } else if (ctype == "application/x-www-form-urlencoded") { - sigValue = req.body; - } - console.log(sigValue); - - // send the signal - // TODO - - /*if (req.body['data-value']) { - var spec = {}; - spec[dataId] = {"value": req.body['data-value']}; - wflib.setDataState(wfId, spec, function(err, rep) { - engine[wfId].markDataReady(dataId, function(err) { - if (err) { - res.statusCode = 404; - res.send(err.toString()); - } else { - res.redirect(req.url, 302); - } - }); - }); - } else { - engine[wfId].markDataReady(dataId, function(err) { - if (err) { - res.statusCode = 404; - res.send(err.toString()); - } else { - res.redirect(req.url, 302); - } - }); - }*/ -}); - -//////////////////////////////////////////////////////////////////////// -//// NEW HTTP API (END) ///// -//////////////////////////////////////////////////////////////////////// - - - - /* support various content-types from clients */ @@ -605,6 +687,17 @@ function forbidden(res) { res.end(body); } +// 404 response +function notfound(res) { + var body = 'Resource not found (404)'; + + res.setHeader('Content-Type', 'text/plain'); + res.setHeader('Content-Length', body.length); + res.statusCode = 404; + res.end(body); +} + + /* return standard 'auth required' response */ function authRequired(res, realm) { diff --git a/scripts/restapi_test.sh b/scripts/restapi_test.sh new file mode 100644 index 0000000..f613312 --- /dev/null +++ b/scripts/restapi_test.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +if [ "$#" -ne 1 ]; then + echo "restapi_test.sh: creates an instance of the Ping Pong workflow" + echo " and sends a signal to it using the HyperFlow REST API." + echo + echo "Usage:" + echo "- first run the HyperFlow server: node app.js" + echo "- then run this script: scripts/restapi_test.sh " + echo "where is the port number on which the server is running" + exit +fi + + +uri="http://localhost:$1/apps" + +# POST {host}/apps - creates a workflow instance +# Body: valid workflow description in JSON +# returns 201, Location: {appuri} +# reads "location" from the HTTP header of the response +location=`curl -v -X POST -d @workflows/Wf_Rest_test.json $uri --header "Content-Type:application/json" 2>&1 | grep Location | cut -f 3 -d' '` + +appuri="http://localhost:$1"$location +echo $appuri + +# POST {appuri} - sends a signal to a workflow +# Body: valid signal data (JSON with mandatory "name") +curl -X POST -d '{ "name": "counter1", "data": [0] }' $appuri --header "Content-Type:application/json" diff --git a/scripts/runwf.js b/scripts/runwf.js index 301eb5b..d62cd59 100644 --- a/scripts/runwf.js +++ b/scripts/runwf.js @@ -34,6 +34,8 @@ if (!argv.id && !argv.f) { console.log(" -i WFID : use already created wf instance with WFID as its Redis id"); console.log(" -s : send input signals to the workflow (starts execution)"); console.log(" -d DBID : Redis db number to be used (default=0)"); + console.log(" --cfg [--cfg ...] : configuration files"); + console.log(" will create a config JSON object { conf1: {...}, conf2: {...} }"); process.exit(); } diff --git a/wflib/index.js b/wflib/index.js index c3a5507..312aa25 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -45,10 +45,10 @@ exports.init = function(redisClient) { var wfname = filename.split('.')[0]; rcl.hmset("wftempl:"+wfname, "name", wfname, "maxInstances", "3", function(err, ret) { var start = (new Date()).getTime(), finish; - public_createInstance(JSON.parse(data), baseUrl, function(err, ret) { + public_createInstance(JSON.parse(data), baseUrl, function(err, wfId) { finish = (new Date()).getTime(); console.log("createInstance time: "+(finish-start)+"ms"); - err ? cb(err): cb(null, ret); + err ? cb(err): cb(null, wfId); }); }); }); @@ -120,7 +120,7 @@ exports.init = function(redisClient) { var addSigInfo = function(sigId) { var score = -1; var sigObj = sigs[sigId-1]; - sigObj.status = "not_ready"; + sigObj.status = "not_ready"; // FIXME: remove (deprecated) sigKey = wfKey+":data:"+sigId; if (sigObj.control) { // this is a control signal sigObj.type = "control"; @@ -129,7 +129,7 @@ exports.init = function(redisClient) { } else { // this is a data signal score = 0; } - sigObj.uri = baseUri + '/signals/' + sigId; + sigObj.uri = baseUri + '/sigs/' + sigId; if (sigObj.schema && value(sigObj.schema).typeOf(Object)) { // this is an inline schema //onsole.log("INLINE SCHEMA", sigObj.schema); @@ -146,15 +146,18 @@ exports.init = function(redisClient) { delete sigObj.data; // don't store instances in signal info } + // create a reverse index to look up sig Id by its name (assumes unique names!) + multi.hset(wfKey+":siglookup:name", sigObj.name, sigId, function(err, ret) { }); + multi.hmset(sigKey, sigObj, function(err, ret) { }); - // add this data id to the sorted set of all workflow signals + // add this signal id to the sorted set of all workflow signals // score determines the type/status of the signal: // 0: data signal/not ready, 1: data signal/ready, 2: control signal multi.zadd(wfKey+":data", score, sigId, function(err, ret) { }); } - // add workflow tasks + // add workflow processes var taskKey; for (var i=0; i Date: Tue, 25 Feb 2014 18:01:09 +0100 Subject: [PATCH 26/73] Adapt HF to new EventLog. --- engine/index.js | 4 ++-- eventlog/index.js | 13 +++++++++---- functions/command.js | 6 +++--- scripts/runwf.js | 6 +++--- wflib/index.js | 2 +- 5 files changed, 18 insertions(+), 13 deletions(-) diff --git a/engine/index.js b/engine/index.js index f2117f9..10f5f2b 100644 --- a/engine/index.js +++ b/engine/index.js @@ -16,7 +16,7 @@ var fs = require('fs'), xml2js = require('xml2js'), fsm = require('./automata.js'), async = require('async'), - EventEmitter2 = require('eventemitter2').EventEmitter2; + eventServerFactory = require('../eventlog'); var TaskDataflowFSM = require('./taskDataflowFSM.js'); var TaskForeachFSM = require('./taskForeachFSM.js'); @@ -59,7 +59,7 @@ fsm.registerFSM(TaskChoiceFSM); // - config.emulate (true/false) = should engine work in the emulation mode? var Engine = function(config, wflib, wfId, cb) { this.wflib = wflib; - this.eventServer = new EventEmitter2({wildcard:true}); + this.eventServer = eventServerFactory.createEventServer();; this.wfId = wfId; this.tasks = []; // array of task FSMs this.ins = []; diff --git a/eventlog/index.js b/eventlog/index.js index 872856a..1114e63 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -15,7 +15,12 @@ EventServer.prototype.emit = function (ev, data) { }; EventServer.prototype.on = function (ev, listener) { - this.server.on(ev, listener); + var dump = function() { + console.log(arguments) +// listener.apply(arguments); + } +// this.server.on(ev, listener); + this.server.on(ev, dump); }; function createEventServer() { @@ -32,10 +37,10 @@ function createEventServer() { }; // Here's how to subscribe to events: - eventLog.on('trace.*', function (data) { - console.log(arguments); + eventLog.on('trace.*', function (data, data2) { +// console.log(data); // "this.event" contains the full event name - console.log("EVENT:", this.event, JSON.stringify(data, null, 2)); +// console.log("EVENT:", this.event, JSON.stringify(arguments, null, 2)); }); return eventLog; } diff --git a/functions/command.js b/functions/command.js index 7c376e5..325a2aa 100644 --- a/functions/command.js +++ b/functions/command.js @@ -39,9 +39,9 @@ function command_notifyevents(ins, outs, config, cb) { var exec = config.executor.executable, args = config.executor.args; - var eventserver = config['eventserver']; - if(typeof eventserver !== 'undefined' && eventserver) { - eventserver.emit("job.done", exec, args); + var eventServer = config['eventServer']; + if(typeof eventServer !== 'undefined' && eventServer) { + eventServer.emit("trace.job", exec, args); } else { console.log("loged: " + exec, args); } diff --git a/scripts/runwf.js b/scripts/runwf.js index bacc0cf..bd744e1 100644 --- a/scripts/runwf.js +++ b/scripts/runwf.js @@ -45,9 +45,9 @@ if (argv.d) { var runWf = function(wfId) { engine = new Engine({"emulate":"false"}, wflib, wfId, function(err) { //This represent custom plugin listening on event from available eventServer - engine.eventServer.on('job.done', function(exec, args) { - console.log('Event captured: ' + exec + ' ' + args + ' job done'); - }); +// engine.eventServer.on('trace.*', function(exec, args) { +// console.log('Event captured: ' + exec + ' ' + args + ' job done'); +// }); engine.runInstance(function(err) { console.log("Wf id="+wfId); if (argv.s) { diff --git a/wflib/index.js b/wflib/index.js index 4456089..d249f84 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -1362,7 +1362,7 @@ function public_invokeTaskFunction2(wfId, taskId, insIds_, insValues, outsIds_, //onsole.log("INS VALUES", insValues); if (eventServer !== 'undefined') { - conf['eventserver'] = eventServer; + conf['eventServer'] = eventServer; } f(ins, outs, conf, function(err, outs) { From 3f743c4a4402cde8f6a449be9312ab1cee089996 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Feb 2014 18:24:39 +0100 Subject: [PATCH 27/73] Fix emitting variable number of args. --- eventlog/index.js | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/eventlog/index.js b/eventlog/index.js index 1114e63..5604bf6 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -9,18 +9,15 @@ var EventServer = function () { }); }; -EventServer.prototype.emit = function (ev, data) { - data.time = new Date().toISOString(); // automatically add a timestamp to event data - this.server.emit(ev, data); +EventServer.prototype.emit = function () { + var args = Array.prototype.slice.call(arguments); + args.splice(1, 0, new Date().toISOString()); + this.server.emit.apply(this.server, args); }; EventServer.prototype.on = function (ev, listener) { - var dump = function() { - console.log(arguments) -// listener.apply(arguments); - } // this.server.on(ev, listener); - this.server.on(ev, dump); + this.server.on(ev, listener); }; function createEventServer() { @@ -37,10 +34,9 @@ function createEventServer() { }; // Here's how to subscribe to events: - eventLog.on('trace.*', function (data, data2) { -// console.log(data); + eventLog.on('trace.*', function (data) { // "this.event" contains the full event name -// console.log("EVENT:", this.event, JSON.stringify(arguments, null, 2)); + console.log("EVENT:", this.event, JSON.stringify(arguments, null, 2)); }); return eventLog; } From 35d5dbffe45062d9bece482157d32d6f71124689 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Feb 2014 18:32:04 +0100 Subject: [PATCH 28/73] Update sample usage in comments. --- engine/index.js | 2 +- eventlog/index.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/index.js b/engine/index.js index 10f5f2b..ff04b25 100644 --- a/engine/index.js +++ b/engine/index.js @@ -59,7 +59,7 @@ fsm.registerFSM(TaskChoiceFSM); // - config.emulate (true/false) = should engine work in the emulation mode? var Engine = function(config, wflib, wfId, cb) { this.wflib = wflib; - this.eventServer = eventServerFactory.createEventServer();; + this.eventServer = eventServerFactory.createEventServer(); this.wfId = wfId; this.tasks = []; // array of task FSMs this.ins = []; diff --git a/eventlog/index.js b/eventlog/index.js index 5604bf6..484cdd1 100644 --- a/eventlog/index.js +++ b/eventlog/index.js @@ -43,7 +43,7 @@ function createEventServer() { exports.createEventServer = createEventServer; -// Example use of the event logger from another module: +// Example use of the event logger: // eventlog = require('../eventlog').createEventLog() // // //embedd eventlog in well known place, then call From bf037c2bcbfd6ca98b39118a032961e89404e080 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 26 Feb 2014 23:29:03 +0100 Subject: [PATCH 29/73] Implemented distributed workflow execution, see script "test/restapi_test.sh" --- app.js | 66 ++++++++++++++++++++++---- engine/index.js | 1 - functions/index.js | 9 ++-- package.json | 4 +- scripts/restapi_test.sh | 8 ++-- test/remote_pingpong_test.sh | 47 +++++++++++++++++++ wflib/index.js | 90 ++++++++++++++++++++++++++++++++++-- 7 files changed, 201 insertions(+), 24 deletions(-) create mode 100644 test/remote_pingpong_test.sh diff --git a/app.js b/app.js index 90ea295..8625ed6 100644 --- a/app.js +++ b/app.js @@ -24,7 +24,7 @@ var server = http.createServer(app); var wflib = require('./wflib').init(rcl); var Engine = require('./engine'); var engine = {}; // engine.i contains the engine object for workflow instance 'i' -var urlReq = require('./req_url'); +var request = require('request'); var timers = require('timers'); @@ -85,7 +85,7 @@ app.get('/apps', function(req, res) { }); }); -// creates a new workflow instance (app) +// creates a new workflow instance ('app') // body must be a valid workflow description in JSON app.post('/apps', function(req, res) { var wfJson = req.body; @@ -109,7 +109,7 @@ app.post('/apps', function(req, res) { }); }); -// returns workflow instance (app) info +// returns workflow instance ('app') info app.get('/apps/:i', function(req, res) { var renderHTML = function() { var ctype = acceptsXml(req); @@ -149,17 +149,17 @@ app.post('/apps/:i', function(req, res) { } else if (ctype == "application/x-www-form-urlencoded") { sigValue = req.body; } - console.log(ctype); - console.log(sigValue); - console.log(sigValue.name); - console.log(req.headers); + //onsole.log(ctype); + //onsole.log(sigValue); + //onsole.log(sigValue.name); + //onsole.log(req.headers); if (!("name" in sigValue)) return badrequest(res); var sigName = sigValue.name; wflib.getSigByName(appId, sigName, function(err, sigId) { if (err) return badrequest(res); // FIXME: add detailed error info sigValue._id = sigId; - console.log(sigValue); + //onsole.log(sigValue); engine[appId].emitSignals([ sigValue ], function(err) { if (err) return badrequest(res); // FIXME: add detailed error info res.header('content-type', 'text/plain'); @@ -243,6 +243,52 @@ app.get('/apps/:i/sigs/:j', function(req, res) { }); +// returns a list of remote sinks of a signal +app.get('/apps/:i/sigs/:name/remotesinks', function(req, res) { + var appId = req.params.i; + var sigName = req.params.name; + var remoteSinks = req.body; + + var renderHTML = function(rsinks) { + var ctype = acceptsXml(req); + res.header('content-type', ctype); + res.send(200, JSON.stringify(rsinks)); + } + var renderJSON = function(rsinks) { + res.send(200, "TODO"); + // TODO + } + + wflib.getSigByName(appId, sigName, function(err, sigId) { + wflib.getSigRemoteSinks(appId, sigId, function(err, rsinks) { + renderHTML(rsinks); + /*res.format({ + 'text/html': renderHTML(rsinks), + 'application/json': renderJSON(rsinks) + });*/ + }); + }); +}); + + +// sets remote sinks for a given signal +// body: JSON array of objects: [ { "uri": uri1 }, { "uri": uri2 }, ... ] +app.put('/apps/:i/sigs/:name/remotesinks', function(req, res) { + var appId = req.params.i; + var sigName = req.params.name; + var remoteSinks = req.body; + + wflib.getSigByName(appId, sigName, function(err, sigId) { + if (err) return badrequest(res); + wflib.setSigRemoteSinks(appId, sigId, remoteSinks, { "replace": true }, function(err) { + if (err) return badrequest(res); + res.send(200, "Remote sinks set succesfully"); + }); + }); + +}); + + //////////////////////////////////////////////////////////////////////// //// REST API (END) ///// //////////////////////////////////////////////////////////////////////// @@ -372,11 +418,11 @@ app.post('/workflow/:w/instances/:i', function(req, res) { spec[id] = { "value": req.body[i] } dataIds.push(id); } - //console.log(spec); + //onsole.log(spec); } if (Object.keys(spec).length) { // not empty wflib.setDataState(wfId, spec, function(err, rep) { - //console.log(spec); + //onsole.log(spec); engine[wfId].markDataReady(dataIds, function(err) { res.redirect(req.url, 302); }); diff --git a/engine/index.js b/engine/index.js index 14e29c1..b0a0fe7 100644 --- a/engine/index.js +++ b/engine/index.js @@ -241,7 +241,6 @@ Engine.prototype.emitSignals = function(sigs, cb) { async.each(sigInstances, function(s, doneIterInner) { var _sigId = s._id; engine.wflib.sendSignal(engine.wfId, s, function(err, sinks) { - //onsole.log(sinks); if (!err) { // notify sinks that the signals have arrived for (var j=0; j&1 | grep Location | cut -f 3 -d' '` appuri="http://localhost:$1"$location echo $appuri -# POST {appuri} - sends a signal to a workflow +# 2) POST {appuri} - sends a signal to a workflow # Body: valid signal data (JSON with mandatory "name") curl -X POST -d '{ "name": "counter1", "data": [0] }' $appuri --header "Content-Type:application/json" diff --git a/test/remote_pingpong_test.sh b/test/remote_pingpong_test.sh new file mode 100644 index 0000000..d5cc802 --- /dev/null +++ b/test/remote_pingpong_test.sh @@ -0,0 +1,47 @@ +#!/bin/sh + +if [ "$#" -ne 2 ]; then + echo "remote_pingpong_test.sh: distributed and decentralized workflow execution test" + echo " using the HyperFlow REST API." + echo + echo "Usage:" + echo "- run the 1st HyperFlow server: node app.js ( -> runs on port1)" + echo "- run the 2nd HyperFlow server: node app.js ( -> runs on port2)" + echo "- then run this script: test/remote_pingpong_test.sh port1 port2" + exit +fi + +# URIs: +# {appfactory1} http://localhost:{port1}/apps +# {appfactory2} http://localhost:{port2}/apps +# {app1} http://localhost:{port1}/apps/{appId1} +# {app2} http://localhost:{port2}/apps/{appId2} + +# REST-based protocol is as follows (<...> denotes the content of message body): +# 1) POST {appfactory1} ==> create Pinger instance, returns URI {app1}. +# 2) POST {appfactory2} ==> create Ponger instance, returns URI {app2}. +# 3) PUT {app1}/sigs/Ping/remotesinks <{app2}> ==> connects signal "Ping" from app1 to app2. +# 4) PUT {app2}/sigs/Pong/remotesinks <{app1}> ==> connects signal "Pong" from app2 to app1. +# 5) POST {app1} ==> sends initial signal to Ping to start the Ping-Pong. + +appfact1="http://localhost:$1/apps" +appfact2="http://localhost:$2/apps" +echo $appfact1 +echo $appfact2 + +# 1) +app1=`curl -v -X POST -d @workflows/Wf_RemotePing.json $appfact1 --header "Content-Type:application/json" 2>&1 | grep Location | cut -f 3 -d' '` +app1uri="http://localhost:$1"$app1 + +# 2) +app2=`curl -v -X POST -d @workflows/Wf_RemotePong.json $appfact2 --header "Content-Type:application/json" 2>&1 | grep Location | cut -f 3 -d' '` +app2uri="http://localhost:$2"$app2 + +# 3) +curl -v -X PUT -d "[{ \"uri\": \"$app2uri\" }]" $app1uri/sigs/Pong/remotesinks --header "Content-Type:application/json" + +# 4) +curl -v -X PUT -d "[{ \"uri\": \"$app1uri\" }]" $app2uri/sigs/Ping/remotesinks --header "Content-Type:application/json" + +# 5) +curl -X POST -d '{ "name": "Ping", "data": [0] }' $app1uri --header "Content-Type:application/json" diff --git a/wflib/index.js b/wflib/index.js index 312aa25..572c492 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -7,6 +7,8 @@ var fs = require('fs'), async = require('async'), ZSchema = require('z-schema'), value = require('value'), + request = require('request'), + Q = require('q'), //toobusy = require('toobusy'), rcl; @@ -146,6 +148,13 @@ exports.init = function(redisClient) { delete sigObj.data; // don't store instances in signal info } + if (sigObj.remoteSinks) { // signal info contains URIs to remote sinks + sigObj.remoteSinks.forEach(function(sink) { + multi.sadd(sigKey+":remotesinks", sink.uri, function(err, ret) { }); + }); + sigObj.remoteSinks = true; // don't store remote sinks URIs in sig info, just a flag + } + // create a reverse index to look up sig Id by its name (assumes unique names!) multi.hset(wfKey+":siglookup:name", sigObj.name, sigId, function(err, ret) { }); @@ -1454,7 +1463,31 @@ function sendSignalLua(wfId, sigValue, cb) { /*var delay = 0; if (toobusy()) { onsole.log("TOO BUSY !!!!!!!!!!!!!!!!!!!!!!!!!!"); delay = 40; } setTimeout(function() { cb(err, res); }, delay);*/ - cb(err, res); + if (err) return cb(err); + + if (sigValue.remoteSinks) { + rcl.smembers(sigKey+":remotesinks", function(err, remoteSinks) { + delete sigValue.remoteSinks; + async.each(remoteSinks, function(sinkUri, doneIterCb) { + request.post({ + headers: {'content-type' : 'application/json'}, + url: sinkUri, + json: sigValue + }, function(error, response, body) { + if (error) console.log("ERROR", error); + doneIterCb(); + //console.log(error); + //console.log(response); + //console.log(body); + }); + //onsole.log("REMOTE SINKS: ", ret); + }, function doneAll(err) { + cb(err, res); + }); + }); + } else { + cb(err, res); + } }); } @@ -1493,7 +1526,6 @@ function public_sendSignal(wfId, sig, cb) { //////////////////////////////////////// SENDING THE SIGNAL: ///////////////////////////////////////// // create a new instance of this signal (at hash = "wf:{id}:sigs:{sigId}", field = sig instance id) // - // (signal with a given id may be emitted multiple times within a workflow execution) // // (hash is better than a list because of easier cleanup of old signals) // ////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -1556,6 +1588,56 @@ function public_sendSignal(wfId, sig, cb) { ); } +function getSigRemoteSinks(wfId, sigId, cb) { + var rsKey = "wf:"+wfId+":data:"+sigId+":remotesinks"; + rcl.smembers(rsKey, function(err, ret) { + cb(err, ret); + }); +} + +// sets remote sinks for a signal +// @remoteSinks = array: [ { "uri": uri1 }, { "uri": uri2 }, ... ] +// @options = object; possible values: +// { "replace": true }: if present, currently defined remote sinks will be replaced +// if not, new remote sinks will be added to existing ones +function setSigRemoteSinks(wfId, sigId, remoteSinks, options, cb) { + var replace = options && options.replace == true, + wfKey = "wf:"+wfId; + sigKey = wfKey+":data:"+sigId, + rsKey = wfKey+":data:"+sigId+":remotesinks"; + + Q.fcall(function() { + if (replace) { + rcl.del(rsKey, function(err) { + if (err) throw(err); + return; + }); + } else return; + }) + .then(function() { + async.eachSeries(remoteSinks, function(sink, doneIterCb) { + rcl.sadd(rsKey, sink.uri, function(err, ret) { + doneIterCb(err); + }); + }, function doneAll(err) { + if (err) throw(err); + return; + }); + }) + .then(function() { + rcl.hset(sigKey, "remoteSinks", true, function(err, ret) { + if (err) throw(err); + return; + }); + }) + .catch(function(error) { + cb(error); + }) + .done(function() { + cb(null); + }); +} + // Part of NEW API for continuous processes with FIFO queues // @sig format: // ... TODO @@ -1856,7 +1938,9 @@ return { getInitialSigs: getInitialSignals, sendSignalLua: sendSignalLua, sendSignalSinks: sendSignalSinks, - getSigByName: getSigByName + getSigByName: getSigByName, + getSigRemoteSinks: getSigRemoteSinks, + setSigRemoteSinks: setSigRemoteSinks }; }; From b4beb87566cfd2f8c0401ee695e2c07c2e70fe6a Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 26 Feb 2014 23:31:50 +0100 Subject: [PATCH 30/73] Workflows used in the distributed workflow execution test. --- workflows/Wf_RemotePing.json | 21 +++++++++++++++++++++ workflows/Wf_RemotePong.json | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 workflows/Wf_RemotePing.json create mode 100644 workflows/Wf_RemotePong.json diff --git a/workflows/Wf_RemotePing.json b/workflows/Wf_RemotePing.json new file mode 100644 index 0000000..397b038 --- /dev/null +++ b/workflows/Wf_RemotePing.json @@ -0,0 +1,21 @@ +{ + "name": "Wf_RemotePing", + "functions": [ { + "name": "count", + "module":"functions" + } ], + "processes": [ { + "name": "Pinger", + "type": "dataflow", + "function": "count", + "ins": [ "Ping" ], + "outs": [ "Pong" ] + } ], + "signals": [ { + "name": "Ping" + }, { + "name": "Pong" + } ], + "ins": [ "Ping" ], + "outs": [ "Pong" ] +} diff --git a/workflows/Wf_RemotePong.json b/workflows/Wf_RemotePong.json new file mode 100644 index 0000000..80f1ab1 --- /dev/null +++ b/workflows/Wf_RemotePong.json @@ -0,0 +1,21 @@ +{ + "name": "Wf_RemotePong", + "functions": [ { + "name": "count", + "module":"functions" + } ], + "processes": [ { + "name": "Ponger", + "type": "dataflow", + "function": "count", + "ins": [ "Pong" ], + "outs": [ "Ping" ] + } ], + "signals": [ { + "name": "Ping" + }, { + "name": "Pong" + } ], + "ins": [ "Pong" ], + "outs": [ "Ping" ] +} From 973333f7d8dfdb2ca4d2ed52af3d88d4e9e1a3d5 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 26 Feb 2014 23:42:15 +0100 Subject: [PATCH 31/73] Minor update. --- test/remote_pingpong_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/remote_pingpong_test.sh b/test/remote_pingpong_test.sh index d5cc802..b1280ea 100644 --- a/test/remote_pingpong_test.sh +++ b/test/remote_pingpong_test.sh @@ -22,7 +22,7 @@ fi # 2) POST {appfactory2} ==> create Ponger instance, returns URI {app2}. # 3) PUT {app1}/sigs/Ping/remotesinks <{app2}> ==> connects signal "Ping" from app1 to app2. # 4) PUT {app2}/sigs/Pong/remotesinks <{app1}> ==> connects signal "Pong" from app2 to app1. -# 5) POST {app1} ==> sends initial signal to Ping to start the Ping-Pong. +# 5) POST {app1} ==> sends the initial signal to Pinger to start the Ping-Pong. appfact1="http://localhost:$1/apps" appfact2="http://localhost:$2/apps" From 7c3615360302bceb00bf89d3cce91b9862ec7a26 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Thu, 6 Mar 2014 08:51:01 +0100 Subject: [PATCH 32/73] Refactoring of engine + implementation of a new 'Join' process. --- engine2/ProcChoiceFSM.js | 192 ++++++ engine2/ProcDataflowFSM.js | 160 +++++ engine2/ProcForeachFSM.js | 186 ++++++ engine2/ProcJoinFSM.js | 270 ++++++++ engine2/ProcSplitterFSM.js | 330 ++++++++++ engine2/automata.js | 1265 ++++++++++++++++++++++++++++++++++++ engine2/index.js | 340 ++++++++++ engine2/process.js | 312 +++++++++ 8 files changed, 3055 insertions(+) create mode 100644 engine2/ProcChoiceFSM.js create mode 100644 engine2/ProcDataflowFSM.js create mode 100644 engine2/ProcForeachFSM.js create mode 100644 engine2/ProcJoinFSM.js create mode 100644 engine2/ProcSplitterFSM.js create mode 100644 engine2/automata.js create mode 100644 engine2/index.js create mode 100644 engine2/process.js diff --git a/engine2/ProcChoiceFSM.js b/engine2/ProcChoiceFSM.js new file mode 100644 index 0000000..4c9f5c1 --- /dev/null +++ b/engine2/ProcChoiceFSM.js @@ -0,0 +1,192 @@ +/* HyperFlow engine. + ** Author: Bartosz Balis (2013-2014) + ** + ** Implementation of a 'choice' process. This process waits for all data inputs, invokes the process's function, + ** and emits **any subset** of data outputs, enabling conditional execution patterns. + ** + ** Inputs: + ** - a number of data ports x1 .. xn + ** Outputs: + ** - a number of data ports y1 .. ym + ** Function: + ** - The function determines which of the m outputs should be emitted. To this end, the function + ** should insert a pair << "condition": "true" >> into the json objects of appropriate output + ** data elements. + ** + */ + +var async = require('async'), + log4js = require('log4js'), + ProcLogic = require('./process.js').ProcLogic, + fireInput = require('./process.js').fireInput, + extend = require('./process.js').extend; + + //log4js.configure('log4js.json'); + + //var logger = log4js.getLogger('hf-default-logger'); + + +var ChoiceLogic = function() { + ProcLogic.call(this); + + this.init2 = function() { + console.log("MISTERSON"); + // set firing sigs + for (var i in this.ins) { + var sigId = this.ins[i]; + if (!(sigId in this.ctrIns)) { + this.firingSigs.push([sigId, 1]); + } + } + if ("next" in this.ctrIns) { + this.firingSigs.push([this.ctrIns.next,1]); + } + //onsole.log(JSON.stringify(this.firingSigs)); + } + + + this.ready_enter = function(session, state, transition, msg) { + var proc = session.logic; + proc.ready = true; + proc.tryTransition(proc, session); + + //onsole.log("Enter state ready: "+task.procId); + }; + + this.running_enter = function(session, state, transition, msg) { + //onsole.log("Enter state running: "+this.procId+" ("+this.name+")"); + var proc = this; + var funcIns = [], funcOuts = [], emul = proc.engine.emulate; + + proc.firingId += 1; + + var firingId = proc.firingId, + firingSigs = proc.firingSigs, + sigValues = proc.sigValues, + asyncInvocation; + + async.waterfall([ + // 1. pre-invoke: check firing limits, set proc state to running, etc. + function(cb) { + proc.preInvoke(cb); + }, + // 2. invoke the function + function(cb) { + proc.invokeFunction(cb); + }, + // 3. post-invoke: emit output signals + function(outs, wasAsync, funcIns, funcOuts, cb) { + asyncInvocation = wasAsync; + proc.postInvoke(outs, asyncInvocation, funcIns, funcOuts, firingId, firingSigs, cb); + }, + // 4. make state transition from the running state + function(cb) { + proc.postInvokeTransition(asyncInvocation, cb); + } + ]); + }; + + this.running_exit = function(session, state, transition, msg) { + //onsole.log("Exit state running: "+this.procId+" ("+this.name+")"); + }; + + this.finished_enter = function(session, state, transition, msg) { + var proc = this; + proc.wflib.setTaskState(proc.appId, proc.procId, { "status": "finished" }, function(err, rep) { + if (err) { throw err; } + //onsole.log("Enter state finished: "+proc.procId); + proc.engine.taskFinished(proc.procId); + }); + }; + + this.ReRutransition = function(session, state, transition, msg) { + //onsole.log("Transition ReRu, task "+this.procId); + }; + + this.ReFitransition = function(session, state, transition, msg) { + // emit "done" signal if such a port exists + if (this.ctrOuts.done) { + session.logic.engine.emitSignals([{ "_id": session.ctrOuts.done }]); + } + }; + + this.RuRetransition = function(session, state, transition, msg) { }; + + // overrides postInvoke in order to emit only signals with "condition" flag. + this.postInvoke = function(outs, asyncInvocation, funcIns, funcOuts, firingId, firingSigs, cb) { + var proc = this; + //proc.cnt -= proc.firingSigs.length; // subtract cnt by the number of consumed signals + //if (proc.fullInfo.sticky) + // proc.cnt += proc.fullInfo.sticky; // sticky signals weren't actually consumed! + + var outValues = []; + for (var i=0; i 0) { + proc.engine.emitSignals(outValues, function(err) { + proc.runningCount -= 1; + //onsole.log("runningCount (" + proc.fullInfo.name + ")/2:", proc.runningCount); + err ? cb(err): cb(null); + }); + } + } + + return this; +} + +extend(ChoiceLogic, ProcLogic); + +var ProcChoiceFSM = { + name: "choice", + logic: ChoiceLogic, + + state : [ + { + name: "init", + initial: "true", + onEnter: function (session) { session.dispatch({msgId: "InitRe"}); } + }, { + name: "ready", + }, { + name: "running", + }, { + name: "finished", + } + ], + + transition : [ + { + event : "InitRe", + from : "init", + to : "ready", + onTransition: function() { } + }, { + event : "ReRu", // when all data ins and 'next' (if such port exists) have arrived + from : "ready", + to : "running", + }, { + event : "ReFi", // when 'done' signal has arrived + from : "ready", + to : "finished", + }, { + event : "RuRe", // after firing + from : "running", + to : "ready", + } + ] +}; + +module.exports = ProcChoiceFSM; diff --git a/engine2/ProcDataflowFSM.js b/engine2/ProcDataflowFSM.js new file mode 100644 index 0000000..0095c74 --- /dev/null +++ b/engine2/ProcDataflowFSM.js @@ -0,0 +1,160 @@ + /* HyperFlow engine. + ** Author: Bartosz Balis (2013-2014) + ** + ** Implementation of a basic 'dataflow' process. This process waits for all input signals, + ** invokes the process's function, and emits all data output signals. + ** + ** Inputs: + ** - ... + ** Outputs: + ** - ... + ** Function: + ** - ... + ** + ** TODO: + ** - Implement 'firing rules' to specify how many data elements ('tokens') need to arrive on + ** a given input port + */ + +var async = require('async'), + log4js = require('log4js'), + ProcLogic = require('./process.js').ProcLogic, + fireInput = require('./process.js').fireInput, + extend = require('./process.js').extend; + + //log4js.configure('log4js.json'); + + //var logger = log4js.getLogger('hf-default-logger'); + + +var DataflowLogic = function() { + ProcLogic.call(this); + + this.init2 = function() { + // set firing sigs + for (var i in this.ins) { + var sigId = this.ins[i]; + if (!(sigId in this.ctrIns)) { + this.firingSigs.push([sigId, 1]); + } + } + if ("next" in this.ctrIns) { + this.firingSigs.push([this.ctrIns.next,1]); + } + //console.log(JSON.stringify(this.firingSigs)); + } + + + this.ready_enter = function(session, state, transition, msg) { + var proc = session.logic; + proc.ready = true; + proc.tryTransition(proc, session); + + //onsole.log("Enter state ready: "+task.procId); + }; + + this.running_enter = function(session, state, transition, msg) { + //onsole.log("Enter state running: "+this.procId+" ("+this.name+")"); + var proc = this; + var funcIns = [], funcOuts = [], emul = proc.engine.emulate; + + proc.firingId += 1; + + var firingId = proc.firingId, + firingSigs = proc.firingSigs, + sigValues = proc.sigValues, + asyncInvocation; + + async.waterfall([ + // 1. pre-invoke: check firing limits, set proc state to running, etc. + function(cb) { + proc.preInvoke(cb); + }, + // 2. invoke the function + function(cb) { + proc.invokeFunction(cb); + }, + // 3. post-invoke: emit output signals + function(outs, wasAsync, funcIns, funcOuts, cb) { + asyncInvocation = wasAsync; + proc.postInvoke(outs, asyncInvocation, funcIns, funcOuts, firingId, firingSigs, cb); + }, + // 4. make state transition from the running state + function(cb) { + proc.postInvokeTransition(asyncInvocation, cb); + } + ]); + }; + + this.running_exit = function(session, state, transition, msg) { + //onsole.log("Exit state running: "+this.procId+" ("+this.name+")"); + }; + + this.finished_enter = function(session, state, transition, msg) { + var proc = this; + proc.wflib.setTaskState(proc.appId, proc.procId, { "status": "finished" }, function(err, rep) { + if (err) { throw err; } + //onsole.log("Enter state finished: "+proc.procId); + proc.engine.taskFinished(proc.procId); + }); + }; + + this.ReRutransition = function(session, state, transition, msg) { + //onsole.log("Transition ReRu, task "+this.procId); + }; + + this.ReFitransition = function(session, state, transition, msg) { + // emit "done" signal if such a port exists + if (this.ctrOuts.done) { + session.logic.engine.emitSignals([{ "_id": session.ctrOuts.done }]); + } + }; + + this.RuRetransition = function(session, state, transition, msg) { }; + + return this; +} + +extend(DataflowLogic, ProcLogic); + +var ProcDataflowFSM = { + name: "dataflow", + logic: DataflowLogic, + + state : [ + { + name: "init", + initial: "true", + onEnter: function (session) { session.dispatch({msgId: "InitRe"}); } + }, { + name: "ready", + }, { + name: "running", + }, { + name: "finished", + } + ], + + transition : [ + { + event : "InitRe", + from : "init", + to : "ready", + onTransition: function() { } + }, { + event : "ReRu", // when all data ins and 'next' (if such port exists) have arrived + from : "ready", + to : "running", + }, { + event : "ReFi", // when 'done' signal has arrived + from : "ready", + to : "finished", + }, { + event : "RuRe", // after firing + from : "running", + to : "ready", + } + ] +}; + +module.exports = ProcDataflowFSM; diff --git a/engine2/ProcForeachFSM.js b/engine2/ProcForeachFSM.js new file mode 100644 index 0000000..1a26dfd --- /dev/null +++ b/engine2/ProcForeachFSM.js @@ -0,0 +1,186 @@ + /* Hypermedia workflow. + ** Implementation of Finite State Machine of a 'serial foreach' process. This process waits for its input + ** signals in order and processes each of them synchronously. + ** + ** Author: Bartosz Balis (2013) + */ + +var TaskForeachFSM = { + name: "foreach", + logic: TaskLogic, + + state : [ + { + name: "ready", + onEnter: "ready_onEnter", + initial: true + }, + { + name: "running", + onEnter: "running_onEnter", + onExit: "running_onExit" + }, + { + name: "finished", + onEnter: "finished_onEnter" + } + ], + + transition : [ + { + event : "ReRe", + from : "ready", + to : "ready", + onTransition: "trReRe", + }, + { + event : "ReRu", + from : "ready", + to : "running", + onTransition: "trReRu", + }, + { + event : "RuRe", + from : "running", + to : "ready", + onTransition: "trRuRe", + }, + { + event : "RuFi", + from : "running", + to : "finished", + onTransition: "trRuFi", + }, + ] +}; + + +function TaskLogic() { + this.tasks = []; // array of all task FSM "sessions" (so that tasks can send events to other tasks) + this.wfId = -1; // workflow instance id + this.id = -1; // id of this task + this.n = -1; // number of inputs + this.cnt = 1; // number (port id) of input the task is currently waiting for + this.ins = []; // ids of inputs + this.insReady = []; // insReady[i] == true means that i-th input is ready + this.outs = []; // ids of outputs + this.sources = []; + this.sinks = []; + + // This function is invoked when one of task's inputs becomes ready. What happens next depends + // on the particular task type semantics. In this case, the Ready-Ready transition is fired. + // 'obj.msg' is a JSON object which should contain: + // - wfId: workflow instance id + // - taskId: id of task whose input is fired + // - inId: port id of the input being fired + this.fireInput = function(obj) { + var msg = obj.message, task = obj.session.logic; + obj.session.logic.insReady[msg.inId] = true; + if (obj.session.getCurrentState().name == "ready" && task.insReady[task.cnt]) { + //console.log('Firing ReRu from state: '+obj.session.getCurrentState().name, msg); + msg.msgId = "ReRu"; + obj.session.processMessage(msg); + } + }; + + + this.init = function(engine, wfId, taskId, session) { + this.engine = engine; + this.wflib = engine.wflib; + this.tasks = engine.tasks; + this.wfId = wfId; + this.id = taskId; + this.ins = engine.ins[taskId]; + this.n = engine.ins[taskId].length; + this.outs = engine.outs[taskId]; + this.sources = engine.sources; + this.sinks = engine.sinks; + session.addListener({ + contextCreated : function( obj ) { }, + contextDestroyed : function( obj ) { }, + finalStateReached : function( obj ) { }, + stateChanged : function( obj ) { + console.log("state changed: "+obj.session.getCurrentState().name); + }, + customEvent: this.fireInput + }); + }; + + this.ready_onEnter = function(session, state, transition, msg) { + console.log("Enter state ready"); + if (this.insReady[this.cnt]) { + session.dispatch( {msgId: "ReRu"} ); + } + }; + + this.running_onEnter = function(session, state, transition, msg) { + console.log("Enter state running: "+this.id+session.getCurrentState().name); + (function(task) { + task.wflib.setTaskState(task.wfId, task.id, { "status": "running" }, function(err, rep) { + if (err) { throw err; } + + var ins = task.ins[task.cnt-1], outs = task.outs[task.cnt-1]; + //console.log(task.ins[task.cnt-1], task.outs[task.cnt-1]); // DEBUG + //console.log("ins="+ins, "outs="+outs, "cnt="+task.cnt); // DEBUG + var emul = task.engine.emulate; + task.wflib.invokeTaskFunction(task.wfId, task.id, ins, outs, emul, function(err, fOuts) { + if (err) { + throw(err); + // TODO: how does the engine handle error in invocation of task's + // function? E.g. Does it affect the state machine of the task? + // Should there be an error state and transitions from it, e.g. retry? + } else { + //console.log("invoke reply="+JSON.stringify(rep)); // DEBUG + task.cnt++; + //console.log("Marking ready: "+JSON.stringify(task.outs[task.cnt-2])); + console.log("Marking ready: "+JSON.stringify(fOuts)); + //task.engine.markDataReady(task.outs[task.cnt-2], function() { + task.engine.fireSignals(fOuts, function() { + if (task.cnt <= task.n) { + session.dispatch( { msgId: "RuRe" } ); + } else { + session.dispatch( { msgId: "RuFi" } ); + } + }); + } + }); + }); + })(this); + }; + + this.running_onExit = function(session, state, transition, msg) { + //console.log("Exit state running"); + }; + + this.finished_onEnter = function(session, state, transition, msg) { + (function(task) { + task.wflib.setTaskState(task.wfId, task.id, { "status": "finished" }, function(err, rep) { + if (err) { + throw err; + } + console.log("Enter state finished: "+task.id); + task.engine.taskFinished(task.id); + }); + })(this); + }; + + this.trReRe = function(session, state, transition, msg) { + + }; + + this.trReRu = function(session, state, transition, msg) { + //console.log("Transition ReRu, task "+this.id); + }; + + this.trRuRe = function(session, state, transition, msg) { + //console.log("Transition RuRe, task "+this.id); + }; + + this.trRuFi = function(session, state, transition, msg) { + + }; + + return this; +} + +module.exports = TaskForeachFSM; diff --git a/engine2/ProcJoinFSM.js b/engine2/ProcJoinFSM.js new file mode 100644 index 0000000..53a7034 --- /dev/null +++ b/engine2/ProcJoinFSM.js @@ -0,0 +1,270 @@ + /* HyperFlow engine. + ** Author: Bartosz Balis (2014). + ** + ** Implementation of the 'join' process. This process joins/merges multiple branches + ** of execution associated with its input signals. Two properties of the 'join' process + ** determine its behavior: + ** - Nb (activeBranchesCount): how many branches are active? + ** - Nj (joinCount): how many branches should we wait for before firing the process? + ** + ** The process will: + ** - fire after 'Nj' signals have arrived + ** - "reset" after 'Nb' signals have arrived (only then it will be ready for next firing). + ** + ** This process type allows to implement the following workflow patterns + ** (see http://www.workflowpatterns.com/patterns/control): + ** - Structured discriminator + ** - Blocking discriminator (?) + ** - Structured partial join + ** - Blocking partial join (?) + ** - Local synchronizing merge + ** + ** Inputs: + ** - ... + ** Outputs: + ** - ... + ** Function: + ** - The function is passed only Nj signals (ones that arrived first). Consequently, + ** the signals should be recognized by name, not by index, as it is not known which + ** Nj (out of Nb) signals actually caused the firing. + ** + */ + +var async = require('async'), + ProcLogic = require('./process.js').ProcLogic, + fireInput = require('./process.js').fireInput, + extend = require('./process.js').extend; + +function JoinLogic() { + ProcLogic.call(this); + + this.firingSigsH = []; + this.canReset = false; + + this.init2 = function(session) { + // Nj: how many branches must arrive before firing? + this.Nj = this.fullInfo.joinCount ? this.fullInfo.joinCount: this.nDataIns; + + // Nb: how many branches must arrive before resetting the process? + this.Nb = this.fullInfo.activeBranchesCount ? this.fullInfo.activeBranchesCount: this.nDataIns; + + this.ready = true; + + session.sessionListener[0].customEvent = fireInputJoin; + } + + this.ready_enter = function(session, state, transition, msg) { + var proc = session.logic; + //proc.ready = true; + + if (proc.canReset) { + proc.resetProc(function() { + proc.tryTransition(); + }); + } + }; + + this.running_enter = function(session, state, transition, msg) { + //onsole.log("Enter state running: "+this.procId+" ("+this.name+")"); + var proc = this; + var funcIns = [], funcOuts = [], emul = proc.engine.emulate; + + proc.firingId += 1; + + var firingId = proc.firingId, + firingSigs = proc.firingSigs, + sigValues = proc.sigValues, + asyncInvocation; + + async.waterfall([ + // 1. pre-invoke: check firing limits, set proc state to running, etc. + function(cb) { + proc.preInvoke(cb); + }, + // 2. invoke the function + function(cb) { + proc.invokeFunction(cb); + }, + // 3. post-invoke: emit output signals + function(outs, wasAsync, funcIns, funcOuts, cb) { + asyncInvocation = wasAsync; + proc.postInvoke(outs, asyncInvocation, funcIns, funcOuts, firingId, firingSigs, cb); + }, + // 4. make state transition from the running state + function(cb) { + proc.postInvokeTransition(asyncInvocation, cb); + } + ]); + }; + + this.running_exit = function(session, state, transition, msg) { + //onsole.log("Exit state running: "+this.id+" ("+this.name+")"); + }; + + this.finished_enter = function(session, state, transition, msg) { + var proc = this; + proc.wflib.setTaskState(proc.appId, proc.procId, { "status": "finished" }, function(err, rep) { + if (err) { throw err; } + //onsole.log("Enter state finished: "+proc.procId); + proc.engine.taskFinished(proc.procId); + }); + }; + + this.ReFitransition = function(session, state, transition, msg) { + // emit "done" signal if such a port exists + if (session.logic.ctrOuts.done) { + session.logic.engine.emitSignals([{"_id": session.logic.ctrOuts.done }]); + } + }; + + this.ReRutransition = function(session, state, transition, msg) { }; + this.RuRetransition = function(session, state, transition, msg) { }; + + this.tryTransition = function() { + var proc = this; + if (proc.ready && proc.done) { + proc.ready = false; + proc.makeTransition("ReFi"); + return; + } + + if (proc.ready) { + //onsole.log("TRY TRANSITION", proc.firingSigsH); + if (proc.firingSigsH[0] && proc.firingSigsH[0].length == proc.Nb) { + proc.canReset = true; + } + if (proc.firingSigsH[0] && proc.firingSigsH[0].length >= proc.Nj) { + proc.ready = false; + proc.firingSigs = []; + for (var i=0; i proc.Nj) { + var sigs = proc.firingSigsH[0].slice(proc.Nj); + for (var i in sigs) { + proc.dataIns[sigs[i]]--; + sigs[i] = [ sigs[i], 1 ]; + } + proc.wflib.fetchInputs(proc.appId, proc.procId, sigs, true, function(arrived, sigValues) { + if (arrived) { + } else { + console.error("Join: should not happen!"); // FIXME: throw error + } + reset(); + }); + } else { + reset(); + } + } + + return this; +} + +extend(JoinLogic, ProcLogic); + +var ProcJoinFSM = { + name: "join", + logic: JoinLogic, + + state : [ + { name: "init", + initial: "true", + onEnter: function (session) { session.dispatch({msgId: "InitRe"}); } + }, { + name: "ready", + }, { + name: "running", + }, { + name: "finished", + } + ], + + transition : [ + { + event : "InitRe", + from : "init", + to : "ready", + onTransition: function() { } + }, { + event : "ReRu", // when 'Nj' ins have arrived + from : "ready", + to : "running" + }, { + event : "RuRe", // after firing, when 'Nj' == 'Nb' + from : "running", + to : "ready" + }, { + event : "ReFi", // when 'done' signal has arrived + from : "ready", + to : "finished" + } + ] +}; + +// This function is invoked on arrival of an input signal. +// 'obj.message' is a JSON object which should contain: +// - wfId: workflow instance id +// - sigId: signal id +// - sig: the actual signal +// - ... +function fireInputJoin(obj) { + var msg = obj.message, + proc = obj.session.logic, + state = obj.session.getCurrentState().name, + sigId = msg.sigId, + sig = msg.sig; + + if (sigId == proc.ctrIns.done) { // "done" signal has arrived + proc.done = true; + } else { + if (!proc.dataIns[sigId]) { + proc.dataIns[sigId] = 1; + proc.cnt++; + } else { + proc.dataIns[sigId] += 1; + } + var qsigs = function(idx) { + if (proc.firingSigsH[idx]) { + if ((proc.firingSigsH[idx].length >= proc.Nb) || + (proc.firingSigsH[idx].indexOf(sigId) != -1)) { + return qsigs(idx+1); + } + proc.firingSigsH[idx].push(sigId); + } else { + proc.firingSigsH[idx] = [ sigId ]; + } + + } + var idx = Math.floor(proc.dataIns[sigId]-1 / proc.Nj); + qsigs(idx); + + if (proc.firingSigsH[0] && proc.firingSigsH[0].length == proc.Nb) { + proc.canReset = true; + } + + proc.tryTransition(); + } +} + +module.exports = ProcJoinFSM; diff --git a/engine2/ProcSplitterFSM.js b/engine2/ProcSplitterFSM.js new file mode 100644 index 0000000..6a91710 --- /dev/null +++ b/engine2/ProcSplitterFSM.js @@ -0,0 +1,330 @@ +/* Hypermedia workflow. + ** Implementation of Finite State Machine of a splitter task, continuous version. + ** This task waits for an input and emits a sequence of outputs. Typically used for splitting + ** the input into chunks, but any other function is possible which for a single input emits + ** multiple outputs. + ** Inputs: + ** - MUST have exactly ONE data port (data to be splitted); it MUST be the first input port + ** - MUST have control NEXT port (triggers emission of the next chunk) + ** - MAY have control DONE port (commands to finish the task instead of emitting the + ** next chunk or waiting for the next signal) + ** Outputs: + ** - MUST have exactly ONE data port (emits consecutive chunks of input); it must be the 1st output port + ** - MAY have control NEXT port (emitted after a chunk is emitted to the data port) + ** - MUST have control DONE port (emitted when there no more chunks) + ** Function: + ** - Each invocation of f(x) should return the next chunk of data x or null if there are no more chunks + ** - The task does not specify how to split the data - it's baked into the the function + ** (e.g. split file into lines, collection into items, etc.) + ** - TODO: inject 'i' - the exepcted chunk number - as an additional function argument + ** + ** Author: Bartosz Balis (2013) + */ + +var async = require('async'), + log4js = require('log4js'); + + //log4js.configure('log4js.json'); + + //var logger = log4js.getLogger('hf-default-logger'); + +var TaskSplitterFSM = { + name: "splitter", + logic: TaskLogic, + + state : [ + { + name: "init", + initial: "true", + onEnter: function (session) { session.dispatch({msgId: "InitRe"}); } + }, + { name: "ready" }, + { name: "waiting" }, + { name: "running" }, + { name: "finished" } ], + + transition : [ { + event : "InitRe", + from : "init", + to : "ready", + onTransition: function() { } + }, { + event : "ReRu", // when all data ins and 'next' (if such port exists) have arrived + from : "ready", + to : "running", + }, { + event : "ReFi", // when 'done' signal has arrived + from : "ready", + to : "finished", + }, { + event : "RuRe", // when task served a request, it is ready to serve another one + from : "running", + to : "ready", + }, { + event : "RuWa", + from : "running", + to : "waiting", + }, { + event : "WaRu", + from : "waiting", + to : "running", + } ] +}; + +// This function is invoked on arrival of an input signal. +// 'obj.message' is a JSON object which should contain: +// - wfId: workflow instance id +// - taskId: id of task whose input is fired +// - inId: port id of the input being fired +//var fi_sync = false; +function fireInput(obj) { + var msg = obj.message, + task = obj.session.logic, + state = obj.session.getCurrentState().name, + //sigId = task.ins[msg.inId-1]; + sigId = msg.sigId; + + if (sigId == task.doneInId) { // "done" signal has arrived + task.done = true; + } + tryTransition(task, obj.session); +} + + +function fetchInputs(task, cb) { + var sigs = task.firingSigs; + task.wflib.fetchInputs(task.wfId, task.id, sigs, true, function(arrived, sigValues) { + //onsole.log("Task", task.id, "fetch attempt:", arrived); + if (arrived) { + if (sigs[sigs.length-1][0] == task.nextInId) { + sigValues.pop(); // remove next signal (should not be passed to the function) + } + task.sigValues = sigValues; // set input signal values to be passed to the function + } else { + task.ready = true; + } + cb(arrived, sigValues); + }); +} + +function fetchNext(task, cb) { + var sigs = [[task.nextInId, 1]]; + task.wflib.fetchInputs(task.wfId, task.id, sigs, true, function(arrived, sigValues) { + cb(arrived, sigValues); + }); +} + +function tryTransition(task, session) { + if (task.ready && task.done) { + task.ready = false; + session.dispatch({ msgId: "ReFi"}); + } else if (task.ready) { + task.ready = false; + fetchInputs(task, function(arrived, sigValues) { + if (arrived) { + session.dispatch({msgId: "ReRu"}); + } else { + task.ready = true; + } + }); + } else if (task.waiting) { + task.waiting = false; + fetchNext(task, function(arrived) { + if (arrived) { + session.dispatch({msgId: "WaRu"}); + } else { + task.waiting = true; + } + }); + } +} + +function TaskLogic() { + this.tasks = []; // array of all task FSM "sessions" (so that tasks can send events to other tasks) + this.wfId = -1; // workflow instance id + this.id = -1; // id of this task + this.cnt = 0; // how many signals are waiting on all ports; TODO: store this in redis for persistence + this.dataIns = []; // which data inputs have arrived (are ready)? (dataIns[id]==true) + this.nDataIns = -1; // how many data inputs are there? + this.next = false; // has the signal arrived to the control "next" port? + this.done = false; // has the signal arrived to the control "done" port? + this.nextInId = undefined; // id of 'next' control input port + this.nextOutId = undefined; // id of 'next' control output port + this.doneInId = undefined; // id of 'done' control input port + this.doneOutId = undefined; // id of 'done' control output port + this.ins = []; // ids of inputs (data and control signals) + this.outs = []; // ids of outputs (data and control signals) + this.sources = []; + this.sinks = []; + this.firingInterval = -1; // a task can have 0 data inputs and a firing interval ==> its + // function will be invoked regularly according to this interval + this.firingSigs = []; + this.sigValues = null; + + this.ready = false; + this.waiting = false; + + this.init = function(engine, wfId, taskId, session, fullInfo) { + this.engine = engine; + this.wflib = engine.wflib; + this.tasks = engine.tasks; + this.wfId = wfId; + this.id = taskId; + this.ins = engine.ins[taskId]; + this.outs = engine.outs[taskId]; + this.sources = engine.sources; + this.sinks = engine.sinks; + this.nDataIns = engine.ins[taskId].length; + this.firstInvocation = true; + this.name = fullInfo.name; + this.fullInfo = fullInfo; + + if (this.id in engine.cPorts) { + var taskCPorts = engine.cPorts[this.id]; + if ("ins" in taskCPorts) { + for (var i in taskCPorts.ins) { + // this should be correct: #(data_ins) = #(all_ins) - #(control_ins) + // (not an efficient way to compute but there should be max ~2 control ins) + this.nDataIns--; + } + this.nextInId = taskCPorts.ins.next; + this.doneInId = taskCPorts.ins.done; + } + if ("outs" in taskCPorts) { + this.nextOutId = taskCPorts.outs.next; + this.doneOutId = taskCPorts.outs.done; + } + //onsole.log("Cports: "+this.nextInId, this.doneInId, this.nextOutId, this.doneOutId); // DEBUG + } + + for (var i in this.ins) { + var sigId = this.ins[i]; + if ((sigId != this.nextInId) && (sigId != this.doneInId)) { + this.firingSigs.push([sigId, 1]); + } + } + if (this.nextInId) { + this.firingSigs.push([this.nextInId,1]); + } + + session.addListener({ + contextCreated : function( obj ) { }, + contextDestroyed : function( obj ) { }, + finalStateReached : function( obj ) { }, + stateChanged : function( obj ) { }, + customEvent : fireInput + }); + }; + + this.ready_enter = function(session, state, transition, msg) { + var task = session.logic; + task.ready = true; + task.waiting = false; + + tryTransition(task, session); + + //onsole.log("Enter state ready: "+task.id); + }; + + this.waiting_enter = function(session, state, transition, msg) { + var task = session.logic; + task.waiting = true; + task.ready = false; + + tryTransition(task, session); + + //onsole.log("Enter state waiting: "+task.id); + }; + + this.running_enter = function(session, state, transition, msg) { + //onsole.log("Enter state running: "+this.id+" ("+this.name+")"); + var task = this; + var funcIns = [], funcOuts = [], emul = task.engine.emulate; + async.waterfall([ + // 1. set task state to running + function(cb) { + task.wflib.setTaskState(task.wfId, task.id, { "status": "running" }, function(err, rep) { + err ? cb(err): cb(null); + }); + }, + // 2. invoke the function + function(cb) { + // create arrays of data ins and outs ids + for (var i in task.firingSigs) { + var sigId = task.firingSigs[i][0]; + if (sigId != task.nextInId && sigId != task.doneInId) { + funcIns.push(task.sigId); + } + } + for (var i in task.outs) { + outId = task.outs[i]; + if ((outId != task.nextOutId) && outId != task.doneOutId) { + funcOuts.push(outId); + } + } + + task.wflib.invokeTaskFunction2( + task.wfId, + task.id, + funcIns, + task.sigValues, + funcOuts, emul, + function(err, outs) { + err ? cb(err): cb(null, outs); + } + ); + }, + // 3. emit output signals + function(outs, cb) { + if (outs) { + task.cnt -= task.firingSigs.length; // subtract cnt by the number of consumed signals + if (task.fullInfo.sticky) + task.cnt += task.fullInfo.sticky; // sticky signals weren't actually consumed! + var outValues = outs; + for (var i=0; i object to subclass + * @param superc object to subclass from + */ + function extend(subc, superc) { + var subcp = subc.prototype; + var method; + + // Class pattern. + var F = function() { + }; + F.prototype = superc.prototype; + + subc.prototype = new F(); // chain prototypes. + subc.superclass = superc.prototype; + subc.prototype.constructor = subc; + + // Reset constructor. See Object Oriented Javascript for an in-depth explanation of this. + if (superc.prototype.constructor === Object.prototype.constructor) { + superc.prototype.constructor = superc; + } + + // los metodos de superc, que no esten en esta clase, crear un metodo que + // llama al metodo de superc. + for ( method in subcp ) { + if (subcp.hasOwnProperty(method)) { + subc.prototype[method] = subcp[method]; + } + } + } + + /** + * Bind mechanism. Honors already existing bind functions. + */ + Function.prototype.bind = Function.prototype.bind || function( /* this */ ) { + + var fn= this; // the function + var args= Array.prototype.slice.call(arguments); // copy the arguments. + var obj= args.shift(); // first parameter will be context 'this' + + return function() { + fn.apply( obj, args.concat( Array.prototype.slice(arguments) ) ); + }; + }; + + /** + * TimerTask sequence. + */ + var __TimerIndex= 0; + + /** + * State creation sequence + */ + var __StateIndex= 0; + + /** + * Initial transition msgId identification. + */ + var __InitialTransitionId= "__initial_transition_id"; + + /** + * Automata system context object. Supposed to be unique. + */ + var fsmContext= null; + + /** + * Local module definition. + */ + var FSM= {}; + + /** + * FSMTimerTask + * + * This object encapsulates a task timer. + * They are automatically defined by setting an onTimer block in a state definition. + * + * @constructor + * @param session a session object + * @param event a message object. + * @param time an integer specifying milliseconds. + * @param id a unique integer value. + */ + FSM.TimerTask= function( session, event, time, id ) { + this.session= session; + this.event= event; + this.triggerTime= time; + this.id= id; + + this.contextState= session.getCurrentState(); + this.scheduleTime= new Date().getTime(); + + return this; + }; + + FSM.TimerTask.prototype= { + session : null, + contextState : null, + event : null, + triggerTime : 0, + scheduleTime : 0, + consumed : false, + id : 0, + + /** + * Has this timer task already been fired ? + */ + isConsumed : function() { + return this.consumed; + }, + + /** + * Is this timer task on time so that it must be triggered ? + * @param t + */ + isExpired : function( t ) { + return this.scheduleTime + this.triggerTime < t; + }, + + /** + * This is the timer task control function. + * @param t + */ + consume : function( t ) { + if ( this.isConsumed() ) { + return true; + } + + if ( this.isExpired( t ) ) { + if ( this.contextState === this.session.getCurrentState() ) { + this.session.dispatch( this.event ); + } + + this.consumed= true; + return true; + } + + return false; + } + }; + + /** + * + * FSMContext + * FSMContext is the core of the Automata engine. It server as Finite State Machines registry, timer task + * manager, FSM session creation, etc. + * It is intended to be a unique object. + * + * @constructor + * + */ + FSM.FSMContext= function() { + + this.timerTasks= []; + this.registry= {}; + + return this; + }; + + FSM.FSMContext.prototype= { + + timerId : null, + timerTasks : null, + registry : null, + + initialized : false, + + /** + * Check every FSM running session pending timer tasks. + * @private + */ + __checkTimers : function() { + + var time= new Date().getTime(); + + for( var i=0; i < this.timerTasks.length; i++ ) { + var timerTask= this.timerTasks[i]; + if ( timerTask.consume( time ) ) { + this.timerTasks.splice( i,1 ); + } + } + }, + + /** + * Initialize Automata's engine. + */ + initialize : function() { + if ( this.initialized ) { + throw (new Error("Automata already initialized.")); + } + + this.timerId= root.setInterval( this.__checkTimers.bind(this), 200 ); + this.initialized= true; + return this; + }, + + /** + * Shutdown Automata's engine. + * Pending timer tasks won't be notified. + */ + destroy : function() { + root.clearInterval( this.timerId ); + }, + + /** + * Register a new FSM. + * This is the first step to have a running FSM session in Automata engine. + * + * @param name a FSM name. + * @param fsm an FSM object instance. + */ + registerFSM : function( name, fsm ) { + if ( this.registry[name] ) { + throw (new Error("'"+name+"' FSM already registered.")); + } + + this.registry[ name ]= fsm; + }, + + /** + * Get a FSM.FSM registered instance. + * + * @param name + * @private + */ + getFSM : function( name ) { + return this.registry[ name ]; + }, + + /** + * Create a given FSM session. + * @param fromFSM a FSM name. Must be previously registered by calling registerFSM function. + * @param args an array of parameters passed from context.createSession() + * @return an initialized session object. + */ + createSession : function( fromFSM, args ) { + + var fsm= this.registry[ fromFSM ]; + if ( typeof fsm==="undefined" ) { + throw (new Error("FSM "+fromFSM+" does not exist.")); + } + + return fsm.createSession(args); + }, + + /** + * Add a new Timer Task. + * A timer task means sending a message to a given FSM session after elapsing some time. + * It is automatically managed by onTimer block definition. + * + * Should not be called directly. + * + * @param session a session object + * @param event a message object + * @param time an integer indicating milliseconds. + * + * @return a unique timertask id. + */ + addTimerTask : function( session, event, time ) { + var id= __TimerIndex++; + this.timerTasks.push( new FSM.TimerTask( session, event, time, id ) ); + return id; + }, + + /** + * Remove a previously set timer task. + * It is automatically managed by onTimer block definition. + * + * Should not be called directly. + * + * @param id + */ + removeTimerTask : function( id ) { + for( var i=0; i + */ + setOnPreGuard : function( m ) { + this.onPreGuard= m; + return this; + }, + + createThrowable : function( msg ) { + throw new FSM.GuardException(msg); + }, + + /** + * Set this transition's post guard function or function name form the logic object. + * + * @param m + */ + setOnPostGuard : function( m ) { + this.onPostGuard= m; + return this; + }, + + /** + * Set this transition's callback function executed when the transition is fired. + * @param m + */ + setOnTransition : function( m ) { + this.onTransition= m; + return this; + }, + + /** + * Do this transition's pre-transition code + * @param msg + * @param session + */ + firePreTransition : function( msg, session) { + if ( this.initialState!=null ) { + this.initialState.callOnExit( session, this, msg ); + } + + session.callMethod( this.onTransition, this.initialState, this, msg ); + }, + + /** + * Do this transition's post-transition code + * @param msg + * @param session + */ + firePostTransition : function( msg, session) { + this.finalState.callOnEnter( session, this, msg ); + }, + + /** + * Do this transition's pre-transition code. Though it may seem equal to firePreTransition it is handled + * in another function because an exception could be throws. In such case a pre-guard is assumed to have + * been fired. + * @param msg + * @param session + */ + firePreTransitionGuardedByPostCondition : function( msg, session ) { + if ( this.initialState!=null ) { + this.initialState.callOnExit( session, this, msg ); + } + + session.callMethod( this.onTransition, this.initialState, this, msg); + }, + + /** + * Do this transition's post-transition code. Though it may seem equal to firePreTransition it is handled + * in another function because an exception could be throws. In such case a pre-guard is assumed to have + * been fired. + * @param msg + * @param session + */ + firePostTransitionGuardedByPostCondition : function( msg, session ) { + if ( this.initialState!=null ) { + session.callMethod( this.initialState.onEnter, this.initialState, this, msg ); + } + }, + + /** + * Fire pre-Guard code. + * If the method throws an exception, this transition is aborted as if it hadn't been fired. + * @param msg + * @param session + */ + checkGuardPreCondition : function( msg, session ) { + session.callMethod( this.onPreGuard, this.initialState, this, msg ); + }, + + /** + * Fire post-Guard code. + * If the method throws an exception, this transition is vetoed, and it will issue an auto-transition instead + * of a state-to-state transition. + * @param msg + * @param session + */ + checkGuardPostCondition : function( msg, session ) { + session.callMethod( this.onPostGuard, this.initialState, this, msg ); + }, + + /** + * Notify observers about this transition fire event. + * @param msg the message which fired this transition + * @param session + * + * @private + */ + fireTransition : function( msg, session ) { + if ( this.initialState!==null ) { + this.initialState.callOnExit( session, this, msg ); + } + + session.callMethod( this.onTransition, this.initialState,this,msg ); + this.finalState.callOnEnter( session, this, msg ); + }, + + toString : function() { + return ""+this.event; + } + }; + + /** + * FSMState + * This object defines a FSM state. + * + * @constructor + */ + FSM.State= function( name ) { + + this.exitTransitions= { + count: 0 + }; + this.name= name || ( "state"+__StateIndex++ ); + + this.onEnter= this.name+"_enter"; + this.onExit= this.name+"_exit"; + + return this; + }; + + FSM.State.prototype= { + exitTransitions : null, + name : "", + onEnter : null, + onExit : null, + onTimer : null, + + subState : null, + + /** + * Add an exit transition to this State instance. + * This transition must be uniquely added. + * @param tr + */ + addTransition : function( tr ) { + var event= tr.getEvent(); + + if ( this.exitTransitions[event] ) { + throw (new Error("Already set transition for event "+event)); + } + + this.exitTransitions[event]= tr; + this.exitTransitions.count++; + + return this; + }, + + /** + * Check whether this state has exiting transitions. + * If not, will be defined as final. + */ + isFinalState : function() { + return this.exitTransitions.count===0; + }, + + /** + * Set this state's onEnter callback function. + * @param c + */ + setOnEnter : function( c ) { + this.onEnter= c; + return this; + }, + + /** + * Set this state's onExit callback function. + * @param c + */ + setOnExit : function( c ) { + this.onExit= c; + return this; + }, + + /** + * Add a timed transition to this state. + * @param c , event: > + */ + setOnTimer : function( c ) { + this.onTimer= c; + }, + + /** + * Get a transition for the defined typeof message. + * @param msg + */ + getTransitionFor : function( msg ) { + return this.exitTransitions[ msg.msgId ]; + }, + + /** + * @private + */ + __getTimerKey : function( ) { + return this.name; // + "#" + this.onTimer.event.msgId; + }, + + /** + * Execute the procedure on entering this State. + * It may seem to set a timer, and calling the optional onEnter callback function. + * @param session + * @param transition + * @param msg + */ + callOnEnter : function( session, transition, msg ) { + if ( this.onTimer ) { + session.addProperty( + this.__getTimerKey( ), + fsmContext.addTimerTask( session, this.onTimer.event, this.onTimer.timeout ) + ); + } + session.callMethod( this.onEnter, this, transition, msg ); + }, + + /** + * Execute the procedure on exiting this State. + * It may seem to reset a timer, and calling the optional onEnter callback function. + * + * @param session + * @param transition + * @param msg + */ + callOnExit : function( session, transition, msg ) { + if( this.onTimer ) { + var pr= session.getProperty( this.__getTimerKey() ); + fsmContext.removeTimerTask( pr ); + session.removeProperty(pr); + } + session.callMethod( this.onExit, this, transition, msg ); + }, + + toString : function() { + return ""+this.name; + } + + }; + + /** + * FSM + * FSM defines a complete finite state machine. + * A FSM.FSM object extends a FSM.State object, so polymorphically a complete FSM is an State. This way, we can + * supply with sub-states to Automata's engine. + * + * @constructor + * + */ + FSM.FSM= function( sessionObjectFactory, name ) { + + FSM.FSM.superclass.constructor.call(this, name); + + this.sessionObjectFactory= sessionObjectFactory; + + this._onEnter= this.name+"_enter"; + + + return this; + }; + + FSM.FSM.prototype= { + + sessionObjectFactory : null, + + initialTransition : null, + initialState : null, + + /** + * Initialize a Finite State Machine. + * + * @param initialState + */ + initialize : function( initialState ) { + + var me= this; + + FSM.FSM.superclass.setOnEnter.call( this, function( session, state, transition, msg ) { + me.initialTransition.fireTransition( { + msgId : __InitialTransitionId + }, + session ); + } ); + + this.initialState= initialState; + this.initialTransition= new FSM.Transition(__InitialTransitionId, null, initialState ); + this.initialTransition.setOnTransition( function( session, state, transition, msg ) { + session.push( me.initialState ); + }); + }, + + setOnEnter : function( m ) { + this._onEnter= m; + return this; + }, + + callOnEnter : function( session, transition, msg ) { + session.callMethod( this._onEnter, this, transition, msg ); + FSM.FSM.superclass.callOnEnter.call( this, session, transition, msg ); + }, + + /** + * Build a Session for this FSM object. + * A session is (of course) initially empty. + * This method is called only once, and from this on, sub-state automata go on with the normal lifecycle + * calling their custom onEnter method which launcher the initialTransition. + * Strictly talking, automata object should be constructed from a building block where just an FSM defined + * just one state being a substate of the target FSM. + * + * To avoid such automata definition inefficiencies, here I'm calling the block manually: + * + pushing a top level FSM context + * + calling its onEnter method as if an initialTransition was fired. + * + * I'm not happy with the semantics of manually calling a (supposed to be) initial transition. Will keep it + * this way for the sake of simplicity, but will probably change this semantics in the future, + * (by adding an Automata with just one substate) which could cause backward incompatibilities. + */ + createSession : function(args) { + + if ( !this.sessionObjectFactory ) { + return null; + } + + var session= new FSM.Session( ); + var logic= new this.sessionObjectFactory(session, args); + session.setLogic( logic ); + session.push( this ); + this.callOnEnter( session, null, null ); + + return session; + } + }; + + extend( FSM.FSM, FSM.State ); + + + /** + * SessionContext + * A session context is just a holder for a current state across the different nesting levels of a given FSM. + * This class is some sugar to deal with an State. + * A FSM.Session is an stack of different contexts. + * + * @constructor + */ + FSM.SessionContext= function( state ) { + this.currentState= state; + + return this; + }; + + FSM.SessionContext.prototype= { + + currentState : null, + + /** + * Set this context current state. + * This method will be called by Automata's engine when a state change is fired. + * @param s + */ + setCurrentState : function( s ) { + this.currentState= s; + }, + + /** + * Get this context's current state. + * @return + */ + getState : function() { + return this.currentState; + }, + + /** + * Get an exiting transition defined by this message for the current State. + * @param msg + */ + getTransitionFor : function( msg ) { + return this.currentState.getTransitionFor( msg ); + }, + + /** + * Call this current State onExit callback function. + * @param session + * @param transition + * @param msg + */ + exit : function( session, transition, msg) { + this.currentState.callOnExit(session, transition, msg); + }, + + /** + * Print this context current state info. + */ + printStackTrace : function() { + root.console.log(" "+this.currentState.name); + } + + + }; + + /** + * FSM.Session + * A Session is the real artifact to deal with in Automata engine. + * A session must be created and will the core object to send messages to. Automata framework will take care + * of choreograph the calls, context push/pop, session observer notification, etc. + * + * @constructor + */ + FSM.Session= function( ) { + + this.sessionContextList= []; + this.sessionListener= []; + this.properties= {}; + + return this; + }; + + FSM.Session.prototype= { + + id : null, + sessionContextList : null, + sessionListener : null, + properties : null, + + transitioning : false, + + logic : null, + + setLogic : function( logic ) { + this.logic= logic; + }, + + /** + * Never call this method directly. + * For a given Automata event triggering (state.onEnter, state.onExit, transition.onPre/PostGuard, + * transition.onTransition), this method makes the appropriate call, either to the logic object, or to + * the supplied callback function instead. + */ + callMethod : function( /* method, argument1, ... */ ) { + var args= Array.prototype.slice.call( arguments ); + var method= args.shift(); + + if ( null===method ) { // just in case. + return; + } + + args.splice(0,0,this); + + if ( typeof method==="function" ) { + method.apply( this.logic, args ); + } else { + if ( typeof this.logic[method]!=="undefined" ) { + this.logic[ method ].apply( this.logic, args ); + } + } + }, + + /** + * Add an observer to this session. + * @param sl + */ + addListener : function( sl ) { + this.sessionListener.push( sl ); + }, + + /** + * Remove an observer from this session. + * @param sl + */ + removeListener : function( sl ) { + var pos= this.sessionListener.indexOf( sl ); + if ( -1!==pos ) { + this.sessionListener.splice( pos, 1 ); + } + }, + + /** + * Push and set up a new FSM.Context level. + * @param state + * + * @private + */ + push : function( state ) { + var sc= new FSM.SessionContext( state ); + + this.sessionContextList.push( sc ); + this.fireContextCreated( sc ); + this.fireStateChanged( sc, state, __InitialTransitionId ); + }, + + /** + * Pop and reset the last FSM.Context object level. + * @param transition the firing transition + * @param msg the message that triggered the transition + * + * @private + */ + pop : function( transition, msg ) { + var sc= this.sessionContextList.pop(); + sc.exit( this, transition, msg ); + + this.fireContextRemoved( sc ); + + if ( this.sessionContextList.length===0 ) { + this.fireSessionEmpty(); + } + }, + + /** + * Asynchronously consume a message. + * @param msg + */ + dispatch : function( msg ) { + var me= this; + process.nextTick( function() { + try { + me.processMessage( msg ); + } catch(e) { + throw (e); + }; + + }); + + }, + + /** + * Synchronoulsy consume a message. + * @param msg + */ + processMessage : function( msg ) { + if ( this.transitioning ) { + throw (new Error("Processing message during transition")); + } + + if ( this.sessionContextList.length===0 ) { + throw (new Error("Empty Session")); + } + + var firingTransition= null; // FSM.Transition + var target= null; // FSM.SessionContext + var i; + for( i= this.sessionContextList.length - 1; i>=0; i-- ) { + target= this.sessionContextList[i]; + firingTransition= target.getTransitionFor( msg ); + if ( null!=firingTransition ) { + break; + } + } + + if ( !firingTransition ) { + throw (new Error("No transition on state "+this.getCurrentState().name+" for message "+msg.msgId)); + } + + // check guard pre condition. + try { + firingTransition.checkGuardPreCondition( msg, this ); + } catch( e ) { + if ( e instanceof FSM.GuardException ) { + this.fireGuardPreCondition(firingTransition, msg, e); + return; // fails on pre-guard. simply return. + } else { + console.error("An error ocurred: "+ e.message, e.stack); + } + } + + this.transitioning= true; + + try { + firingTransition.checkGuardPostCondition( msg, this ); + + try { + for( var j= this.sessionContextList.length-1; j>i; j-- ) { + this.pop( firingTransition, msg ); + } + + firingTransition.firePreTransition( msg, this ); + + var newState= firingTransition.finalState; + target.setCurrentState( newState ); + this.fireStateChanged( target, newState, msg ); + + firingTransition.firePostTransition( msg, this ); + + while( + this.sessionContextList.length!==0 && + this.getCurrentSessionContext().getState().isFinalState() ) { + + this.pop( null, msg ); + } + } catch( ex ) { + console.error("An error ocurred: "+ ex.message, ex.stack); + } + } catch( guardException ) { + if ( guardException instanceof FSM.GuardException ) { + this.fireGuardPostCondition(firingTransition, msg, guardException); + firingTransition.firePreTransitionGuardedByPostCondition( msg, this ); + this.fireStateChanged( target, firingTransition.initialState, msg ); + firingTransition.firePostTransitionGuardedByPostCondition( msg, this ); + } else { + console.error("An error ocurred: "+ guardException.message, guardException.stack); + } + } + + this.transitioning= false; + }, + + /** + * Get the current execution context. + */ + getCurrentSessionContext : function() { + return this.sessionContextList[ this.sessionContextList.length-1 ]; + }, + + /** + * Get current's context state. + */ + getCurrentState : function() { + try { + return this.getCurrentSessionContext().getState(); + } catch( e ) { + return null; + } + }, + + /** + * Print information about the context stack state. + */ + printStackTrace : function() { + if ( this.sessionContextList.length===0 ) { + root.console.log("session empty"); + } else { + root.console.log("session stack trace:"); + for( var i=0; i + * @param value + */ + addProperty : function( key, value ) { + this.properties[key]= value; + }, + + /** + * Remove a property. + * @param key + */ + removeProperty : function( key ) { + delete this.properties[ key ]; + }, + + getProperty : function( key ) { + return this.properties[key]; + }, + + /////////////// firing methods ////////////// + + fireSessionEmpty : function() { + for( var i=0; i A FSM object definition. + */ + function registerFSM( fsmd ) { + var fsm= new FSM.FSM( fsmd.logic, fsmd.name ); + + var i; + var states_a= fsmd.state; + var states= {}; + var initial_state= null; + for( i=0; i a FSM registered name. + */ + function createSession( fsm ) { + var args= Array.prototype.slice.call(arguments); + args.shift(); + return fsmContext.createSession( fsm, args ); + } + + /** + * node module definition. + */ + var _export= { + registerFSM : registerFSM, + registerFDA : registerFSM, + createSession : createSession + }; + + + if (typeof define!=='undefined' && define.amd) { // AMD / RequireJS + define('async', [], function () { + return _export; + }); + + } else if (typeof module!=='undefined' && module.exports) { // Node.js + module.exports= _export; + + } else { + root.Automata= _export; + + } + +})( typeof window!=='undefined' ? window : global ); diff --git a/engine2/index.js b/engine2/index.js new file mode 100644 index 0000000..4c24b8e --- /dev/null +++ b/engine2/index.js @@ -0,0 +1,340 @@ +/** Hypermedia workflow execution engine. + ** Author: Bartosz Balis (2013) + */ + +/* + * Uses workflow map retrieved from redis: + * - ins[i][j] = data id mapped to j-th output port of i-th task + * - outs[i][j] = data id mapped to j-th input port of i-th task + * - sources[i][1] = task id which produces data element with id=i (if none, sources[i]=[]) + * - sources[i][2] = port id in this task the data element is mapped to + * - sinks[i][j] = task id which consumes data element with id=i (if none, sinks[i]=[]) + * - sinks[i][j+1] = port id in this task the data element is mapped to + */ + +var fs = require('fs'), + xml2js = require('xml2js'), + fsm = require('./automata.js'), + async = require('async'); + +var ProcDataflowFSM = require('./ProcDataflowFSM.js'); +var ProcChoiceFSM = require('./ProcChoiceFSM.js'); +var ProcForeachFSM = require('./ProcForeachFSM.js'); +var ProcJoinFSM = require('./procJoinFSM.js'); +var ProcSplitterFSM = require('./ProcSplitterFSM.js'); + + +// TODO: automatically import and register all task FSMs in the current directory +fsm.registerFSM(ProcDataflowFSM); +fsm.registerFSM(ProcChoiceFSM); +fsm.registerFSM(ProcForeachFSM); +fsm.registerFSM(ProcJoinFSM); +fsm.registerFSM(ProcSplitterFSM); + + +// Engine constructor +// @config: JSON object which contains Engine configuration: +// - config.emulate (true/false) = should engine work in the emulation mode? +var Engine = function(config, wflib, wfId, cb) { + this.wflib = wflib; + this.wfId = wfId; + this.tasks = []; // array of task FSMs + this.ins = []; + this.outs = []; + this.sources = []; + this.sinks = []; + this.wfOuts = []; + this.trace = ""; // trace: list of task ids in the sequence they were finished + this.nTasksLeft = 0; // how many tasks left (not finished)? + this.nWfOutsLeft = 0; // how many workflow outputs are still to be produced? + this.syncCb = null; // callback invoked when wf instance finished execution (passed to runInstanceSync) + + this.emulate = config.emulate == "true" ? true: false; + + this.startTime = (new Date()).getTime(); // the start time of this engine (workflow) + + var engine = this; + engine.wflib.getWfMap(wfId, function(err, nTasks, nData, ins, outs, sources, sinks, types, cPortsInfo, fullInfo) { + //onsole.log("cPortsInfo:"); onsole.log(cPortsInfo); // DEBUG + engine.nTasksLeft = nTasks; + engine.ins = ins; + engine.outs = outs; + engine.sources = sources; + engine.sinks = sinks; + engine.cPorts = cPortsInfo; + + // create processes of types other than default "dataflow" (e.g. "foreach", "splitter", etc.) + for (var type in types) { + //onsole.log("type: "+type+", "+types[type]); // DEBUG + types[type].forEach(function(taskId) { + engine.tasks[taskId] = fsm.createSession(type); + engine.tasks[taskId].logic.init(engine, wfId, taskId, engine.tasks[taskId], fullInfo[taskId]); + }); + } + // create all other processes (assuming the default type "dataflow") + for (var taskId=1; taskId<=nTasks; ++taskId) { + if (!engine.tasks[taskId]) { + engine.tasks[taskId] = fsm.createSession("dataflow"); + engine.tasks[taskId].logic.init(engine, wfId, taskId, engine.tasks[taskId], fullInfo[taskId]); + } + } + + engine.wflib.getWfOuts(engine.wfId, false, function(err, wfOuts) { + engine.wfOuts = wfOuts; + engine.nWfOutsLeft = wfOuts.length ? wfOuts.length: -1; + cb(null); + }); + }); +} + + ////////////////////////////////////////////////////////////////////////// + ///////////////////////// public functions /////////////////////////////// + ////////////////////////////////////////////////////////////////////////// + +Engine.prototype.runInstance = function (cb) { + var engine = this; + engine.wflib.setWfInstanceState(engine.wfId, { "status": "running" }, function(err, rep) { + if (err) return cb(err); + // send initial signals (if any) to the instance + engine.wflib.getInitialSigs(engine.wfId, function(err, sigs) { + if (sigs) { + engine.emitSignals(sigs, function(err) { + cb(err); + }); + } + }); + }); + + // Emulate workflow execution: change state of all workflow inputs to "ready" and send + // signal to all sink tasks; pretend task Functions are invoked and results computed. + if (this.emulate) { + (function(engine) { + engine.wflib.getWfIns(engine.wfId, false, function(err, wfIns) { + for (var i=0; i wf is finished (always?) + } else { + this.trace += ","; + } +} + +Engine.prototype.workflowFinished = function() { + console.log("Workflow ["+this.wfId+"] finished. Exec trace:", this.trace+"." ); + //onsole.log(this.syncCb); + if (this.syncCb) { + this.syncCb(); + } +} + +// NEW API for sending signals for continuous processes with FIFO queues +// WILL DEPRECATE fireSignals +// @sigs (array): ids of signals (data and control ones) to be emitted +// format: [ { attr: value, +// ... +// "_id": sigId, +// "data": [ { ... }, ... { ... } ] => multiple instances of this signal +// }, +// { attr: value, +// ... +// "_id": sigId, +// "data": [ { ... }, ... { ... } ] => multiple instances of this signal +// } +// ] +Engine.prototype.emitSignals = function(sigs, cb) { + var timeStamp; // time stamp to be added to each signal (relative to workflow start time) + var engine = this; + + var copySignal = function(sig) { + var copy = {}; + if (null == sig || "object" != typeof sig) return sig; + for (var attr in sig) { + if (sig.hasOwnProperty(attr) && attr != "data") + copy[attr] = sig[attr]; + } + return copy; + } + + timeStamp = (new Date()).getTime() - engine.startTime; + + // iterate over all signals to be sent + async.each(sigs, function iterator(sig, doneIter) { + var sigInstances = []; + sig._ts = timeStamp; + if (sig.data) { // there is a 'data' array which may contain multiple instances of this signal + for (var i in sig.data) { + var s = copySignal(sig); + s.data = [sig.data[i]]; + sigInstances.push(s); + } + } else { + sigInstances.push(sig); + } + + // send all instances of a given signal + async.each(sigInstances, function(s, doneIterInner) { + var _sigId = s._id; + engine.wflib.sendSignal(engine.wfId, s, function(err, sinks) { + if (!err) { + // notify sinks that the signals have arrived + for (var j=0; j= proc.firingSigs.length) { // not accurate if signal counts > 1 in the firing pattern + proc.tryTransition(); + } + } +} + +var ProcLogic = function() { + this.procs = []; // array of all process FSM "sessions" (so processes can send events to other processes) + this.appId = -1; // workflow instance id + this.procId = -1; // id of this process + this.cnt = 0; // how many inputs have at least one signal on the queue + this.dataIns = []; // dataIns[sigId] = count of instances of signal 'sigId' awaiting on the queue + this.nDataIns = -1; // how many data inputs are there? + this.next = false; // has the "next" control sig arrived? + this.done = false; // has the "done" control sig arrived? + this.ctrIns = {}; // control inputs (map sig name -> sigId) + this.ctrOuts = {}; // control outputs (map sig name -> sigId) + this.ins = []; // ids of inputs (data and control signals) + this.outs = []; // ids of outputs (data and control signals) + this.sources = []; + this.sinks = []; + this.firingInterval = -1; // a process can have 0 data inputs and a firing interval ==> its + // function will be invoked regularly according to this interval + this.firingSigs = []; // sigs required to fire the process + this.firingLimit = -1; // max number of times the process can fire (-1 = unlimited) + this.sigValues = null; + this.firingId = 0; // which firing of the process is this? + this.runningCount = 0; // how many firings are currently running in parallel (max = parlevel)? + + this.ready = false; // is the proc ready to read input signals? + + this.init = function(engine, appId, procId, session, fullInfo) { + this.engine = engine; + this.wflib = engine.wflib; + this.procs = engine.tasks; + this.appId = appId; + this.procId = procId; + this.ins = engine.ins[procId]; + this.outs = engine.outs[procId]; + this.sources = engine.sources; + this.sinks = engine.sinks; + this.nDataIns = engine.ins[procId].length; + this.firstInvocation = true; + this.fullInfo = fullInfo; + this.name = fullInfo.name; + this.parlevel = fullInfo.parlevel ? fullInfo.parlevel : 1; // maximum level of parallelism + this.session = session; + + this.firingLimit = this.fullInfo.firingLimit ? this.fullInfo.firingLimit : -1; + + if (this.procId in engine.cPorts) { + var procCPorts = engine.cPorts[this.procId]; + if ("ins" in procCPorts) { + for (var i in procCPorts.ins) { + // this should be correct: #(data_ins) = #(all_ins) - #(control_ins) + // (not an efficient way to compute but there should be max ~2 control ins) + this.nDataIns--; + } + this.ctrIns = procCPorts.ins; + } + if ("outs" in procCPorts) { + this.ctrOuts = procCPorts.outs; + } + //onsole.log("Cports: "+this.ctrIns.next, this.ctrIns.next, this.ctrOuts.next, this.ctrOuts.next); // DEBUG + } + + if (this.nDataIns == 0) { // special case with no data inputs (a 'source' pocess) + // FIXME: add assertion/validation that firing interval is defined! + // TODO: change semantics of firingInterval to *minimal* firing interval regardless of # of inputs + this.firingInterval = this.fullInfo.firingInterval; + } + + session.addListener({ + contextCreated : function( obj ) { }, + contextDestroyed : function( obj ) { }, + finalStateReached : function( obj ) { }, + stateChanged : function( obj ) { }, + customEvent : fireInput + }); + + // process-specific initialization + if (this.init2) { + this.init2(session); + } + } + + this.tryTransition = function() { + var proc = this; + if (proc.ready && proc.done) { + proc.ready = false; + proc.makeTransition("ReFi"); + } + + if (proc.nDataIns == 0 && proc.ready) { + // a "source" process: to be fired regularly according to a firing interval + proc.ready = false; + if (proc.firstInvocation) { + proc.firstInvocation = false; + proc.makeTransition("ReRu"); + } else { + setTimeout(function() { + proc.makeTransition("ReRu"); + }, proc.firingInterval); + } + } else if (proc.ready) { + proc.ready = false; + proc.fetchInputs(proc, function(arrived, sigValues) { + if (arrived) { + proc.makeTransition("ReRu"); + } else { + proc.ready = true; + } + }); + } + } + + this.fetchInputs = function(proc, cb) { + var sigs = proc.firingSigs; + proc.wflib.fetchInputs(proc.appId, proc.procId, sigs, true, function(arrived, sigValues) { + if (arrived) { + //console.log("FETCHED", sigs, proc.procId); + if (sigs[sigs.length-1][0] == proc.ctrIns.next) { + sigValues.pop(); // remove 'next' signal (should not be passed to the function) + } + proc.sigValues = sigValues; // set input signal values to be passed to the function + } else { + proc.ready = true; + } + cb(arrived, sigValues); + }); + } + + this.preInvoke = function(cb) { + var proc = this; + proc.runningCount += 1; + if (proc.firingLimit != -1) { + proc.firingLimit -= 1; + if (proc.firingLimit == 0) { + proc.done = true; + } + } + //onsole.log("runningCount (" + proc.fullInfo.name + "):", proc.runningCount); + proc.wflib.setTaskState(proc.appId, proc.procId, { "status": "running" }, function(err, rep) { + err ? cb(err): cb(null); + }); + } + + this.invokeFunction = function(cb) { + var proc = this, emul = proc.engine.emulate; + var asyncInvocation = false; + var funcIns = [], funcOuts = []; + + //console.log(proc); + // create arrays of data ins and outs ids + for (var i=0; i Date: Thu, 6 Mar 2014 08:52:04 +0100 Subject: [PATCH 33/73] Test workflow for the new 'join' process type. --- workflows/Wf_join_test.json | 65 +++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 workflows/Wf_join_test.json diff --git a/workflows/Wf_join_test.json b/workflows/Wf_join_test.json new file mode 100644 index 0000000..486b5f0 --- /dev/null +++ b/workflows/Wf_join_test.json @@ -0,0 +1,65 @@ +{ + "name": "Wf_join_test", + "functions": [ { + "name": "chooseEvenOdd", + "module": "functions" + }, { + "name": "print", + "module": "functions" + }, { + "name": "echo", + "module": "functions" + } ], + "processes": [ { + "name": "Choice", + "type": "choice", + "function": "chooseEvenOdd", + "parlevel": 0, + "ins": [ "num" ], + "outs": [ "OutEven", "OutOdd" ] + }, { + "name": "SumEven", + "type": "dataflow", + "function": "echo", + "parlevel": 0, + "ins": [ "OutEven" ], + "outs": [ "ResultEven" ] + }, { + "name": "SumOdd", + "type": "dataflow", + "function": "echo", + "parlevel": 0, + "ins": [ "OutOdd" ], + "outs": [ "ResultOdd" ] + }, { + "name": "Join", + "type": "join", + "function": "print", + "joinCount": 1, + "activeBranchesCount": 2, + "ins": [ "ResultEven", "ResultOdd" ], + "outs": [ ] + } ], + "signals": [ { + "name": "num", + "data": [ { "value": "1" }, + { "value": "2" }, + { "value": "1" }, + { "value": "2" }, + { "value": "1" }, + { "value": "2" }, + { "value": "2" }, + { "value": "1" } + ] + }, { + "name": "OutEven" + }, { + "name": "OutOdd" + }, { + "name": "ResultEven" + }, { + "name": "ResultOdd" + } ], + "ins": [ "num" ], + "outs": [ ] +} From 20b6b9f5b1997271e7dabe5e7a19d1b1dd1cde7b Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Thu, 6 Mar 2014 09:38:54 +0100 Subject: [PATCH 34/73] Generic logic of a process, extended by concrete process types. --- engine2/process.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/engine2/process.js b/engine2/process.js index 41cc34c..c69983a 100644 --- a/engine2/process.js +++ b/engine2/process.js @@ -298,8 +298,6 @@ function extend(subc, superc) { superc.prototype.constructor = superc; } - // los metodos de superc, que no esten en esta clase, crear un metodo que - // llama al metodo de superc. for ( method in subcp ) { if (subcp.hasOwnProperty(method)) { subc.prototype[method] = subcp[method]; From 123553bc5c57259fcffdf07e4fb2ee1e91caf48e Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Thu, 6 Mar 2014 11:10:00 +0100 Subject: [PATCH 35/73] Fixed bug when setting 'eventServer' where process had no 'config' attribute. --- wflib/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wflib/index.js b/wflib/index.js index d249f84..247fb49 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -226,8 +226,8 @@ exports.init = function(redisClient) { } } copy.fun = task.function ? task.function: "null"; // FIXME: unify this attr name - if (!copy.config) - copy.config = "null"; + /*if (!copy.config) + copy.config = "null";*/ copy.status = "waiting"; return copy; } @@ -1357,7 +1357,7 @@ function public_invokeTaskFunction2(wfId, taskId, insIds_, insValues, outsIds_, //onsole.log("INS:", ins); //onsole.log("OUTS:", outs); //onsole.log(JSON.stringify(taskInfo.config)); //DEBUG - var conf = taskInfo.config ? JSON.parse(taskInfo.config): null; + var conf = taskInfo.config ? JSON.parse(taskInfo.config): {}; //var executor = taskInfo.executor ? taskInfo.executor: null; //onsole.log("INS VALUES", insValues); From 70a117572f108769274daed17688501222a70a0c Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Thu, 6 Mar 2014 23:25:38 +0100 Subject: [PATCH 36/73] - Added handling of remote sinks for a signal - Extended WfMap: control signal sets, corrected sticky signal set --- wflib/index.js | 173 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 160 insertions(+), 13 deletions(-) diff --git a/wflib/index.js b/wflib/index.js index c3a5507..fb4fd63 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -7,6 +7,8 @@ var fs = require('fs'), async = require('async'), ZSchema = require('z-schema'), value = require('value'), + request = require('request'), + Q = require('q'), //toobusy = require('toobusy'), rcl; @@ -45,10 +47,10 @@ exports.init = function(redisClient) { var wfname = filename.split('.')[0]; rcl.hmset("wftempl:"+wfname, "name", wfname, "maxInstances", "3", function(err, ret) { var start = (new Date()).getTime(), finish; - public_createInstance(JSON.parse(data), baseUrl, function(err, ret) { + public_createInstance(JSON.parse(data), baseUrl, function(err, wfId) { finish = (new Date()).getTime(); console.log("createInstance time: "+(finish-start)+"ms"); - err ? cb(err): cb(null, ret); + err ? cb(err): cb(null, wfId); }); }); }); @@ -120,7 +122,7 @@ exports.init = function(redisClient) { var addSigInfo = function(sigId) { var score = -1; var sigObj = sigs[sigId-1]; - sigObj.status = "not_ready"; + sigObj.status = "not_ready"; // FIXME: remove (deprecated) sigKey = wfKey+":data:"+sigId; if (sigObj.control) { // this is a control signal sigObj.type = "control"; @@ -129,7 +131,7 @@ exports.init = function(redisClient) { } else { // this is a data signal score = 0; } - sigObj.uri = baseUri + '/signals/' + sigId; + sigObj.uri = baseUri + '/sigs/' + sigId; if (sigObj.schema && value(sigObj.schema).typeOf(Object)) { // this is an inline schema //onsole.log("INLINE SCHEMA", sigObj.schema); @@ -146,15 +148,25 @@ exports.init = function(redisClient) { delete sigObj.data; // don't store instances in signal info } + if (sigObj.remoteSinks) { // signal info contains URIs to remote sinks + sigObj.remoteSinks.forEach(function(sink) { + multi.sadd(sigKey+":remotesinks", sink.uri, function(err, ret) { }); + }); + sigObj.remoteSinks = true; // don't store remote sinks URIs in sig info, just a flag + } + + // create a reverse index to look up sig Id by its name (assumes unique names!) + multi.hset(wfKey+":siglookup:name", sigObj.name, sigId, function(err, ret) { }); + multi.hmset(sigKey, sigObj, function(err, ret) { }); - // add this data id to the sorted set of all workflow signals + // add this signal id to the sorted set of all workflow signals // score determines the type/status of the signal: // 0: data signal/not ready, 1: data signal/ready, 2: control signal multi.zadd(wfKey+":data", score, sigId, function(err, ret) { }); } - // add workflow tasks + // add workflow processes var taskKey; for (var i=0; i Date: Thu, 6 Mar 2014 23:28:52 +0100 Subject: [PATCH 37/73] - More engine refactoring - Improved Join process implementation; added "merge" input control signal. --- engine2/ProcDataflowFSM.js | 3 +- engine2/ProcJoinFSM.js | 91 +++++++++++++++++++++++++++++++------- engine2/index.js | 3 +- engine2/process.js | 10 ++--- 4 files changed, 82 insertions(+), 25 deletions(-) diff --git a/engine2/ProcDataflowFSM.js b/engine2/ProcDataflowFSM.js index 0095c74..6932d13 100644 --- a/engine2/ProcDataflowFSM.js +++ b/engine2/ProcDataflowFSM.js @@ -34,14 +34,13 @@ var DataflowLogic = function() { // set firing sigs for (var i in this.ins) { var sigId = this.ins[i]; - if (!(sigId in this.ctrIns)) { + if (!this.fullInfo.cinset[sigId]) { this.firingSigs.push([sigId, 1]); } } if ("next" in this.ctrIns) { this.firingSigs.push([this.ctrIns.next,1]); } - //console.log(JSON.stringify(this.firingSigs)); } diff --git a/engine2/ProcJoinFSM.js b/engine2/ProcJoinFSM.js index 53a7034..8f75b7f 100644 --- a/engine2/ProcJoinFSM.js +++ b/engine2/ProcJoinFSM.js @@ -6,12 +6,14 @@ ** determine its behavior: ** - Nb (activeBranchesCount): how many branches are active? ** - Nj (joinCount): how many branches should we wait for before firing the process? + ** where + ** Nj <= Nb <= N (total number of branches / input signals) ** ** The process will: ** - fire after 'Nj' signals have arrived ** - "reset" after 'Nb' signals have arrived (only then it will be ready for next firing). ** - ** This process type allows to implement the following workflow patterns + ** This process type allows one to implement the following workflow patterns ** (see http://www.workflowpatterns.com/patterns/control): ** - Structured discriminator ** - Blocking discriminator (?) @@ -20,16 +22,19 @@ ** - Local synchronizing merge ** ** Inputs: - ** - ... + ** - One or more data signals + ** - Optional 'merge' control signal (emitted by the 'choice' process, so that 'join' can + ** merge branches activated by 'choice') + ** - Optional 'done' and 'next' control signals ** Outputs: - ** - ... + ** - Zero or more data outputs + ** - Optional 'done' and 'next' control signals ** Function: ** - The function is passed only Nj signals (ones that arrived first). Consequently, ** the signals should be recognized by name, not by index, as it is not known which ** Nj (out of Nb) signals actually caused the firing. ** */ - var async = require('async'), ProcLogic = require('./process.js').ProcLogic, fireInput = require('./process.js').fireInput, @@ -38,16 +43,30 @@ var async = require('async'), function JoinLogic() { ProcLogic.call(this); + // firingSigsH[0] and paramsH[0] store, respectively, signal ids and Nb/Nj params to be used + // in the next firing. firingSigsH[i] / paramsH[i] store the same for the i-th firing from "now". + // paramsH is used only when there is a 'merge' control input signal. + // The algorithm is quite convoluted; the problem is to compute which signals should be + // fired and which discarded in each firing. + // For example, given Nb=2, Nj=1, and the following order of signals ("1"=branch 1, "2"=branch 2): + // [2,2,2,1,1,1,1,2], the process will fire four times as follows (fired/discarded): + // - 2/1 + // - 2/1 + // - 2/1 + // - 1/2 + // With 'merge' signal, Nb/Nj additionally changes for each firing. this.firingSigsH = []; + this.paramsH = []; + this.canReset = false; this.init2 = function(session) { + // when there is a 'merge' input signal, Nb/Nj are actually computed based on the signal data // Nj: how many branches must arrive before firing? this.Nj = this.fullInfo.joinCount ? this.fullInfo.joinCount: this.nDataIns; - // Nb: how many branches must arrive before resetting the process? this.Nb = this.fullInfo.activeBranchesCount ? this.fullInfo.activeBranchesCount: this.nDataIns; - + this.ready = true; session.sessionListener[0].customEvent = fireInputJoin; @@ -128,15 +147,23 @@ function JoinLogic() { return; } + // prevent computation of Nb0/Nj0 after reset, when there are + // no more 'merge' signals waiting (paramsH is empty!). + if (proc.ctrIns.merge && !proc.paramsH.length) + return; + + var Nb0 = proc.ctrIns.merge ? proc.paramsH[0].Nb: proc.Nb, + Nj0 = proc.ctrIns.merge ? proc.paramsH[0].Nj: proc.Nj; + if (proc.ready) { //onsole.log("TRY TRANSITION", proc.firingSigsH); - if (proc.firingSigsH[0] && proc.firingSigsH[0].length == proc.Nb) { + if (proc.firingSigsH[0] && proc.firingSigsH[0].length == Nb0) { proc.canReset = true; } - if (proc.firingSigsH[0] && proc.firingSigsH[0].length >= proc.Nj) { + if (proc.firingSigsH[0] && proc.firingSigsH[0].length >= Nj0) { proc.ready = false; proc.firingSigs = []; - for (var i=0; i= proc.Nb) || + if ((proc.firingSigsH[idx].length >= Nb) || (proc.firingSigsH[idx].indexOf(sigId) != -1)) { return qsigs(idx+1); } @@ -254,12 +310,13 @@ function fireInputJoin(obj) { } else { proc.firingSigsH[idx] = [ sigId ]; } - + //onsole.log("QSIG", idx, proc.firingSigsH); } - var idx = Math.floor(proc.dataIns[sigId]-1 / proc.Nj); + var idx = Math.floor(proc.dataIns[sigId]-1); qsigs(idx); - if (proc.firingSigsH[0] && proc.firingSigsH[0].length == proc.Nb) { + var Nb0 = proc.ctrIns.merge ? proc.paramsH[0].Nb: proc.Nb; + if (proc.firingSigsH[0] && proc.firingSigsH[0].length == Nb0) { proc.canReset = true; } diff --git a/engine2/index.js b/engine2/index.js index 4c24b8e..ed4a5ee 100644 --- a/engine2/index.js +++ b/engine2/index.js @@ -227,7 +227,8 @@ Engine.prototype.emitSignals = function(sigs, cb) { engine.tasks[sinks[j]].fireCustomEvent({ wfId: engine.wfId, taskId: sinks[j], - sigId: _sigId + sigId: _sigId, + sig: s }); //onsole.log("sending signal", _sigId, "to task", sinks[j]); } diff --git a/engine2/process.js b/engine2/process.js index c69983a..7b9719e 100644 --- a/engine2/process.js +++ b/engine2/process.js @@ -21,7 +21,7 @@ function fireInput(obj) { sigId = msg.sigId, sig = msg.sig; - //console.log("FIRE INPUT Proc", proc.procId); + //onsole.log("FIRE INPUT Proc", proc.procId); if (sigId == proc.ctrIns.done) { // "done" signal has arrived proc.done = true; @@ -154,7 +154,7 @@ var ProcLogic = function() { var sigs = proc.firingSigs; proc.wflib.fetchInputs(proc.appId, proc.procId, sigs, true, function(arrived, sigValues) { if (arrived) { - //console.log("FETCHED", sigs, proc.procId); + //onsole.log("FETCHED", sigs, proc.procId); if (sigs[sigs.length-1][0] == proc.ctrIns.next) { sigValues.pop(); // remove 'next' signal (should not be passed to the function) } @@ -186,17 +186,17 @@ var ProcLogic = function() { var asyncInvocation = false; var funcIns = [], funcOuts = []; - //console.log(proc); + //onsole.log(proc); // create arrays of data ins and outs ids for (var i=0; i Date: Thu, 6 Mar 2014 23:30:05 +0100 Subject: [PATCH 38/73] Changed function. --- workflows/Wf_indeterminate.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflows/Wf_indeterminate.json b/workflows/Wf_indeterminate.json index 629fc6d..403f2d9 100644 --- a/workflows/Wf_indeterminate.json +++ b/workflows/Wf_indeterminate.json @@ -4,7 +4,7 @@ "name": "echoWithDelay", "module":"functions" }, { - "name": "echo", + "name": "print2", "module":"functions" } ], "processes": [ { @@ -38,7 +38,7 @@ }, { "name": "Merger", "type": "dataflow", - "function": "echo", + "function": "print2", "firingLimit": 16, "ins": [ 4 ], "outs": [ 5 ] From 7d0874b2732e63a1ec916f13dde928d051d0bfd8 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Thu, 6 Mar 2014 23:30:21 +0100 Subject: [PATCH 39/73] Join test workflow now uses "merge" control signal. --- workflows/Wf_join_test.json | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/workflows/Wf_join_test.json b/workflows/Wf_join_test.json index 486b5f0..716cdf4 100644 --- a/workflows/Wf_join_test.json +++ b/workflows/Wf_join_test.json @@ -36,8 +36,8 @@ "type": "join", "function": "print", "joinCount": 1, - "activeBranchesCount": 2, - "ins": [ "ResultEven", "ResultOdd" ], + "activeBranchesCount": 1, + "ins": [ "ResultEven", "ResultOdd", "mergeControl" ], "outs": [ ] } ], "signals": [ { @@ -48,8 +48,10 @@ { "value": "2" }, { "value": "1" }, { "value": "2" }, + { "value": "1" }, { "value": "2" }, - { "value": "1" } + { "value": "1" }, + { "value": "2" } ] }, { "name": "OutEven" @@ -59,6 +61,16 @@ "name": "ResultEven" }, { "name": "ResultOdd" + }, { + "name": "mergeControl", + "control": "merge", + "data": [ {"Nb": 2, "Nj": 2}, + {"Nb": 1, "Nj": 1}, + {"Nb": 2, "Nj": 1}, + {"Nb": 2, "Nj": 2}, + {"Nb": 2, "Nj": 1}, + {"Nb": 1, "Nj": 1} + ] } ], "ins": [ "num" ], "outs": [ ] From eeb38614d9e3306bc2d95834126c92ff4adc0efb Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Fri, 7 Mar 2014 21:34:56 +0100 Subject: [PATCH 40/73] Choice process now can emit the 'merge' control signal. --- engine2/ProcChoiceFSM.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/engine2/ProcChoiceFSM.js b/engine2/ProcChoiceFSM.js index 4c9f5c1..7592f3c 100644 --- a/engine2/ProcChoiceFSM.js +++ b/engine2/ProcChoiceFSM.js @@ -30,7 +30,6 @@ var ChoiceLogic = function() { ProcLogic.call(this); this.init2 = function() { - console.log("MISTERSON"); // set firing sigs for (var i in this.ins) { var sigId = this.ins[i]; @@ -132,6 +131,14 @@ var ChoiceLogic = function() { } } + // if there exists "merge" output, emit the 'merge' control signal first. + // see 'join' process for explanation + if (proc.ctrOuts.merge) { + var Nj = outValues.length, Nb = Nj; + proc.engine.emitSignals([{"_id": proc.ctrOuts.merge, "data": [{"Nb": Nb, "Nj": Nj}]}], + function(err) { }); + //onsole.log("CHOICE EMIT MERGE", proc.ctrOuts.merge); + } if (proc.ctrOuts.next) { // emit "next" signal if there is such an output port outValues.push({"_id": proc.ctrOuts.next }); } From c80ca28bd5b1b498e1474465309fc8dca652aaa4 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Fri, 7 Mar 2014 21:35:26 +0100 Subject: [PATCH 41/73] Implemented the 'merge' control signal. --- engine2/ProcJoinFSM.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/engine2/ProcJoinFSM.js b/engine2/ProcJoinFSM.js index 8f75b7f..9861f9d 100644 --- a/engine2/ProcJoinFSM.js +++ b/engine2/ProcJoinFSM.js @@ -281,10 +281,11 @@ function fireInputJoin(obj) { sigId = msg.sigId, sig = msg.sig; - //onsole.log("RECV SIG", sig.name); + console.log("RECV SIG", sig.name, proc.ctrIns.merge); if (sigId == proc.ctrIns.done) { // "done" signal has arrived proc.done = true; } else if (proc.ctrIns.merge && sigId == proc.ctrIns.merge) { // there is a 'merge' input port + //onsole.log("JOIN RECV MERGE"); proc.paramsH.push({ "Nj": sig.data[0].Nj, "Nb": sig.data[0].Nb}); } else { if (!proc.dataIns[sigId]) { @@ -299,6 +300,7 @@ function fireInputJoin(obj) { // algorithm which places a new sig in the appropriate "set" and determines // whether the signal will be fired or discarded, and in which firing var qsigs = function(idx) { + //console.log("QSIG", idx, proc.firingSigsH); Nb = proc.ctrIns.merge ? proc.paramsH[idx].Nb: proc.Nb; Nj = proc.ctrIns.merge ? proc.paramsH[idx].Nj: proc.Nj; if (proc.firingSigsH[idx]) { @@ -310,9 +312,8 @@ function fireInputJoin(obj) { } else { proc.firingSigsH[idx] = [ sigId ]; } - //onsole.log("QSIG", idx, proc.firingSigsH); } - var idx = Math.floor(proc.dataIns[sigId]-1); + var idx = proc.dataIns[sigId]-1; qsigs(idx); var Nb0 = proc.ctrIns.merge ? proc.paramsH[0].Nb: proc.Nb; From 37ae9bc44f8d007d8ec0f84831ec2f714a971aed Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Fri, 7 Mar 2014 21:47:30 +0100 Subject: [PATCH 42/73] Cleanup of old, unused code. --- engine2/index.js | 121 +---------------------------------------------- 1 file changed, 1 insertion(+), 120 deletions(-) diff --git a/engine2/index.js b/engine2/index.js index ed4a5ee..41ff2cb 100644 --- a/engine2/index.js +++ b/engine2/index.js @@ -129,30 +129,6 @@ Engine.prototype.runInstanceSync = function(callback) { } -// Marks data elements as 'ready' and notify their sinks -// dataIds - single data Id or an array of dataIds -// FIXME: check what happens if data is marked more than once -// quasi-deprecated (should refactor remaining places which still us this old API) -// now fireSignals should be used -Engine.prototype.markDataReady = function(dataIds, cb) { - function isArray(what) { - return Object.prototype.toString.call(what) === '[object Array]'; - } - - var Ids = []; - isArray(dataIds) ? Ids = dataIds: Ids.push(dataIds); - //var start = (new Date()).getTime(), finish; - (function(engine) { - Ids.forEach(function(dataId) { - markDataReadyAndNotifySinks(engine.wfId, dataId, engine.tasks, engine.wflib, function() { - //finish = (new Date()).getTime(); - //onsole.log("markDataReady exec time: "+(finish-start)); - }); - }); - })(this); - cb(null); -} - Engine.prototype.taskFinished = function(taskId) { this.trace += taskId; this.nTasksLeft--; @@ -172,8 +148,7 @@ Engine.prototype.workflowFinished = function() { } } -// NEW API for sending signals for continuous processes with FIFO queues -// WILL DEPRECATE fireSignals +// Function used by processes to emit signals // @sigs (array): ids of signals (data and control ones) to be emitted // format: [ { attr: value, // ... @@ -244,98 +219,4 @@ Engine.prototype.emitSignals = function(sigs, cb) { }); } - -// Fires a set of signals notifying all tasks which are their sinks. -// For data signals also updates their state (e.g. marks data elements as ready) -// @sigs (array): ids of signals (data and control ones) to be fired -// format: [ { attr: value, -// ... -// "id": sigId -// }, -// { attr: value, -// ... -// "id": sigId -// } -// ] -// FIXME: protect against firing signals more than once (currently some task -// implementations will break in such a case). -// Will be DEPRECATED by emitSignals -Engine.prototype.fireSignals = function(sigs, cb) { - var spec = {}, ids = []; - for (var i in sigs) { - var sigId = sigs[i].id; - ids.push(sigId); - if (sigs[i].type == "control") { // this is a control signal - spec[sigId] = {}; // TODO: should we also mark this signal as ready? - } else { // this is a data signal - spec[sigId] = sigs[i]; // copy all attributes... - delete spec[sigId].id; // ... except 'id' - spec[sigId].status = "ready"; // mark data element as "ready" (produced) - if (this.wfOuts.indexOf(sigId) != -1) { // a workflow output has been produced - this.nWfOutsLeft--; - } - } - } - - // notify sinks of all fired signals - (function(engine) { - //onsole.log(spec); - engine.wflib.setDataState(engine.wfId, spec, function(err, reps) { - //onsole.log("Will notify: "+JSON.stringify(sigs)); - async.each(ids, function iterator(sigId, doneIter) { - notifySinks(engine.wfId, sigId, engine.tasks, engine.wflib, function() { - doneIter(null); - }); - }, function doneAll(err) { - if (cb) { cb(err); } - }); - }); - })(this); -} - module.exports = Engine; - - ////////////////////////////////////////////////////////////////////////// - ///////////////////////// private functions ////////////////////////////// - ////////////////////////////////////////////////////////////////////////// - -function notifySinks(wfId, sigId, taskFSMs, wflib, cb) { - wflib.getDataSinks(wfId, sigId, true, function(err, sinks) { - //onsole.log(sigId, sinks); - if (err) { throw(err); } - for (var j=0; j Date: Fri, 7 Mar 2014 21:59:25 +0100 Subject: [PATCH 43/73] Corrections in comments --- test/remote_pingpong_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/remote_pingpong_test.sh b/test/remote_pingpong_test.sh index b1280ea..a922825 100644 --- a/test/remote_pingpong_test.sh +++ b/test/remote_pingpong_test.sh @@ -17,11 +17,11 @@ fi # {app1} http://localhost:{port1}/apps/{appId1} # {app2} http://localhost:{port2}/apps/{appId2} -# REST-based protocol is as follows (<...> denotes the content of message body): +# REST-based protocol is as follows ('<...>' denotes the content of a message body): # 1) POST {appfactory1} ==> create Pinger instance, returns URI {app1}. # 2) POST {appfactory2} ==> create Ponger instance, returns URI {app2}. -# 3) PUT {app1}/sigs/Ping/remotesinks <{app2}> ==> connects signal "Ping" from app1 to app2. -# 4) PUT {app2}/sigs/Pong/remotesinks <{app1}> ==> connects signal "Pong" from app2 to app1. +# 3) PUT {app1}/sigs/Pong/remotesinks <{app2}> ==> connects signal "Pong" from app1 to app2. +# 4) PUT {app2}/sigs/Ping/remotesinks <{app1}> ==> connects signal "Ping" from app2 to app1. # 5) POST {app1} ==> sends the initial signal to Pinger to start the Ping-Pong. appfact1="http://localhost:$1/apps" From 9fbe6714a7b4198d4d0fa8b8e97ee32b824f0246 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 10 Mar 2014 15:40:17 +0100 Subject: [PATCH 44/73] Added .idea to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 70acd70..182ae20 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ *~ *.swp +.idea From 7fc83a4fd72a7eeccdb9da8cd22a1669584fb2e8 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Mon, 10 Mar 2014 17:14:32 +0100 Subject: [PATCH 45/73] Add dependencies to package.json and fix minor typo. --- engine2/index.js | 2 +- package.json | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/engine2/index.js b/engine2/index.js index 41ff2cb..d90e578 100644 --- a/engine2/index.js +++ b/engine2/index.js @@ -20,7 +20,7 @@ var fs = require('fs'), var ProcDataflowFSM = require('./ProcDataflowFSM.js'); var ProcChoiceFSM = require('./ProcChoiceFSM.js'); var ProcForeachFSM = require('./ProcForeachFSM.js'); -var ProcJoinFSM = require('./procJoinFSM.js'); +var ProcJoinFSM = require('./ProcJoinFSM.js'); var ProcSplitterFSM = require('./ProcSplitterFSM.js'); diff --git a/package.json b/package.json index 8356a56..4d88266 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,8 @@ "file": "0.2.x", "value": "0.3.0", "form-data": "", - "nodeunit": "" + "nodeunit": "", + "request": "", + "q": "" } } From faa1979b58236b07b01a362d9ae358b24f47c21a Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 11 Mar 2014 15:57:06 +0100 Subject: [PATCH 46/73] Add support for eventServer in engine2. --- engine2/index.js | 5 ++++- engine2/process.js | 11 ++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/engine2/index.js b/engine2/index.js index d90e578..023fa0e 100644 --- a/engine2/index.js +++ b/engine2/index.js @@ -15,7 +15,9 @@ var fs = require('fs'), xml2js = require('xml2js'), fsm = require('./automata.js'), - async = require('async'); + async = require('async'), + eventServerFactory = require('../eventlog'); + var ProcDataflowFSM = require('./ProcDataflowFSM.js'); var ProcChoiceFSM = require('./ProcChoiceFSM.js'); @@ -37,6 +39,7 @@ fsm.registerFSM(ProcSplitterFSM); // - config.emulate (true/false) = should engine work in the emulation mode? var Engine = function(config, wflib, wfId, cb) { this.wflib = wflib; + this.eventServer = eventServerFactory.createEventServer(); this.wfId = wfId; this.tasks = []; // array of task FSMs this.ins = []; diff --git a/engine2/process.js b/engine2/process.js index 7b9719e..15c3efe 100644 --- a/engine2/process.js +++ b/engine2/process.js @@ -234,11 +234,12 @@ var ProcLogic = function() { */ proc.wflib.invokeTaskFunction2( - proc.appId, - proc.procId, - funcIns, - proc.sigValues, - funcOuts, emul, + proc.appId, + proc.procId, + funcIns, + proc.sigValues, + funcOuts, emul, + proc.engine.eventServer, function(err, outs) { err ? cb(err): cb(null, outs, asyncInvocation, funcIns, funcOuts); } From 0d023c40534918fed99d32ead1efedc4ad1393ad Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Tue, 11 Mar 2014 17:13:08 +0100 Subject: [PATCH 47/73] Minor updates. --- functions/index.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/functions/index.js b/functions/index.js index ddc51ca..bf184f6 100644 --- a/functions/index.js +++ b/functions/index.js @@ -4,6 +4,7 @@ var fsp = require('./fileSplitter.js'), scanDir = require('./DirScanner').scanDir; function print(ins, outs, config, cb) { + //console.log("PRINT", JSON.stringify(ins)); ins.forEach(function(input) { //console.log("sigId=", input.sigId + ":", input.data[0]) //console.log(JSON.stringify(input, null, 2)); @@ -16,6 +17,13 @@ function print(ins, outs, config, cb) { cb(null, outs); } +function print2(ins, outs, config, cb) { + ins.forEach(function(input) { + console.log(input.data[0]); + }); + cb(null, outs); +} + function echo(ins, outs, config, cb) { var data = JSON.stringify(ins[0].data); //console.log(data); @@ -25,7 +33,7 @@ function echo(ins, outs, config, cb) { //if (typeof data == "object" || typeof data == "array") // data = JSON.stringify(data); - process.stdout.write(data[2]); + //process.stdout.write(data[2]); //console.log(data); cb(null, outs); } @@ -132,13 +140,16 @@ function grepFile(ins, outs, config, cb) { var cnt = 0; function count(ins, outs, config, cb) { - //console.log(ins); + //console.log(ins, outs); cnt++; outs[0].data = []; outs[0].data[0] = cnt; if (cnt % 1000 == 0) { console.log("count:", cnt) } + if (cnt % 1000 == 1) { + console.log(ins, outs); + } if (cnt == 10000) process.exit(); cb(null, outs); @@ -158,6 +169,7 @@ function montage_mProjectPP(ins, outs, config, cb) { */ exports.print = print; +exports.print2 = print2; exports.add = add; exports.sqr = sqr; exports.length = length; From 9064d8d89e634eb4df9b878dc45517cfb887fa56 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Tue, 11 Mar 2014 18:18:18 +0100 Subject: [PATCH 48/73] Added implementation of a few GET operations. --- app.js | 49 +++++++++++++++++++++++++++++++++-------- scripts/restapi_test.sh | 4 ++-- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/app.js b/app.js index 8625ed6..7b60f35 100644 --- a/app.js +++ b/app.js @@ -111,20 +111,51 @@ app.post('/apps', function(req, res) { // returns workflow instance ('app') info app.get('/apps/:i', function(req, res) { + var appId = req.params.i; + var appIns, appOuts; + var wfInstanceStatus = "unknown"; + + if (!(appId in engine)) return notfound(res); // 404 + var renderHTML = function() { + var start, end; var ctype = acceptsXml(req); res.header('content-type', ctype); - res.send('GET /apps/{appId}'); - //res.render ... TODO + start = (new Date()).getTime(); + res.render('workflow-instance', { + title: 'Application', + nr: appId, + host: req.headers.host, + wfname: "Application", + wfins: appIns, + wfouts: appOuts, + stat: wfInstanceStatus, + now: (new Date()).getTime(), + submit_inputs_uri: '/apps/'+appId + }, function(err, html) { + if (err) { throw(err); } + end = (new Date()).getTime(); + console.log("rendering page: "+(end-start)+"ms, length: "+html.length); + res.statuscode = 200; + res.send(html); + }); + } + var renderJSON = function() { res.header('content-type', 'text/plain'); res.send('GET /apps/{appId}'); // res.render ... TODO } - res.format({ - 'text/html': renderHTML, - 'application/json': renderJSON + + wflib.getWfInsAndOutsInfoFull(req.params.i, function(err, ins, outs) { + if (err) return notfound(res); + appIns = ins; + appOuts = outs; + res.format({ + 'text/html': renderHTML, + 'application/json': renderJSON + }); }); }); @@ -222,8 +253,8 @@ app.get('/apps/:i/ins', function(req, res) { // returns info about a signal exchanged within the workflow app.get('/apps/:i/sigs/:j', function(req, res) { - var wfId = req.params.i, dataId = req.params.j; - wflib.getDataInfoFull(wfId, dataId, function(err, wfData, dSource, dSinks) { + var appId = req.params.i, sigId = req.params.j; + wflib.getDataInfoFull(appId, sigId, function(err, wfData, dSource, dSinks) { if (err) { res.statusCode = 404; res.send(inst.toString()); @@ -232,10 +263,10 @@ app.get('/apps/:i/sigs/:j', function(req, res) { res.header('content-type', ctype); res.render('workflow-data', { title: 'workflow data', - wfname: req.params.w, + wfname: "Application", data: wfData, source: dSource, - data_id: dataId, + data_id: sigId, sinks: dSinks }); } diff --git a/scripts/restapi_test.sh b/scripts/restapi_test.sh index c6984fa..75da7cc 100644 --- a/scripts/restapi_test.sh +++ b/scripts/restapi_test.sh @@ -18,11 +18,11 @@ uri="http://localhost:$1/apps" # Body: valid workflow description in JSON # on success returns: 201, Location: {appuri} # "location" - wf instance URI extracted from the HTTP header -location=`curl -v -X POST -d @workflows/Wf_Rest_test.json $uri --header "Content-Type:application/json" 2>&1 | grep Location | cut -f 3 -d' '` +location=`curl -v -X POST -d @workflows/Montage_143.json $uri --header "Content-Type:application/json" 2>&1 | grep Location | cut -f 3 -d' '` appuri="http://localhost:$1"$location echo $appuri # 2) POST {appuri} - sends a signal to a workflow # Body: valid signal data (JSON with mandatory "name") -curl -X POST -d '{ "name": "counter1", "data": [0] }' $appuri --header "Content-Type:application/json" +# curl -X POST -d '{ "name": "counter1", "data": [0] }' $appuri --header "Content-Type:application/json" From 17e2558709e0b3cb451da49b6aa62f3d01c558f2 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 12 Mar 2014 10:09:43 +0100 Subject: [PATCH 49/73] First workflow draft for ISMOP monitoring application (P1) --- workflows/ismop/LMonFunctions.js | 84 ++++++++++++++++++++++++++++++++ workflows/ismop/LMonP1.json | 40 +++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 workflows/ismop/LMonFunctions.js create mode 100644 workflows/ismop/LMonP1.json diff --git a/workflows/ismop/LMonFunctions.js b/workflows/ismop/LMonFunctions.js new file mode 100644 index 0000000..d1790b2 --- /dev/null +++ b/workflows/ismop/LMonFunctions.js @@ -0,0 +1,84 @@ +var request = require('request'); // http client + + +var EmergLevel = { + NONE: "none", + HIGHTENED: "hightened", + SEVERE: "severe" +}; + +var ThreatLevel = { + NONE: "none", + HIGHTENED: "hightened", + SEVERE: "severe" +}; + +// emulates values written in DAP (to be removed) +var emLevelPersistent = EmergLevel.NONE, thrLevelPersistent = ThreatLevel.NONE; + +// Step 1: periodically reads the Levee state from DAP. +// The main parameter of interest is `emergencyLevel': +// - If 'none', no action taken +// - If 'hightened', triggers computation of the current threat level +// - If 'severe', triggers appropriate actions +function getLeveeState(ins, outs, config, cb) { + // var leveeURI = config.leveeUri; // URI could be passed through config + // TODO: invoke DAP's REST API to retrieve levee state + + var emergencyLevel, threatLevel; + + var rand = Math.random(); // TODO: to be set by the result of REST invocation + if (rand > 0.95) { + emergencyLevel = EmergLevel.SEVERE; + } else if (rand > 0.7) { + emergencyLevel = EmergLevel.HIGHTENED; + } else { + emergencyLevel = EmergLevel.NONE; + } + console.log("emergencyLevel="+emergencyLevel); + + if (emergencyLevel == EmergLevel.HIGHTENED && thrLevelPersistent == ThreatLevel.NONE) { + console.log("Setting hightened emergency level"); + outs[0].condition = "true"; // emit "ELHightened" signal + outs[0].data = [ { } ]; + } + + if (emergencyLevel == EmergLevel.SEVERE) { + console.log("Setting severe emergency level"); + outs[1].condition = "true"; // emit "ELSevere" signal + outs[1].data = [ { } ]; + } + + cb(null, outs); +} + +// Step 2a: run estimation of the threat level (here will be the Map/Reduce jobs!) +function computeThreatLevel(ins, outs, config, cb) { + var threatLevel; + + var rand = Math.random(); + if (rand > 0.95) { + threatLevel = ThreatLevel.SEVERE; + } else if (rand > 0.7) { + threatLevel = ThreatLevel.HIGHTENED; + } else { + threatLevel = ThreatLevel.NONE; + } + console.log("Computing threat level..."); + + setTimeout(function() { + console.log("threatLevel="+threatLevel); + thrLevelPersistent = threatLevel; // TODO: replace it with POST to DAP + cb(null, outs); + }, 5000); +} + +// Set 2b: perform actions in the severe emergency level +function severeEmergencyActions(ins, outs, config, cb) { + console.log("Severe Emergency Actions!"); + cb(null, outs); +} + +exports.getLeveeState = getLeveeState; +exports.computeThreatLevel = computeThreatLevel; +exports.severeEmergencyActions = severeEmergencyActions; diff --git a/workflows/ismop/LMonP1.json b/workflows/ismop/LMonP1.json new file mode 100644 index 0000000..2f51658 --- /dev/null +++ b/workflows/ismop/LMonP1.json @@ -0,0 +1,40 @@ +{ + "name": "LeveeMonitorP1", + "functions": [ { + "name": "getLeveeState", + "module": "workflows/ismop/LMonFunctions.js" + }, { + "name": "computeThreatLevel", + "module": "workflows/ismop/LMonFunctions.js" + }, { + "name": "severeEmergencyActions", + "module": "workflows/ismop/LMonFunctions.js" + } ], + "processes": [ { + "name": "LeveeStateChecker", + "type": "choice", + "function": "getLeveeState", + "firingInterval": 5000, + "ins": [ ], + "outs": [ "ELHightened", "ELSevere" ] + }, { + "name": "ThreatLevelComputation", + "type": "dataflow", + "function": "computeThreatLevel", + "ins": [ "ELHightened" ], + "outs": [ ] + }, { + "name": "SevereEmergency", + "type": "dataflow", + "function": "severeEmergencyActions", + "ins": [ "ELSevere" ], + "outs": [ ] + } ], + "signals": [ { + "name": "ELHightened" + }, { + "name": "ELSevere" + } ], + "ins": [ ], + "outs": [ ] +} From ac6d7719b74fc004b747b1656d1c102dacb1c060 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 12 Mar 2014 10:10:43 +0100 Subject: [PATCH 50/73] Path of function's module now computed based on "process.cwd()" --- wflib/index.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wflib/index.js b/wflib/index.js index 934e885..39e269f 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -9,6 +9,7 @@ var fs = require('fs'), value = require('value'), request = require('request'), Q = require('q'), + pathTool = require('path'), //toobusy = require('toobusy'), rcl; @@ -1415,8 +1416,8 @@ function public_invokeTaskFunction2(wfId, taskId, insIds_, insValues, outsIds_, rcl.hgetall("wf:functions:"+taskInfo.fun, function(err, fun) { if (err) return cb(err); - // FIXME: how to know the (relative?) path to the module? - var f = require('../'+fun.module)[taskInfo.fun]; + var fpath = pathTool.join(process.cwd(), fun.module); + var f = require(fpath)[taskInfo.fun]; //onsole.log("INS:", ins); //onsole.log("OUTS:", outs); //onsole.log(JSON.stringify(taskInfo.config)); //DEBUG From 6d6f6cabbd3560129a7c38431a611feb324e7b4d Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 12 Mar 2014 10:11:51 +0100 Subject: [PATCH 51/73] Bugfix: callback was never called when there were no output signals to emit. --- engine2/ProcChoiceFSM.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/engine2/ProcChoiceFSM.js b/engine2/ProcChoiceFSM.js index 7592f3c..b6fae94 100644 --- a/engine2/ProcChoiceFSM.js +++ b/engine2/ProcChoiceFSM.js @@ -49,7 +49,7 @@ var ChoiceLogic = function() { proc.ready = true; proc.tryTransition(proc, session); - //onsole.log("Enter state ready: "+task.procId); + //onsole.log("Enter state ready: "+proc.procId); }; this.running_enter = function(session, state, transition, msg) { @@ -128,7 +128,6 @@ var ChoiceLogic = function() { outs[i]["source"] = proc.procId; outs[i]["firingId"] = firingId; outValues.push(outs[i]); - } } // if there exists "merge" output, emit the 'merge' control signal first. @@ -148,6 +147,8 @@ var ChoiceLogic = function() { //onsole.log("runningCount (" + proc.fullInfo.name + ")/2:", proc.runningCount); err ? cb(err): cb(null); }); + } else { + cb(null); } } From 8e33514f4d8baa62f94637e79014382a42d2db1a Mon Sep 17 00:00:00 2001 From: balis Date: Wed, 12 Mar 2014 10:31:07 +0100 Subject: [PATCH 52/73] Minor comment correction --- workflows/ismop/LMonFunctions.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/ismop/LMonFunctions.js b/workflows/ismop/LMonFunctions.js index d1790b2..255af08 100644 --- a/workflows/ismop/LMonFunctions.js +++ b/workflows/ismop/LMonFunctions.js @@ -73,7 +73,7 @@ function computeThreatLevel(ins, outs, config, cb) { }, 5000); } -// Set 2b: perform actions in the severe emergency level +// Step 2b: perform actions in the severe emergency level function severeEmergencyActions(ins, outs, config, cb) { console.log("Severe Emergency Actions!"); cb(null, outs); From d33cef92da2d5e1f607d9d9597ac0496d78b68fb Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 12 Mar 2014 14:52:02 +0100 Subject: [PATCH 53/73] Add missing parameters to invokeTaskFunction2 calls. --- engine/taskChoiceFSM.js | 3 ++- engine/taskSplitterFSM.js | 3 ++- engine2/ProcSplitterFSM.js | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/engine/taskChoiceFSM.js b/engine/taskChoiceFSM.js index a0700c2..7899268 100644 --- a/engine/taskChoiceFSM.js +++ b/engine/taskChoiceFSM.js @@ -267,7 +267,8 @@ function TaskLogic() { task.id, funcIns, task.sigValues, - funcOuts, emul, + funcOuts, emul, + task.engine.eventServer, function(err, outs) { err ? cb(err): cb(null, outs); } diff --git a/engine/taskSplitterFSM.js b/engine/taskSplitterFSM.js index 6a91710..1fe6257 100644 --- a/engine/taskSplitterFSM.js +++ b/engine/taskSplitterFSM.js @@ -268,7 +268,8 @@ function TaskLogic() { task.id, funcIns, task.sigValues, - funcOuts, emul, + funcOuts, emul, + task.engine.eventServer, function(err, outs) { err ? cb(err): cb(null, outs); } diff --git a/engine2/ProcSplitterFSM.js b/engine2/ProcSplitterFSM.js index 6a91710..1fe6257 100644 --- a/engine2/ProcSplitterFSM.js +++ b/engine2/ProcSplitterFSM.js @@ -268,7 +268,8 @@ function TaskLogic() { task.id, funcIns, task.sigValues, - funcOuts, emul, + funcOuts, emul, + task.engine.eventServer, function(err, outs) { err ? cb(err): cb(null, outs); } From 7cd6b0ae6128ba9eca47897d3a7aa97024529251 Mon Sep 17 00:00:00 2001 From: balis Date: Thu, 13 Mar 2014 14:17:53 +0100 Subject: [PATCH 54/73] Delete outdated montage.test.js Generic script scripts/runwf.js should be used to run workflows. --- test/montage.test.js | 42 ------------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 test/montage.test.js diff --git a/test/montage.test.js b/test/montage.test.js deleted file mode 100644 index 04e81c0..0000000 --- a/test/montage.test.js +++ /dev/null @@ -1,42 +0,0 @@ -var redis = require('redis'), - rcl = redis.createClient(), - wflib = require('../wflib').init(rcl), - Engine = require('../engine'), - async = require('async'), - argv = require('optimist').argv, - amqpCommand = require('../functions/amqpCommand'), - engine; - -amqpCommand.connect(function() { - console.log("Connected"); - - function init(cb) { - rcl.select(1, function(err, rep) { - rcl.flushdb(function(err, rep) { - wflib.createInstanceFromFile(argv._[0], '', - function(err, id) { - cb(err, id); - } - ); - }); - }); - } - - - if (!argv._[0]) { - console.log("Usage: node montage.test.js "); - process.exit(); - } - - init(function(err, wfId) { - engine = new Engine({ - "emulate": "false" - }, wflib, wfId, function(err) { - engine.runInstance(function(err) { - wflib.getWfIns(wfId, false, function(err, wfIns) { - engine.markDataReady(wfIns, function(err) {}); - }); - }); - }); - }); - }); From d66bb4f792bc8b250925e9fe162e43ead4b9d0be Mon Sep 17 00:00:00 2001 From: balis Date: Thu, 13 Mar 2014 14:19:39 +0100 Subject: [PATCH 55/73] Delete wfchoice.test.js Generic script scripts/runwf.js should be used to test workflow runs. Initial input signals can now be placed directly into workflow json file. --- test/wfchoice.test.js | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 test/wfchoice.test.js diff --git a/test/wfchoice.test.js b/test/wfchoice.test.js deleted file mode 100644 index f3eb73b..0000000 --- a/test/wfchoice.test.js +++ /dev/null @@ -1,29 +0,0 @@ -var redis = require('redis'), - rcl = redis.createClient(), - wflib = require('../wflib').init(rcl), - Engine = require('../engine'), - async = require('async'), - engine; - -function init(cb) { - rcl.select(1, function(err, rep) { - rcl.flushdb(function(err, rep) { - wflib.createInstanceFromFile('../workflows/Wf_choice_test.json', '', - function(err, id) { - cb(err, id); - } - ); - }); - }); -} - -init(function(err, wfId) { - if (err) { throw err; } - engine = new Engine({"emulate":"false"}, wflib, wfId, function(err) { - engine.runInstance(function(err) { - var spec = [{'id': '1', 'value': '1'}, - {'id': '2', 'value': '2'}]; - engine.fireSignals(spec); - }); - }); -}); From 25dbeaac7af290d4914badf8b93aea5e66ba4973 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Thu, 13 Mar 2014 14:27:15 +0100 Subject: [PATCH 56/73] Removed unused req_url http client ('request' should be used). --- req_url/index.js | 85 ------------------------------------------------ 1 file changed, 85 deletions(-) delete mode 100644 req_url/index.js diff --git a/req_url/index.js b/req_url/index.js deleted file mode 100644 index 100aec0..0000000 --- a/req_url/index.js +++ /dev/null @@ -1,85 +0,0 @@ -// module dependencies -var http = require('http'), - url = require('url'); - - - /** - * UrlReq - Wraps the http.request function making it nice for unit testing APIs. - * - * @param {string} reqUrl The required url in any form - * @param {object} options An options object (this is optional) - * @param {Function} cb This is passed the 'res' object from your request - * - */ -exports.urlReq = function(reqUrl, options, cb){ - if(typeof options === "function"){ cb = options; options = {}; }// incase no options passed in - - // parse url to chunks - reqUrl = url.parse(reqUrl); - - // http.request settings - var settings = { - host: reqUrl.hostname, - port: reqUrl.port || 80, - path: reqUrl.pathname, - headers: options.headers || {}, - method: options.method || 'GET' - }; - - // if there are params: - if(options.params){ - options.params = JSON.stringify(options.params); - settings.headers['Content-Type'] = 'application/json'; - settings.headers['Content-Length'] = options.params.length; - }; - - // MAKE THE REQUEST - var req = http.request(settings); - - // if there are params: write them to the request - if(options.params){ req.write(options.params) }; - - // when the response comes back - req.on('response', function(res){ - res.body = ''; - res.setEncoding('utf-8'); - - // concat chunks - res.on('data', function(chunk){ res.body += chunk }); - - // when the response has finished - res.on('end', function(){ - - // fire callback - cb(res.body, res); - }); - }); - - // end the request - req.end(); -} - -/* -// Simple Example (defaults to GET) -mylib.urlReq('http://mysite.local:82/newUser', function(body, res){ - - // do your stuff - -}); - - - -// More complex Example 2 -mylib.urlReq('http://mysite.local:82/newUser', { - method: 'POST', - params:{ - name: 'Tester', - email: 'Tester@example.com', - password: 'password' - } -}, function(body, res){ - - // do your stuff - -}); -*/ \ No newline at end of file From 471c6d0ea49233c035950fce84f42fd5c949031a Mon Sep 17 00:00:00 2001 From: balis Date: Fri, 14 Mar 2014 19:35:14 +0100 Subject: [PATCH 57/73] Delete splitter.test.js Now generic script 'scripts/runwf.js' should be used to run workflows. Initial signals can be set directly in the workflow file. --- test/splitter.test.js | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 test/splitter.test.js diff --git a/test/splitter.test.js b/test/splitter.test.js deleted file mode 100644 index 2430bf7..0000000 --- a/test/splitter.test.js +++ /dev/null @@ -1,36 +0,0 @@ -var redis = require('redis'), - rcl = redis.createClient(), - wflib = require('../wflib').init(rcl), - Engine = require('../engine'), - async = require('async'), - argv = require('optimist').argv, - engine; - -function init(cb) { - rcl.select(1, function(err, rep) { - rcl.flushdb(function(err, rep) { - wflib.createInstanceFromFile('../workflows/Wf_splitter.json', '', - function(err, id) { - cb(err, id); - } - ); - }); - }); -} - - -if (!argv._[0]) { - console.log("Usage: node splitter.test.js "); - process.exit(); -} - - -init(function(err, wfId) { - if (err) { throw err; } - engine = new Engine({"emulate":"false"}, wflib, wfId, function(err) { - engine.runInstance(function(err) { - var spec = [{'id': '1', 'value': argv._[0]}]; - engine.fireSignals(spec); - }); - }); -}); From 57640a69bb20cc8063df30e8d15ddb453e13ac43 Mon Sep 17 00:00:00 2001 From: balis Date: Fri, 14 Mar 2014 19:37:32 +0100 Subject: [PATCH 58/73] Delete sqrsum.test.js Generic script 'scripts/runwf.js' should be used to run workflows. Initial signals can be set directly in the workflow file. --- test/sqrsum.test.js | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 test/sqrsum.test.js diff --git a/test/sqrsum.test.js b/test/sqrsum.test.js deleted file mode 100644 index 3370323..0000000 --- a/test/sqrsum.test.js +++ /dev/null @@ -1,31 +0,0 @@ -var redis = require('redis'), - rcl = redis.createClient(), - wflib = require('../wflib').init(rcl), - Engine = require('../engine'), - async = require('async'), - engine; - -function init(cb) { - rcl.select(1, function(err, rep) { - rcl.flushdb(function(err, rep) { - wflib.createInstanceFromFile('../workflows/Wf_sqrsum.json', '', - function(err, id) { - cb(err, id); - } - ); - }); - }); -} - -init(function(err, wfId) { - if (err) { throw err; } - engine = new Engine({"emulate":"false"}, wflib, wfId, function(err) { - engine.runInstance(function(err) { - var spec = [{'id': '1', 'value': '1'}, - {'id': '2', 'value': '2'}, - {'id': '3', 'value': '3'}, - {'id': '4', 'value': '4'}]; - engine.fireSignals(spec); - }); - }); -}); From 504a2269521212ff2e71370c606f924ec7622574 Mon Sep 17 00:00:00 2001 From: balis Date: Fri, 14 Mar 2014 19:37:54 +0100 Subject: [PATCH 59/73] Delete engine.test.js --- test/engine.test.js | 4 ---- 1 file changed, 4 deletions(-) delete mode 100644 test/engine.test.js diff --git a/test/engine.test.js b/test/engine.test.js deleted file mode 100644 index 88306e4..0000000 --- a/test/engine.test.js +++ /dev/null @@ -1,4 +0,0 @@ -var engine = require('../engine').init(); - -engine.runInstance(34, true, function(err) { -}); From 4b9cd3ca960c5f8a37dee3ee1f396518ddd53610 Mon Sep 17 00:00:00 2001 From: balis Date: Fri, 14 Mar 2014 19:38:27 +0100 Subject: [PATCH 60/73] Delete dax.test.js --- test/dax.test.js | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 test/dax.test.js diff --git a/test/dax.test.js b/test/dax.test.js deleted file mode 100644 index 2361f5e..0000000 --- a/test/dax.test.js +++ /dev/null @@ -1,13 +0,0 @@ -var argv = require('optimist').argv, - PegasusConverter = require('../converters/pegasus_dax.js'); - -var daxf = new PegasusConverter(); - -if (!argv._[0]) { - console.log("Usage: node dax.test.js "); - process.exit(); -} - -daxf.convertFromFile(argv._[0], function(err, rep) { - console.log(JSON.stringify(rep, null, 2)); -}); From f802a587451c1031034665ef26fd27c024a54157 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Mar 2014 12:00:40 +0100 Subject: [PATCH 61/73] Introduce calling DAP services in LMonFunctions.js. --- workflows/ismop/LMonFunctions.js | 86 ++++++++++++++++++++------------ 1 file changed, 53 insertions(+), 33 deletions(-) diff --git a/workflows/ismop/LMonFunctions.js b/workflows/ismop/LMonFunctions.js index 255af08..76bfb01 100644 --- a/workflows/ismop/LMonFunctions.js +++ b/workflows/ismop/LMonFunctions.js @@ -13,10 +13,7 @@ var ThreatLevel = { SEVERE: "severe" }; -// emulates values written in DAP (to be removed) -var emLevelPersistent = EmergLevel.NONE, thrLevelPersistent = ThreatLevel.NONE; - -// Step 1: periodically reads the Levee state from DAP. +// Step 1: periodically reads the Levee state from DAP. // The main parameter of interest is `emergencyLevel': // - If 'none', no action taken // - If 'hightened', triggers computation of the current threat level @@ -24,32 +21,42 @@ var emLevelPersistent = EmergLevel.NONE, thrLevelPersistent = ThreatLevel.NONE; function getLeveeState(ins, outs, config, cb) { // var leveeURI = config.leveeUri; // URI could be passed through config // TODO: invoke DAP's REST API to retrieve levee state - - var emergencyLevel, threatLevel; - - var rand = Math.random(); // TODO: to be set by the result of REST invocation - if (rand > 0.95) { - emergencyLevel = EmergLevel.SEVERE; - } else if (rand > 0.7) { - emergencyLevel = EmergLevel.HIGHTENED; - } else { - emergencyLevel = EmergLevel.NONE; - } - console.log("emergencyLevel="+emergencyLevel); - if (emergencyLevel == EmergLevel.HIGHTENED && thrLevelPersistent == ThreatLevel.NONE) { - console.log("Setting hightened emergency level"); - outs[0].condition = "true"; // emit "ELHightened" signal - outs[0].data = [ { } ]; - } + request( + { + "timeout": 1000, + "url": config.url + }, + function (error, response, body) { + if (!error && response.statusCode == 200) { + var result = JSON.parse(body); + var emergencyLevel = EmergLevel[result.emergencyLevel.toUpperCase()]; + var threatLevel = EmergLevel[result.emergencyLevel.toUpperCase()]; - if (emergencyLevel == EmergLevel.SEVERE) { - console.log("Setting severe emergency level"); - outs[1].condition = "true"; // emit "ELSevere" signal - outs[1].data = [ { } ]; - } + //TODO; check for emergencyLevel == undefined, if so fail + console.log("emergencyLevel=" + emergencyLevel); - cb(null, outs); + if (emergencyLevel == EmergLevel.HIGHTENED && threatLevel == ThreatLevel.NONE) { + console.log("Setting hightened emergency level"); + outs[0].condition = "true"; // emit "ELHightened" signal + outs[0].data = [ + { } + ]; + } + + if (emergencyLevel == EmergLevel.SEVERE) { + console.log("Setting severe emergency level"); + outs[1].condition = "true"; // emit "ELSevere" signal + outs[1].data = [ + { } + ]; + } + + cb(null, outs); + } else { + cb("Error reading response!", outs); + } + }); } // Step 2a: run estimation of the threat level (here will be the Map/Reduce jobs!) @@ -64,13 +71,26 @@ function computeThreatLevel(ins, outs, config, cb) { } else { threatLevel = ThreatLevel.NONE; } - console.log("Computing threat level..."); - setTimeout(function() { - console.log("threatLevel="+threatLevel); - thrLevelPersistent = threatLevel; // TODO: replace it with POST to DAP - cb(null, outs); - }, 5000); + request.post( + { + "timeout": 1000, + "url": config.url, + "form": {"threatLevel": threatLevel} + }, + function(error, response, body) { + if(!error && response.statusCode == 201) { + parsedResponse = JSON.parse(body); + if (parsedResponse.result === "ok") { + cb(null, outs); + } else { + cb("Invalid response!", outs); + } + } else { + cb("Error reading response!", outs); + } + } + ); } // Step 2b: perform actions in the severe emergency level From 7c0cf07b967d57e71e04d52dec6c861853736af5 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Mar 2014 12:01:10 +0100 Subject: [PATCH 62/73] Added tests for LMonFunctions. --- tests/request.js | 81 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tests/request.js diff --git a/tests/request.js b/tests/request.js new file mode 100644 index 0000000..f37dd9f --- /dev/null +++ b/tests/request.js @@ -0,0 +1,81 @@ +var request = require('request'); +var http = require('http'); + +var functions = require('../workflows/ismop/LMonFunctions.js'); + +exports.setUp = function (callback) { + this.server = createServer(); + this.server.listen(8080); + callback(); +}; + +exports.tearDown = function (callback) { + this.server.close(); + callback(); +}; + +exports.call_get_levee_levels = function (test) { + var ins = [], + outs = [], + config = { + "url": "http://localhost:8080/levee_state/1" + }; + + functions.getLeveeState(ins, outs, config, function (err, outs) { + if (!err) { + //TODO: add more assertions? + test.done(); + } else { + test.fail("getLeveeState response is invalid!"); + } + }); +}; + +exports.call_store_threat_level = function (test) { + var ins = [], + outs = [], + config = { + "url": "http://localhost:8080/levee_threatLevel/1" + }; + + functions.computeThreatLevel(ins, outs, config, function (err, outs) { + if (!err) { + test.done(); + } else { + test.fail("computeThreatLevel response is invalid!"); + } + }); +}; + + +function createServer() { + //mock of services exposed by AIR + return http.createServer(function (req, resp) { + if (req.method === "GET" && req.url === "/levee_state/1") { + //response for call_get_levee_levels + resp.writeHead(200, {"Content-Type": "text/plain"}); + resp.write(JSON.stringify( + { + "emergencyLevel": "hightened", + "threatLevel": "none" + } + )); + resp.end(); + } else if (req.method === "POST" && req.url === "/levee_threatLevel/1") { + //response for call_store_threat_level + var body = ""; + req.on("data", function (data) { + body += data; + }); + req.on("end", function () { + resp.writeHead(201, {"Content-Type": "text/plain"}); + resp.write(JSON.stringify({"result": "ok"})); //report ok + resp.end(); + }); + } else { + resp.writeHead(404, {"Content-Type": "text/plain"}); + resp.write("ERROR! Unknown operation or URL."); + resp.end(); + } + }); +} \ No newline at end of file From 88aa076c1c80209d800b5ba8dd8460e350a8ae1c Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Mar 2014 12:04:50 +0100 Subject: [PATCH 63/73] Improve test naming. --- tests/request.js | 7 +++++-- workflows/ismop/LMonFunctions.js | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/request.js b/tests/request.js index f37dd9f..f211416 100644 --- a/tests/request.js +++ b/tests/request.js @@ -14,7 +14,7 @@ exports.tearDown = function (callback) { callback(); }; -exports.call_get_levee_levels = function (test) { +exports.call_getLeveeState = function (test) { var ins = [], outs = [], config = { @@ -31,7 +31,7 @@ exports.call_get_levee_levels = function (test) { }); }; -exports.call_store_threat_level = function (test) { +exports.call_storeThreatLevels = function (test) { var ins = [], outs = [], config = { @@ -47,6 +47,9 @@ exports.call_store_threat_level = function (test) { }); }; +functions.call_severeEmergencyActions = function (test) { + +}; function createServer() { //mock of services exposed by AIR diff --git a/workflows/ismop/LMonFunctions.js b/workflows/ismop/LMonFunctions.js index 76bfb01..3f0526d 100644 --- a/workflows/ismop/LMonFunctions.js +++ b/workflows/ismop/LMonFunctions.js @@ -81,7 +81,7 @@ function computeThreatLevel(ins, outs, config, cb) { function(error, response, body) { if(!error && response.statusCode == 201) { parsedResponse = JSON.parse(body); - if (parsedResponse.result === "ok") { + if (parsedResponse.result == "ok") { cb(null, outs); } else { cb("Invalid response!", outs); From c601b3d95bd96895e1c2c55c4b13f2d456b93e8d Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Mar 2014 12:12:07 +0100 Subject: [PATCH 64/73] Added test for severeEmergencyActions. --- tests/request.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/request.js b/tests/request.js index f211416..fd19630 100644 --- a/tests/request.js +++ b/tests/request.js @@ -24,6 +24,7 @@ exports.call_getLeveeState = function (test) { functions.getLeveeState(ins, outs, config, function (err, outs) { if (!err) { //TODO: add more assertions? + console.log(JSON.stringify(outs)); test.done(); } else { test.fail("getLeveeState response is invalid!"); @@ -47,8 +48,15 @@ exports.call_storeThreatLevels = function (test) { }); }; -functions.call_severeEmergencyActions = function (test) { +exports.call_severeEmergencyActions = function (test) { + var ins = [], + outs = [], + config = {}; + functions.severeEmergencyActions(ins, outs, config, function (err, outs) { + test.ok(!err); + test.done(); + }); }; function createServer() { From c38143bf6f8dabbdeb94c2cf56bfc9c9e4179ed3 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Tue, 25 Mar 2014 12:47:23 +0100 Subject: [PATCH 65/73] Fix DAP name. --- tests/request.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/request.js b/tests/request.js index fd19630..e43bd12 100644 --- a/tests/request.js +++ b/tests/request.js @@ -60,7 +60,7 @@ exports.call_severeEmergencyActions = function (test) { }; function createServer() { - //mock of services exposed by AIR + //mock of services exposed by DAP return http.createServer(function (req, resp) { if (req.method === "GET" && req.url === "/levee_state/1") { //response for call_get_levee_levels From bc85b1623a654aacdca3097fabc95822e6d7328c Mon Sep 17 00:00:00 2001 From: balis Date: Wed, 26 Mar 2014 10:37:40 +0100 Subject: [PATCH 66/73] Added license description --- engine2/automata.js | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/engine2/automata.js b/engine2/automata.js index 75ff71c..1a75fcd 100644 --- a/engine2/automata.js +++ b/engine2/automata.js @@ -1,7 +1,26 @@ /** * @author Ibon Tolosana, @hyperandroid * - * See LICENSE file. + * The MIT License + * Copyright (c) 2012 Ibon Tolosana [@hyperandroid] + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. * */ From fee11a8cb03e46bd21868f0ce98e93a0e88474f3 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 26 Mar 2014 12:26:47 +0100 Subject: [PATCH 67/73] Improve error signaling in LMonFunctions.js. --- workflows/ismop/LMonFunctions.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflows/ismop/LMonFunctions.js b/workflows/ismop/LMonFunctions.js index 3f0526d..83b78e7 100644 --- a/workflows/ismop/LMonFunctions.js +++ b/workflows/ismop/LMonFunctions.js @@ -54,7 +54,7 @@ function getLeveeState(ins, outs, config, cb) { cb(null, outs); } else { - cb("Error reading response!", outs); + cb(new Error("Error reading response from getLeveeState!"), outs); } }); } @@ -84,10 +84,10 @@ function computeThreatLevel(ins, outs, config, cb) { if (parsedResponse.result == "ok") { cb(null, outs); } else { - cb("Invalid response!", outs); + cb(new Error("Error reading response from storeThreatLevel!"), outs); } } else { - cb("Error reading response!", outs); + cb(new Error("Error reading response from storeThreatLevel!"), outs); } } ); From 3c3287d5de97abfd3dee4deb204b82eed1279d39 Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 26 Mar 2014 12:31:08 +0100 Subject: [PATCH 68/73] Change Hightened to Heightened. --- tests/request.js | 2 +- workflows/ismop/LMonFunctions.js | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/request.js b/tests/request.js index e43bd12..5c4f098 100644 --- a/tests/request.js +++ b/tests/request.js @@ -67,7 +67,7 @@ function createServer() { resp.writeHead(200, {"Content-Type": "text/plain"}); resp.write(JSON.stringify( { - "emergencyLevel": "hightened", + "emergencyLevel": "heightened", "threatLevel": "none" } )); diff --git a/workflows/ismop/LMonFunctions.js b/workflows/ismop/LMonFunctions.js index 83b78e7..a574362 100644 --- a/workflows/ismop/LMonFunctions.js +++ b/workflows/ismop/LMonFunctions.js @@ -3,20 +3,20 @@ var request = require('request'); // http client var EmergLevel = { NONE: "none", - HIGHTENED: "hightened", + HEIGHTENED: "heightened", SEVERE: "severe" }; var ThreatLevel = { NONE: "none", - HIGHTENED: "hightened", + HEIGHTENED: "heightened", SEVERE: "severe" }; // Step 1: periodically reads the Levee state from DAP. // The main parameter of interest is `emergencyLevel': // - If 'none', no action taken -// - If 'hightened', triggers computation of the current threat level +// - If 'heightened', triggers computation of the current threat level // - If 'severe', triggers appropriate actions function getLeveeState(ins, outs, config, cb) { // var leveeURI = config.leveeUri; // URI could be passed through config @@ -36,9 +36,9 @@ function getLeveeState(ins, outs, config, cb) { //TODO; check for emergencyLevel == undefined, if so fail console.log("emergencyLevel=" + emergencyLevel); - if (emergencyLevel == EmergLevel.HIGHTENED && threatLevel == ThreatLevel.NONE) { - console.log("Setting hightened emergency level"); - outs[0].condition = "true"; // emit "ELHightened" signal + if (emergencyLevel == EmergLevel.HEIGHTENED && threatLevel == ThreatLevel.NONE) { + console.log("Setting heightened emergency level"); + outs[0].condition = "true"; // emit "ELHeightened" signal outs[0].data = [ { } ]; @@ -67,7 +67,7 @@ function computeThreatLevel(ins, outs, config, cb) { if (rand > 0.95) { threatLevel = ThreatLevel.SEVERE; } else if (rand > 0.7) { - threatLevel = ThreatLevel.HIGHTENED; + threatLevel = ThreatLevel.HEIGHTENED; } else { threatLevel = ThreatLevel.NONE; } From 4c194b0a1be16310e354b9b37b6ff7bb5dd9a13e Mon Sep 17 00:00:00 2001 From: Maciej Pawlik Date: Wed, 26 Mar 2014 12:34:49 +0100 Subject: [PATCH 69/73] Rename testfile request.js to LMonP1.js. Move LMonFunctions.js to proper place. --- {workflows => functions}/ismop/LMonFunctions.js | 0 tests/{request.js => LMonP1.js} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename {workflows => functions}/ismop/LMonFunctions.js (100%) rename tests/{request.js => LMonP1.js} (97%) diff --git a/workflows/ismop/LMonFunctions.js b/functions/ismop/LMonFunctions.js similarity index 100% rename from workflows/ismop/LMonFunctions.js rename to functions/ismop/LMonFunctions.js diff --git a/tests/request.js b/tests/LMonP1.js similarity index 97% rename from tests/request.js rename to tests/LMonP1.js index 5c4f098..8a71fad 100644 --- a/tests/request.js +++ b/tests/LMonP1.js @@ -1,7 +1,7 @@ var request = require('request'); var http = require('http'); -var functions = require('../workflows/ismop/LMonFunctions.js'); +var functions = require('../functions/ismop/LMonFunctions.js'); exports.setUp = function (callback) { this.server = createServer(); From 5079230819b63011fb60c57c7c8d52c0db7adbcf Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Sun, 30 Mar 2014 11:29:59 +0200 Subject: [PATCH 70/73] Added provenance logging (disabled by default). --- engine2/ProcChoiceFSM.js | 2 +- engine2/ProcDataflowFSM.js | 2 +- engine2/ProcSplitterFSM.js | 2 +- engine2/index.js | 12 ++++++ engine2/process.js | 68 +++++++++++++++++++++++++++------- wflib/index.js | 75 ++++++++++++++++++++------------------ 6 files changed, 110 insertions(+), 51 deletions(-) diff --git a/engine2/ProcChoiceFSM.js b/engine2/ProcChoiceFSM.js index b6fae94..b7c601f 100644 --- a/engine2/ProcChoiceFSM.js +++ b/engine2/ProcChoiceFSM.js @@ -37,10 +37,10 @@ var ChoiceLogic = function() { this.firingSigs.push([sigId, 1]); } } + // "next" signal (if present) is also required for firing (even the first one) if ("next" in this.ctrIns) { this.firingSigs.push([this.ctrIns.next,1]); } - //onsole.log(JSON.stringify(this.firingSigs)); } diff --git a/engine2/ProcDataflowFSM.js b/engine2/ProcDataflowFSM.js index 6932d13..b7797c7 100644 --- a/engine2/ProcDataflowFSM.js +++ b/engine2/ProcDataflowFSM.js @@ -38,12 +38,12 @@ var DataflowLogic = function() { this.firingSigs.push([sigId, 1]); } } + // "next" signal (if present) is also required for firing (even the first one) if ("next" in this.ctrIns) { this.firingSigs.push([this.ctrIns.next,1]); } } - this.ready_enter = function(session, state, transition, msg) { var proc = session.logic; proc.ready = true; diff --git a/engine2/ProcSplitterFSM.js b/engine2/ProcSplitterFSM.js index 1fe6257..79703f2 100644 --- a/engine2/ProcSplitterFSM.js +++ b/engine2/ProcSplitterFSM.js @@ -175,7 +175,7 @@ function TaskLogic() { this.sources = engine.sources; this.sinks = engine.sinks; this.nDataIns = engine.ins[taskId].length; - this.firstInvocation = true; + this.firstFiring = true; this.name = fullInfo.name; this.fullInfo = fullInfo; diff --git a/engine2/index.js b/engine2/index.js index 023fa0e..991d6da 100644 --- a/engine2/index.js +++ b/engine2/index.js @@ -51,7 +51,13 @@ var Engine = function(config, wflib, wfId, cb) { this.nTasksLeft = 0; // how many tasks left (not finished)? this.nWfOutsLeft = 0; // how many workflow outputs are still to be produced? this.syncCb = null; // callback invoked when wf instance finished execution (passed to runInstanceSync) + + this.logProvenance = false; + this.eventServer.on('prov', function(data) { + console.log(arguments[1]); + }); + this.emulate = config.emulate == "true" ? true: false; this.startTime = (new Date()).getTime(); // the start time of this engine (workflow) @@ -198,6 +204,12 @@ Engine.prototype.emitSignals = function(sigs, cb) { async.each(sigInstances, function(s, doneIterInner) { var _sigId = s._id; engine.wflib.sendSignal(engine.wfId, s, function(err, sinks) { + // at this point "s" contains unique 'sigIdx' set in 'sendSignal' => we can emit "write" + // provenance events (for signals which have "source", i.e. were written by a process) + if (s.source && engine.logProvenance) { + engine.eventServer.emit("prov", ["write", +engine.wfId, s.source, s.firingId, s._id, s.sigIdx]); + } + if (!err) { // notify sinks that the signals have arrived for (var j=0; j do "state-reset" in next firing + if (!proc.fullInfo.stateful) { + proc.provStash.push( + ["state-reset", proc.appId, proc.procId, proc.firingId+1, null, null]); + + // (b) some sigs are stateful => do "state-remove" for those which aren't + } else if (!isStateful(sig._id)) { + proc.provStash.push( + ["state-remove", proc.appId, proc.procId, proc.firingId+1, sig._id, sig.sigIdx]); + } + }); + }); + } + proc.wflib.invokeTaskFunction2( proc.appId, proc.procId, @@ -240,7 +282,7 @@ var ProcLogic = function() { proc.sigValues, funcOuts, emul, proc.engine.eventServer, - function(err, outs) { + function(err, outs, options) { err ? cb(err): cb(null, outs, asyncInvocation, funcIns, funcOuts); } ); @@ -251,9 +293,9 @@ var ProcLogic = function() { var outValues = outs; for (var i=0; i Date: Mon, 14 Apr 2014 19:07:09 +0200 Subject: [PATCH 71/73] Switched to new Engine implementation --- scripts/runwf.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/runwf.js b/scripts/runwf.js index caa1f9d..a80bbe9 100644 --- a/scripts/runwf.js +++ b/scripts/runwf.js @@ -8,7 +8,7 @@ var redis = require('redis'), rcl = redis.createClient(), wflib = require('../wflib').init(rcl), - Engine = require('../engine'), + Engine = require('../engine2'), async = require('async'), argv = require('optimist').argv, dbId = 0, From 620482ee0eb613f95b5016073fabf664cdf3c623 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 16 Apr 2014 13:49:04 +0200 Subject: [PATCH 72/73] Path to function module now computed based on '__dirname' --- wflib/index.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wflib/index.js b/wflib/index.js index 96cdff6..5907964 100644 --- a/wflib/index.js +++ b/wflib/index.js @@ -1416,7 +1416,8 @@ function public_invokeTaskFunction2(wfId, taskId, insIds_, insValues, outsIds_, rcl.hgetall("wf:functions:"+taskInfo.fun, function(err, fun) { if (err) return cb(err); - var fpath = pathTool.join(process.cwd(), fun.module); + //var fpath = pathTool.join(process.cwd(), fun.module); + var fpath = pathTool.join(__dirname, "..", fun.module); var f = require(fpath)[taskInfo.fun]; //onsole.log("INS:", ins); //onsole.log("OUTS:", outs); From f117c12cbea30a56bff5a4e98b61469cb515adf3 Mon Sep 17 00:00:00 2001 From: Bartosz Balis Date: Wed, 16 Apr 2014 14:01:19 +0200 Subject: [PATCH 73/73] Version bump. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 80daa5e..5616044 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,10 @@ Browse the [wiki pages](https://github.com/balis/hyperflow/wiki) to learn more a ##Getting started -Latest release of HyperFlow is 1.0.0-beta-5 +Latest release of HyperFlow is 1.0.0-beta-6 Installation & running: -* Download the package: https://github.com/dice-cyfronet/hyperflow/archive/v1.0.0-beta-5.zip +* Download the package: https://github.com/dice-cyfronet/hyperflow/archive/v1.0.0-beta-6.zip * Install dependencies (in `hyperflow` directory): `npm install -d` * Install the latest node.js (http://nodejs.org) * Install the Redis server 2.6.x or higher (http://redis.io) (tested with version 2.6.x)