From 1d3278f47b2f25e77127dea0e55c272b4b017879 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 16 Apr 2024 15:40:29 -0600 Subject: [PATCH 01/55] Made METexpress regions more readable --- .../imports/startup/server/data_util.js | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/meteor_packages/mats-common/imports/startup/server/data_util.js b/meteor_packages/mats-common/imports/startup/server/data_util.js index fbf5daa94..aaf89f33d 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_util.js @@ -700,6 +700,59 @@ const excludeStatFromScorecard = function (stat) { return statsToExclude.indexOf(stat) !== -1; }; +// turn METexpress regions into readable text +const readableStandardRegions = function () { + return { + // EMC regions + FULL: "FULL: Full Domain", + APL: "APL: Appalachia", + ATC: "ATC: North American Arctic", + CAM: "CAM: Central America", + CAR: "CAR: Caribbean", + CONUS: "CONUS: Continental US", + EAST: "EAST: Eastern US", + ECA: "ECA: Eastern Canada", + GLF: "GLF: Gulf of Mexico", + GMC: "GMC: Gulf of Mexico Coast", + GRB: "GRB: Great Basin", + HWI: "HWI: Hawaii", + LMV: "LMV: Lower Mississippi Valley", + MDW: "MDW: Midwest US", + MEX: "MEX: Mexico", + N60: "N60: Northern Polar Latitudes", + NAK: "NAK: Northern Alaska", + NAO: "NAO: Northern North Atlantic Ocean", + NEC: "NEC: Northeastern US Coast", + NHX: "NHX: Northern Hemisphere (20N <= lat <= 80N)", + NMT: "NMT: Northern Rocky Mountains", + NPL: "NPL: Northern Great Plains", + NPO: "NPO: Northern North Pacific Ocean", + NSA: "NSA: Northern South America", + NWC: "NWC: Northwestern US Coast", + PNA: "PNA: Pacific / North America", + PRI: "PRI: Puerto Rico", + S60: "S60: Southern Polar Latitudes", + SAC: "SAC: Southern Alaska", + SAO: "SAO: Southern North Atlantic Ocean", + SEC: "SEC: Southeastern US Coast", + SHX: "SHX: Southern Hemisphere (20S >= lat >= 80S)", + SMT: "SMT: Southern Rocky Mountains", + SPL: "SPL: Southern Great Plains", + SPO: "SPO: Southern North Pacific Ocean", + SWC: "SWC: Southwestern US Coast", + SWD: "SWD: Southwestern US Desert", + TRO: "TRO: Global Tropics (20N >= lat >= 20S)", + WCA: "WCA: Western Canada", + WEST: "WEST: Western US", + // GSL additions + AAK: "AAK: Alaska", + ALL_HRRR: "HRRR Domain", + E_HRRR: "Eastern HRRR Domain", + W_HRRR: "Western HRRR Domain", + GtLk: "Great Lakes", + }; +}; + // calculates mean, stdev, and other statistics for curve data points in all apps and plot types const get_err = function (sVals, sSecs, sLevs, appParams) { /* refer to perl error_library.pl sub get_stats @@ -1743,6 +1796,7 @@ export default matsDataUtils = { calculateStatScalar, consolidateContour, excludeStatFromScorecard, + readableStandardRegions, get_err, ctcErrorPython, checkDiffContourSignificance, From a7f7d6a80b1ef936510841ece096f8683933a54e Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 23 Apr 2024 13:31:51 -0600 Subject: [PATCH 02/55] Fixed scorecard bug where app and level wouldn't set correctly --- .../mats-common/templates/plot/plot_list.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/meteor_packages/mats-common/templates/plot/plot_list.js b/meteor_packages/mats-common/templates/plot/plot_list.js index 7c6776f46..2cee6d704 100644 --- a/meteor_packages/mats-common/templates/plot/plot_list.js +++ b/meteor_packages/mats-common/templates/plot/plot_list.js @@ -30,6 +30,18 @@ import { is what sets up the graph page. */ +const _setApplication = async (app) => { + // application usually refers to either database or variable in MATS + if (document.getElementById("database-item")) { + matsParamUtils.setValueTextForParamName("database", app); + } else if ( + document.getElementById("variable-item") && + document.getElementById("threshold-item") + ) { + matsParamUtils.setValueTextForParamName("variable", app); + } +}; + const _changeParameter = async (parameter, newValue) => { matsParamUtils.setValueTextForParamName(parameter, newValue); }; @@ -44,6 +56,10 @@ const _setCommonParams = async (commonParamKeys, commonParams) => { } else if (thisKey === "region" && document.getElementById("vgtyp-item")) { // landuse regions go in the vgtyp selector matsParamUtils.setValueTextForParamName("vgtyp", thisValue); + } else if (thisKey === "level" && document.getElementById("top-item")) { + // some apps don't actually have a level selector + matsParamUtils.setValueTextForParamName("top", thisValue); + matsParamUtils.setValueTextForParamName("bottom", thisValue); } } } @@ -58,10 +74,12 @@ const _plotGraph = async () => { }; const addCurvesAndPlot = async (parsedSettings, commonParamKeys, commonParams) => { + await _setApplication(parsedSettings.appName); await _changeParameter("data-source", parsedSettings.curve0DataSource); await _setCommonParams(commonParamKeys, commonParams); await _addCurve(); await _changeParameter("data-source", parsedSettings.curve1DataSource); + await _setCommonParams(commonParamKeys, commonParams); await _addCurve(); await _changeParameter("dates", parsedSettings.dateRange); await _plotGraph(); From e90d5d735c71c5b1cc6f1e83fb530c1870c0712c Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 23 Apr 2024 15:23:20 -0600 Subject: [PATCH 03/55] Fixed broken scorecard color selectors --- meteor_packages/mats-common/lib/param_util.js | 6 ++++ .../mats-common/templates/ScorecardHome.js | 28 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/meteor_packages/mats-common/lib/param_util.js b/meteor_packages/mats-common/lib/param_util.js index 4a85cc8dd..4f5cd5aca 100644 --- a/meteor_packages/mats-common/lib/param_util.js +++ b/meteor_packages/mats-common/lib/param_util.js @@ -255,6 +255,8 @@ const getElementValues = function () { return $(el).val(); }) .get(); + } else if (param.type === matsTypes.InputTypes.color) { + val = document.querySelector(`[name='${param.name}-icon']`).style.color; } else { const idSelect = `#${getInputIdForParamName(param.name)}`; val = $(idSelect).val(); @@ -275,6 +277,8 @@ const getElementValues = function () { return $(el).val(); }) .get(); + } else if (param.type === matsTypes.InputTypes.color) { + val = document.querySelector(`[name='${param.name}-icon']`).style.color; } else { var idSelect = `#${getInputIdForParamName(param.name)}`; val = $(idSelect).val(); @@ -295,6 +299,8 @@ const getElementValues = function () { return $(el).val(); }) .get(); + } else if (param.type === matsTypes.InputTypes.color) { + val = document.querySelector(`[name='${param.name}-icon']`).style.color; } else { var idSelect = `#${getInputIdForParamName(name)}`; val = $(idSelect).val(); diff --git a/meteor_packages/mats-common/templates/ScorecardHome.js b/meteor_packages/mats-common/templates/ScorecardHome.js index 70dec0891..7724f3b9f 100644 --- a/meteor_packages/mats-common/templates/ScorecardHome.js +++ b/meteor_packages/mats-common/templates/ScorecardHome.js @@ -109,14 +109,42 @@ Template.ScorecardHome.events({ }, "change #scorecard-color-theme-radioGroup-RedGreen"(event) { document.querySelector('[name="major-truth-color-icon"]').style.color = "#ff0000"; + document.getElementById("major-truth-color-color").value = "#ff0000"; document.querySelector('[name="minor-truth-color-icon"]').style.color = "#ff0000"; + document.getElementById("minor-truth-color-color").value = "#ff0000"; document.querySelector('[name="major-source-color-icon"]').style.color = "#00ff00"; + document.getElementById("major-source-color-color").value = "#00ff00"; document.querySelector('[name="minor-source-color-icon"]').style.color = "#00ff00"; + document.getElementById("minor-source-color-color").value = "#00ff00"; }, "change #scorecard-color-theme-radioGroup-RedBlue"(event) { document.querySelector('[name="major-truth-color-icon"]').style.color = "#ff0000"; + document.getElementById("major-truth-color-color").value = "#ff0000"; document.querySelector('[name="minor-truth-color-icon"]').style.color = "#ff0000"; + document.getElementById("minor-truth-color-color").value = "#ff0000"; document.querySelector('[name="major-source-color-icon"]').style.color = "#0000ff"; + document.getElementById("major-source-color-color").value = "#0000ff"; document.querySelector('[name="minor-source-color-icon"]').style.color = "#0000ff"; + document.getElementById("minor-source-color-color").value = "#0000ff"; + }, + "change [name='major-truth-color-icon']"(event) { + document.getElementById("major-truth-color-color").value = document.querySelector( + '[name="major-truth-color-icon"]' + ).style.color; + }, + "change [name='minor-truth-color-icon']"(event) { + document.getElementById("minor-truth-color-color").value = document.querySelector( + '[name="minor-truth-color-icon"]' + ).style.color; + }, + "change [name='major-source-color-icon']"(event) { + document.getElementById("major-source-color-color").value = document.querySelector( + '[name="major-source-color-icon"]' + ).style.color; + }, + "change [name='minor-source-color-icon']"(event) { + document.getElementById("minor-source-color-color").value = document.querySelector( + '[name="minor-source-color-icon"]' + ).style.color; }, }); From 5cd5c26991423ecfab96f38ae5064d5608ebe77c Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 23 Apr 2024 15:27:38 -0600 Subject: [PATCH 04/55] Updated release notes --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 882487764..8a4efd70f 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -7,10 +7,15 @@

Production build date: Current revision

-

PUT APP VERSIONS HERE

+

All apps v5.2.4

Changes:

-

* PUT CHANGES HERE

+

* METexpress common regions are now human-readable.

+

* Fixed bug where upper air scorecards would never come up.

+

* Fixed bug where scorecard colors ignored user selection.

+

* Fixed bug where scorecards would return no data for AMDAR retros.

+

* Fixed bug where scorecards would sometimes use the wrong database and level when switching to a specific timeseries.

+

* Fixed bug where scorecards would sometimes display too many regions or forecast lead times.


Date: Tue, 23 Apr 2024 16:17:50 -0600 Subject: [PATCH 05/55] Fixed typo --- .../mats-common/public/MATSReleaseNotes.html | 10 ++++++++-- .../templates/scorecard/scorecardStatusPage.js | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 8a4efd70f..ce3046a51 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -14,8 +14,14 @@

All apps v5.2.4

* Fixed bug where upper air scorecards would never come up.

* Fixed bug where scorecard colors ignored user selection.

* Fixed bug where scorecards would return no data for AMDAR retros.

-

* Fixed bug where scorecards would sometimes use the wrong database and level when switching to a specific timeseries.

-

* Fixed bug where scorecards would sometimes display too many regions or forecast lead times.

+

+ * Fixed bug where scorecards would sometimes use the wrong database and level when + switching to a specific timeseries. +

+

+ * Fixed bug where scorecards would sometimes display too many regions or forecast + lead times. +


Date: Wed, 24 Apr 2024 13:18:29 -0600 Subject: [PATCH 06/55] Update documentation for release --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index ce3046a51..da3010e89 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -5,7 +5,7 @@

-

Production build date: Current revision

+

Production build date: 2024.04.24

All apps v5.2.4

From 2405957a93dae5f137ec8b3038c5d6a5aab4b675 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Mon, 6 May 2024 14:27:26 -0600 Subject: [PATCH 07/55] Folded MET Air Quality into MET Surface, and allowed plotting of ceiling stats --- .../imports/startup/both/mats-curve-params.js | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js index e7e6e8130..bc848d6ba 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js @@ -99,31 +99,6 @@ export const curveParamsByApp = { "bin-parameter", "curve-dates", ], - "met-airquality": [ - "label", - "group", - "database", - "data-source", - "plot-type", - "region", - "statistic", - "y-statistic", - "variable", - "y-variable", - "threshold", - "interp-method", - "scale", - "forecast-length", - "dieoff-type", - "valid-time", - "utc-cycle-start", - "average", - "level", - "description", - "aggregation-method", - "bin-parameter", - "curve-dates", - ], "met-anomalycor": [ "label", "group", @@ -246,6 +221,7 @@ export const curveParamsByApp = { "y-statistic", "variable", "y-variable", + "threshold", "interp-method", "scale", "truth", From 9d46839b7e5e162e2a779f8cfb5d0202ede1ac17 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Mon, 6 May 2024 14:28:30 -0600 Subject: [PATCH 08/55] Revert "Folded MET Air Quality into MET Surface, and allowed plotting of ceiling stats" This reverts commit 2405957a93dae5f137ec8b3038c5d6a5aab4b675. --- .../imports/startup/both/mats-curve-params.js | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js index bc848d6ba..e7e6e8130 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js @@ -99,6 +99,31 @@ export const curveParamsByApp = { "bin-parameter", "curve-dates", ], + "met-airquality": [ + "label", + "group", + "database", + "data-source", + "plot-type", + "region", + "statistic", + "y-statistic", + "variable", + "y-variable", + "threshold", + "interp-method", + "scale", + "forecast-length", + "dieoff-type", + "valid-time", + "utc-cycle-start", + "average", + "level", + "description", + "aggregation-method", + "bin-parameter", + "curve-dates", + ], "met-anomalycor": [ "label", "group", @@ -221,7 +246,6 @@ export const curveParamsByApp = { "y-statistic", "variable", "y-variable", - "threshold", "interp-method", "scale", "truth", From 1275063f90bb972d467126d20c8d178e7c72b105 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Mon, 6 May 2024 14:31:00 -0600 Subject: [PATCH 09/55] Folded MET Air Quality into MET Surface, and allowed plotting of ceiling stats --- .../imports/startup/both/mats-curve-params.js | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js index e7e6e8130..bc848d6ba 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js @@ -99,31 +99,6 @@ export const curveParamsByApp = { "bin-parameter", "curve-dates", ], - "met-airquality": [ - "label", - "group", - "database", - "data-source", - "plot-type", - "region", - "statistic", - "y-statistic", - "variable", - "y-variable", - "threshold", - "interp-method", - "scale", - "forecast-length", - "dieoff-type", - "valid-time", - "utc-cycle-start", - "average", - "level", - "description", - "aggregation-method", - "bin-parameter", - "curve-dates", - ], "met-anomalycor": [ "label", "group", @@ -246,6 +221,7 @@ export const curveParamsByApp = { "y-statistic", "variable", "y-variable", + "threshold", "interp-method", "scale", "truth", From 8ba6f82b3f3bbe20fa757644caf5b181e8e384e2 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Mon, 6 May 2024 15:55:42 -0600 Subject: [PATCH 10/55] Updated release notes --- .../mats-common/public/MATSReleaseNotes.html | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index da3010e89..57136ea93 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -3,6 +3,21 @@ style="display: block; height: 2px; margin: 1em 0; border-top: 2px solid #000000" />
+
+

+

Production build date: Current revision

+

+

All apps v5.3.0

+

+

Changes:

+

* Combined MET Surface and Air Quality apps.

+

* Allowed MET Surface to plot contingency table statistics.

+
+
+
+

Production build date: 2024.04.24

From 2ff6c158977ac561f146183f316db676a6ad515e Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Wed, 8 May 2024 11:03:16 -0600 Subject: [PATCH 11/55] Updated release notes --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 1 + 1 file changed, 1 insertion(+) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 57136ea93..ed24ff56a 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -12,6 +12,7 @@

All apps v5.3.0

Changes:

* Combined MET Surface and Air Quality apps.

* Allowed MET Surface to plot contingency table statistics.

+

* METexpress can now handle databases, models, and regions that have dots in the name.


Date: Wed, 8 May 2024 11:04:51 -0600 Subject: [PATCH 12/55] Linter nonsense --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index ed24ff56a..73661ef70 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -12,7 +12,10 @@

All apps v5.3.0

Changes:

* Combined MET Surface and Air Quality apps.

* Allowed MET Surface to plot contingency table statistics.

-

* METexpress can now handle databases, models, and regions that have dots in the name.

+

+ * METexpress can now handle databases, models, and regions that have dots in the + name. +


Date: Tue, 14 May 2024 14:45:55 -0600 Subject: [PATCH 13/55] Updated release notes --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 73661ef70..469c7cbcc 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -5,7 +5,7 @@

-

Production build date: Current revision

+

Production build date: 2024.05.14

All apps v5.3.0

From d3fff19dfbe409b1f92043fed45ea59c898156e1 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 17 May 2024 15:25:06 -0600 Subject: [PATCH 14/55] RESET RELEASE NOTES --- .../mats-common/public/MATSReleaseNotes.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 469c7cbcc..7c607a32a 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -3,6 +3,20 @@ style="display: block; height: 2px; margin: 1em 0; border-top: 2px solid #000000" />
+
+

+

Production build date: Current revision

+

+

PUT APP VERSIONS HERE

+

+

Changes:

+

* PUT CHANGES HERE

+
+
+
+

Production build date: 2024.05.14

From 689c1e920117367a75b322e1b1225196b7caa69b Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 24 May 2024 13:56:11 -0600 Subject: [PATCH 15/55] Fixed bug where some zeros were still being filtered out with NaNs in data routines --- .../imports/startup/server/data_diff_util.js | 62 +++---------------- .../imports/startup/server/data_match_util.js | 16 +---- .../startup/server/data_process_util.js | 40 +++++------- .../imports/startup/server/data_query_util.js | 6 +- 4 files changed, 27 insertions(+), 97 deletions(-) diff --git a/meteor_packages/mats-common/imports/startup/server/data_diff_util.js b/meteor_packages/mats-common/imports/startup/server/data_diff_util.js index 6623448e2..798c161eb 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_diff_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_diff_util.js @@ -503,24 +503,12 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes } // calculate the max and min for this curve - const filteredx = d.x.filter((x) => x); - const filteredy = d.y.filter((y) => y); + const filteredx = d.x.filter((x) => x || x === 0); + const filteredy = d.y.filter((y) => y || y === 0); d.xmin = Math.min(...filteredx); - if (d.x.indexOf(0) !== -1 && d.xmin > 0) { - d.xmin = 0; - } d.xmax = Math.max(...filteredx); - if (d.x.indexOf(0) !== -1 && d.xmax < 0) { - d.xmax = 0; - } d.ymin = Math.min(...filteredy); - if (d.y.indexOf(0) !== -1 && d.ymin > 0) { - d.ymin = 0; - } d.ymax = Math.max(...filteredy); - if (d.y.indexOf(0) !== -1 && d.ymax < 0) { - d.ymax = 0; - } return { dataset: d }; }; @@ -1405,54 +1393,18 @@ const getDataForDiffContour = function ( } // calculate statistics - const filteredx = diffDataset.x.filter((x) => x); - const filteredy = diffDataset.y.filter((y) => y); - const filteredz = diffDataset.zTextOutput.filter((z) => z); + const filteredx = diffDataset.x.filter((x) => x || x === 0); + const filteredy = diffDataset.y.filter((y) => y || y === 0); + const filteredz = diffDataset.zTextOutput.filter((z) => z || z === 0); diffDataset.xmin = Math.min(...filteredx); - if ( - !Number.isFinite(diffDataset.xmin) || - (diffDataset.x.indexOf(0) !== -1 && diffDataset.xmin > 0) - ) { - diffDataset.xmin = 0; - } diffDataset.xmax = Math.max(...filteredx); - if ( - !Number.isFinite(diffDataset.xmax) || - (diffDataset.x.indexOf(0) !== -1 && diffDataset.xmax < 0) - ) { - diffDataset.xmax = 0; - } diffDataset.ymin = Math.min(...filteredy); - if ( - !Number.isFinite(diffDataset.ymin) || - (diffDataset.y.indexOf(0) !== -1 && diffDataset.ymin > 0) - ) { - diffDataset.ymin = 0; - } diffDataset.ymax = Math.max(...filteredy); - if ( - !Number.isFinite(diffDataset.ymax) || - (diffDataset.y.indexOf(0) !== -1 && diffDataset.ymax < 0) - ) { - diffDataset.ymax = 0; - } diffDataset.zmin = Math.min(...filteredz); - if ( - !Number.isFinite(diffDataset.zmin) || - (diffDataset.z.indexOf(0) !== -1 && diffDataset.zmin > 0) - ) { - diffDataset.zmin = 0; - } diffDataset.zmax = Math.max(...filteredz); - if ( - !Number.isFinite(diffDataset.zmax) || - (diffDataset.z.indexOf(0) !== -1 && diffDataset.zmax < 0) - ) { - diffDataset.zmax = 0; - } - const filteredMinDate = diffDataset.minDateTextOutput.filter((t) => t); - const filteredMaxDate = diffDataset.maxDateTextOutput.filter((t) => t); + const filteredMinDate = diffDataset.minDateTextOutput.filter((t) => t || t === 0); + const filteredMaxDate = diffDataset.maxDateTextOutput.filter((t) => t || t === 0); diffDataset.glob_stats.mean = diffDataset.sum / nPoints; diffDataset.glob_stats.minDate = Math.min(...filteredMinDate); diffDataset.glob_stats.maxDate = Math.max(...filteredMaxDate); diff --git a/meteor_packages/mats-common/imports/startup/server/data_match_util.js b/meteor_packages/mats-common/imports/startup/server/data_match_util.js index 8b94e051c..0b86b3773 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_match_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_match_util.js @@ -822,24 +822,12 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat } // save matched data and recalculate the max and min for this curve - const filteredx = data.x.filter((x) => x); - const filteredy = data.y.filter((y) => y); + const filteredx = data.x.filter((x) => x || x === 0); + const filteredy = data.y.filter((y) => y || y === 0); data.xmin = Math.min(...filteredx); - if (data.x.indexOf(0) !== -1 && data.xmin > 0) { - data.xmin = 0; - } data.xmax = Math.max(...filteredx); - if (data.x.indexOf(0) !== -1 && data.xmax < 0) { - data.xmax = 0; - } data.ymin = Math.min(...filteredy); - if (data.y.indexOf(0) !== -1 && data.ymin > 0) { - data.ymin = 0; - } data.ymax = Math.max(...filteredy); - if (data.y.indexOf(0) !== -1 && data.ymax < 0) { - data.ymax = 0; - } dataset[curveIndex] = data; } diff --git a/meteor_packages/mats-common/imports/startup/server/data_process_util.js b/meteor_packages/mats-common/imports/startup/server/data_process_util.js index da14ca56c..ece9342d6 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_process_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_process_util.js @@ -322,7 +322,10 @@ const processDataXYCurve = function ( } // enable error bars if matching and they aren't null. - if (appParams.matching && data.error_y.array.filter((x) => x).length > 0) { + if ( + appParams.matching && + data.error_y.array.filter((x) => x || x === 0).length > 0 + ) { if (statType !== "ctc" || (diffFrom !== undefined && diffFrom !== null)) { data.error_y.visible = true; } @@ -330,7 +333,7 @@ const processDataXYCurve = function ( // get the overall stats for the text output. const stats = matsDataUtils.get_err(values, indVars, [], appParams); - const filteredValues = values.filter((x) => x); + const filteredValues = values.filter((x) => x || x === 0); let miny = Math.min(...filteredValues); let maxy = Math.max(...filteredValues); if (values.indexOf(0) !== -1 && miny > 0) { @@ -784,7 +787,10 @@ const processDataProfile = function ( } // enable error bars if matching and they aren't null. - if (appParams.matching && data.error_x.array.filter((x) => x).length > 0) { + if ( + appParams.matching && + data.error_x.array.filter((x) => x || x === 0).length > 0 + ) { if (statType !== "ctc" || (diffFrom !== undefined && diffFrom !== null)) { data.error_x.visible = true; } @@ -797,17 +803,9 @@ const processDataProfile = function ( [], appParams ); // have to reverse because of data inversion - const filteredValues = values.filter((x) => x); - let minx = Math.min(...filteredValues); - let maxx = Math.max(...filteredValues); - if (values.indexOf(0) !== -1 && minx > 0) { - minx = 0; - } - if (values.indexOf(0) !== -1 && maxx < 0) { - maxx = 0; - } - stats.minx = minx; - stats.maxx = maxx; + const filteredValues = values.filter((x) => x || x === 0); + stats.minx = Math.min(...filteredValues); + stats.maxx = Math.max(...filteredValues); dataset[curveIndex].glob_stats = stats; // recalculate axis options after QC and matching @@ -1704,17 +1702,9 @@ const processDataEnsembleHistogram = function ( // get the overall stats for the text output - this uses the means not the stats. const stats = matsDataUtils.get_err(values, indVars, [], appParams); - const filteredValues = values.filter((x) => x); - let miny = Math.min(...filteredValues); - let maxy = Math.max(...filteredValues); - if (values.indexOf(0) !== -1 && miny > 0) { - miny = 0; - } - if (values.indexOf(0) !== -1 && maxy < 0) { - maxy = 0; - } - stats.miny = miny; - stats.maxy = maxy; + const filteredValues = values.filter((x) => x || x === 0); + stats.miny = Math.min(...filteredValues); + stats.maxy = Math.max(...filteredValues); dataset[curveIndex].glob_stats = stats; // recalculate axis options after QC and matching diff --git a/meteor_packages/mats-common/imports/startup/server/data_query_util.js b/meteor_packages/mats-common/imports/startup/server/data_query_util.js index d0fadcc75..a1f2574ac 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_query_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_query_util.js @@ -2953,7 +2953,7 @@ const parseQueryDataMapScalar = function ( } // get range of values for colorscale, eliminating the highest and lowest as outliers - let filteredValues = d.queryVal.filter((x) => x); + let filteredValues = d.queryVal.filter((x) => x || x === 0); filteredValues = filteredValues.sort(function (a, b) { return Number(a) - Number(b); }); @@ -4036,8 +4036,8 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { d.zmax = zmax; d.sum = sum; - const filteredMinDate = d.minDateTextOutput.filter((t) => t); - const filteredMaxDate = d.maxDateTextOutput.filter((t) => t); + const filteredMinDate = d.minDateTextOutput.filter((t) => t || t === 0); + const filteredMaxDate = d.maxDateTextOutput.filter((t) => t || t === 0); d.glob_stats.mean = sum / nPoints; d.glob_stats.minDate = Math.min(...filteredMinDate); d.glob_stats.maxDate = Math.max(...filteredMaxDate); From 0ede2f545eac9fe587c98cbf2f7e211b243e3f43 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 24 May 2024 16:53:55 -0600 Subject: [PATCH 16/55] Fixed bug discovered by automated tests --- .../startup/server/data_process_util.js | 48 ++++++++----------- 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/meteor_packages/mats-common/imports/startup/server/data_process_util.js b/meteor_packages/mats-common/imports/startup/server/data_process_util.js index ece9342d6..265f3b8aa 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_process_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_process_util.js @@ -334,16 +334,8 @@ const processDataXYCurve = function ( // get the overall stats for the text output. const stats = matsDataUtils.get_err(values, indVars, [], appParams); const filteredValues = values.filter((x) => x || x === 0); - let miny = Math.min(...filteredValues); - let maxy = Math.max(...filteredValues); - if (values.indexOf(0) !== -1 && miny > 0) { - miny = 0; - } - if (values.indexOf(0) !== -1 && maxy < 0) { - maxy = 0; - } - stats.miny = miny; - stats.maxy = maxy; + stats.miny = Math.min(...filteredValues); + stats.maxy = Math.max(...filteredValues); dataset[curveIndex].glob_stats = stats; // recalculate axis options after QC and matching @@ -354,14 +346,14 @@ const processDataXYCurve = function ( const minx = Math.min(...filteredIndVars); const maxx = Math.max(...filteredIndVars); curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax < maxy || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] - ? maxy + curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax < + stats.maxy || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + ? stats.maxy : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax; curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin > miny || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] - ? miny + curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin > + stats.miny || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + ? stats.miny : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin; curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax = curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax < maxx || @@ -826,14 +818,14 @@ const processDataProfile = function ( ? miny : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin; curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax < maxx || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] - ? maxx + curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax < + stats.maxx || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + ? stats.maxx : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax; curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin > minx || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] - ? minx + curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin > + stats.minx || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + ? stats.minx : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin; // recalculate curve annotation after QC and matching @@ -1711,14 +1703,14 @@ const processDataEnsembleHistogram = function ( const minx = Math.min(...indVars); const maxx = Math.max(...indVars); curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax < maxy || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] - ? maxy + curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax < + stats.maxy || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + ? stats.maxy : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax; curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin > miny || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] - ? miny + curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin > + stats.miny || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + ? stats.miny : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin; curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax = curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax < maxx || From 59d4b4be7f8377e353b6000da1fa03c5c75d0e4c Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 4 Jun 2024 13:21:32 -0600 Subject: [PATCH 17/55] Added MET Cyclone RI metadata script --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 7c607a32a..8fc3b20f5 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -7,10 +7,15 @@

Production build date: Current revision

-

PUT APP VERSIONS HERE

+

All apps v5.3.1

Changes:

-

* PUT CHANGES HERE

+

* Added RI stat plotting to MET Cyclone app.

+

* Added ability to plot very historical RAOB data in upper air app.

+

+ * Fixed bug in data routines where some zeros were still being erroneously filtered + out as if they were NaNs. +


Date: Tue, 4 Jun 2024 15:18:36 -0600 Subject: [PATCH 18/55] MET Cyclone app parses new metadata --- .../imports/startup/server/data_util.js | 382 ++++++++++++++++++ 1 file changed, 382 insertions(+) diff --git a/meteor_packages/mats-common/imports/startup/server/data_util.js b/meteor_packages/mats-common/imports/startup/server/data_util.js index aaf89f33d..71bd18c44 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_util.js @@ -753,6 +753,386 @@ const readableStandardRegions = function () { }; }; +// turn METexpress TC categories into readable text +const readableTCCategories = function () { + return { + TD: { + name: "Tropical depression (wind <34kt)", + order: "0", + }, + TS: { + name: "Tropical storm (wind 34–63 kt)", + order: "1", + }, + HU: { + name: "Hurricane (wind ≥64 kt)", + order: "2", + }, + EX: { + name: "Extratropical cyclone (any intensity)", + order: "3", + }, + SD: { + name: "Subtropical depression (wind <34kt)", + order: "4", + }, + SS: { + name: "Subtropical storm (wind ≥34kt)", + order: "5", + }, + LO: { + name: "Low that isn't a tropical, subtropical, or extratropical cyclone", + order: "6", + }, + WV: { + name: "Tropical wave (any intensity)", + order: "7", + }, + DB: { + name: "Disturbance (any intensity)", + order: "8", + }, + }; +}; + +// turn METexpress adeck models into readable text +const readableAdeckModels = function () { + return { + OFCL: "OFCL: Official NHC/CPHC Forecast", + OFCI: "OFCI: Official NHC/CPHC Forecast", + OFC2: "OFC2: Official NHC/CPHC Forecast", + OFCP: "OFCP: Provisional NHC/CPHC Forecast", + OFPI: "OFPI: Provisional NHC/CPHC Forecast", + OFP2: "OFP2: Provisional NHC/CPHC Forecast", + OHPC: "OHPC: Official WPC Forecast", + OOPC: "OOPC: Official OPC Forecast", + OMPC: "OMPC: Official MPC Forecast", + JTWC: "JTWC: Official JTWC Forecast", + JTWI: "JTWI: Official JTWC Forecast", + AVNO: "AVNO: GFS", + AVNI: "AVNI: GFS", + AVN2: "AVN2: GFS", + AHNI: "AHNI: GFS No Bias Correction", + GFSO: "GFSO: GFS", + GFSI: "GFSI: GFS", + GFS2: "GFS2: GFS", + AVXO: "AVXO: GFS 10-day Tracker", + AVXI: "AVXI: GFS 10-day Tracker", + AVX2: "AVX2: GFS 10-day Tracker", + AC00: "AC00: GFS Ensemble Control", + AEMN: "AEMN: GFS Ensemble Mean", + AEMI: "AEMI: GFS Ensemble Mean", + AEM2: "AEM2: GFS Ensemble Mean", + AMMN: "AMMN: GFS New Ensemble Mean", + CMC: "CMC: Canadian Global Model", + CMCI: "CMCI: Canadian Global Model", + CMC2: "CMC2: Canadian Global Model", + CC00: "CC00: Canadian Ensemble Control", + CEMN: "CEMN: Canadian Ensemble Mean", + CEMI: "CEMI: Canadian Ensemble Mean", + CEM2: "CEM2: Canadian Ensemble Mean", + COTC: "COTC: US Navy COAMPS-TC", + COTI: "COTI: US Navy COAMPS-TC", + COT2: "COT2: US Navy COAMPS-TC", + CTMN: "CTMN: US Navy COAMPS-TC Ensemble Mean", + COAL: "COAL: US Navy COAMPS-TC, Atlantic Basin", + COAI: "COAI: US Navy COAMPS-TC, Atlantic Basin", + COA2: "COA2: US Navy COAMPS-TC, Atlantic Basin", + COCE: "COCE: US Navy COAMPS-TC, E Pacific Basin", + COEI: "COEI: US Navy COAMPS-TC, E Pacific Basin", + COE2: "COE2: US Navy COAMPS-TC, E Pacific Basin", + CTCX: "CTCX: Experimental US Navy COAMPS-TC", + CTCI: "CTCI: Experimental US Navy COAMPS-TC", + CTC2: "CTC2: Experimental US Navy COAMPS-TC", + EGRR: "EGRR: UKMET (GTS Tracker)", + EGRI: "EGRI: UKMET (GTS Tracker)", + EGR2: "EGR2: UKMET (GTS Tracker)", + UKX: "UKX: UKMET (NCEP Tracker)", + UKXI: "UKXI: UKMET (NCEP Tracker)", + UKX2: "UKX2: UKMET (NCEP Tracker)", + UKM: "UKM: UKMET (Automated Tracker)", + UKMI: "UKMI: UKMET (Automated Tracker)", + UKM2: "UKM2: UKMET (Automated Tracker)", + KEGR: "KEGR: UKMET (GTS Tracker; 2014)", + KEGI: "KEGI: UKMET (GTS Tracker; 2014)", + KEG2: "KEG2: UKMET (GTS Tracker; 2014)", + UE00: "UE00: UKMET MOGREPS Ensemble Control", + UEMN: "UEMN: UKMET MOGREPS Ensemble Mean", + UEMI: "UEMI: UKMET MOGREPS Ensemble Mean", + UEM2: "UEM2: UKMET MOGREPS Ensemble Mean", + ECM: "ECM: ECMWF", + ECMI: "ECMI: ECMWF", + ECM2: "ECM2: ECMWF", + ECMO: "ECMO: ECMWF (GTS Tracker)", + ECOI: "ECOI: ECMWF (GTS Tracker)", + ECO2: "ECO2: ECMWF (GTS Tracker)", + EMX: "EMX: ECMWF (NCEP Tracker)", + EMXI: "EMXI: ECMWF (NCEP Tracker)", + EMX2: "EMX2: ECMWF (NCEP Tracker)", + EHXI: "EHXI: ECMWF No Bias Correction (NCEP Tracker)", + ECMF: "ECMF: ECMWF", + ECME: "ECME: ECMWF EPS Ensemble Control (GTS Tracker)", + EC00: "EC00: ECMWF EPS Ensemble Control (NCEP Tracker)", + EEMN: "EEMN: ECMWF EPS Ensemble Mean (NCEP Tracker)", + EEMI: "EEMI: ECMWF EPS Ensemble Mean (NCEP Tracker)", + EMNI: "EMNI: ECMWF EPS Ensemble Mean (NCEP Tracker)", + EMN2: "EMN2: ECMWF EPS Ensemble Mean (NCEP Tracker)", + EMN3: "EMN3: ECMWF EPS Ensemble Mean (NCEP Tracker)", + EMN4: "EMN4: ECMWF EPS Ensemble Mean (NCEP Tracker)", + JGSM: "JGSM: Japanese Global Spectral Model", + JGSI: "JGSI: Japanese Global Spectral Model", + JGS2: "JGS2: Japanese Global Spectral Model", + NAM: "NAM: North American Mesoscale Model", + NAMI: "NAMI: North American Mesoscale Model", + NAM2: "NAM2: North American Mesoscale Model", + NGPS: "NGPS: US Navy NOGAPS", + NGPI: "NGPI: US Navy NOGAPS", + NGP2: "NGP2: US Navy NOGAPS", + NGX: "NGX: US Navy NOGAPS", + NGXI: "NGXI: US Navy NOGAPS", + NGX2: "NGX2: US Navy NOGAPS", + NVGM: "NVGM: US Navy NAVGEM", + NVGI: "NVGI: US Navy NAVGEM", + NVG2: "NVG2: US Navy NAVGEM", + HMON: "HMON: HMON Hurricane Model", + HMNI: "HMNI: HMON Hurricane Model", + HMN2: "HMN2: HMON Hurricane Model", + HMMN: "HMMN: HMON Ensemble Mean", + HAFS: "HAFS: Hurricane Analysis and Forecast System", + HAFI: "HAFI: Hurricane Analysis and Forecast System", + HAF2: "HAF2: Hurricane Analysis and Forecast System", + HFSA: "HFSA: Hurricane Analysis and Forecast System - A", + HFAI: "HFAI: Hurricane Analysis and Forecast System - A", + HFA2: "HFA2: Hurricane Analysis and Forecast System - A", + HFSB: "HFSB: Hurricane Analysis and Forecast System - B", + HFBI: "HFBI: Hurricane Analysis and Forecast System - B", + HFB2: "HFB2: Hurricane Analysis and Forecast System - B", + HAMN: "HAMN: HAFS Ensemble Mean", + HAMI: "HAMI: HAFS Ensemble Mean", + HAM2: "HAM2: HAFS Ensemble Mean", + HWRF: "HWRF: HWRF Hurricane Model", + HWFI: "HWFI: HWRF Hurricane Model", + HWF2: "HWF2: HWRF Hurricane Model", + HWFE: "HWFE: HWRF Model (ECMWF Fields)", + HWEI: "HWEI: HWRF Model (ECMWF Fields)", + HWE2: "HWE2: HWRF Model (ECMWF Fields)", + HW3F: "HW3F: HWRF Model v2013", + HW3I: "HW3I: HWRF Model v2013", + HW32: "HW32: HWRF Model v2013", + HHFI: "HHFI: HWRF Model No Bias Correction", + HWMN: "HWMN: HWRF Ensemble Mean", + HWMI: "HWMI: HWRF Ensemble Mean", + HWM2: "HWM2: HWRF Ensemble Mean", + HHYC: "HHYC: HWRF with HYCOM Ocean Model", + HHYI: "HHYI: HWRF with HYCOM Ocean Model", + HHY2: "HHY2: HWRF with HYCOM Ocean Model", + HWFH: "HWFH: Experimental NOAA/HRD HWRF", + HWHI: "HWHI: Experimental NOAA/HRD HWRF", + HWH2: "HWH2: Experimental NOAA/HRD HWRF", + GFEX: "GFEX Consensus", + HCCA: "HCCA Consensus", + HCON: "HCON Consensus", + ICON: "ICON Consensus", + IVCN: "IVCN Consensus", + IVCR: "IVCR Consensus", + IVRI: "IVRI Consensus", + IV15: "IV15 Consensus", + INT4: "INT4 Consensus", + GUNA: "GUNA Consensus", + GUNS: "GUNS Consensus", + CGUN: "CGUN Consensus", + TCON: "TCON Consensus", + TCOE: "TCOE Consensus", + TCOA: "TCOA Consensus", + TCCN: "TCCN Consensus", + TVCN: "TVCN Consensus", + TVCE: "TVCE Consensus", + TVCA: "TVCA Consensus", + TVCC: "TVCC Consensus", + TVCP: "TVCP Consensus", + TVCX: "TVCX Consensus", + TVCY: "TVCY Consensus", + RYOC: "RYOC Consensus", + MYOC: "MYOC Consensus", + RVCN: "RVCN Consensus", + GENA: "GENA Consensus", + CONE: "CONE Consensus", + CONI: "CONI Consensus", + CONU: "CONU Consensus", + CCON: "CCON: Corrected CONU Consensus", + BAMD: "BAMD: Deep-Layer Beta and Advection Model", + TABD: "TABD: Deep-Layer Trajectory and Beta Model", + BAMM: "BAMM: Medium-Layer Beta and Advection Model", + TABM: "TABM: Medium-Layer Trajectory and Beta Model", + BAMS: "BAMS: Shallow-Layer Beta and Advection Model", + TABS: "TABS: Shallow-Layer Trajectory and Beta Model", + KBMD: "KBMD: Parallel Deep-Layer Beta and Advection Model", + KBMM: "KBMM: Parallel Medium-Layer Beta and Advection Model", + KBMS: "KBMS: Parallel Shallow-Layer Beta and Advection Model", + CLIP: "CLIP: 72-hr Climatology and Persistence", + CLP5: "CLP5: 120-hr Climatology and Persistence", + KCLP: "KCLP: Parallel 72-hr Climatology and Persistence", + KCL5: "KCL5: Parallel 120-hr Climatology and Persistence", + TCLP: "TCLP: 168-hr Trajectory Climatology and Persistence", + LBAR: "LBAR: Limited Area Barotropic Model", + KLBR: "KLBR: Parallel Limited Area Barotropic Model", + LGEM: "LGEM: Logistical Growth Error Model", + KLGM: "KLGM: Parallel Logistical Growth Error Model", + SHFR: "SHFR: 72-hr SHIFOR Model", + SHF5: "SHF5: 120-hr SHIFOR Model", + DSHF: "DSHF: 72-hr Decay SHIFOR Model", + DSF5: "DSF5: 120-hr Decay SHIFOR Model", + KOCD: "KOCD: Parallel CLP5/Decay-SHIFOR", + KSFR: "KSFR: Parallel 72-hr SHIFOR Model", + KSF5: "KSF5: Parallel 120-hr SHIFOR Model", + SHIP: "SHIP: SHIPS Model", + DSHP: "DSHP: Decay SHIPS Model", + SHNS: "SHNS: SHIPS Model No IR Profile Predictors", + DSNS: "DSNS: Decay SHIPS Model No IR Profile Predictors", + KSHP: "KSHP: Parallel SHIPS Model", + KDSP: "KDSP: Parallel Decay SHIPS Model", + OCD5: "OCD5: Operational CLP5 and DSHF Blended Model", + DRCL: "DRCL: DeMaria Climatology and Persistence Model", + DRCI: "DRCI: DeMaria Climatology and Persistence Model", + MRCL: "MRCL: McAdie Climatology and Persistence Model", + MRCI: "MRCI: McAdie Climatology and Persistence Model", + AHQI: "AHQI: NCAR Hurricane Regional Model", + HURN: "HURN: HURRAN Model", + APSU: "APSU: PSU WRF-ARW Model", + APSI: "APSI: PSU WRF-ARW Model", + APS2: "APS2: PSU WRF-ARW Model", + A4PS: "A4PS: PSU WRF-ARW Doppler 2011", + A4PI: "A4PI: PSU WRF-ARW Doppler 2011", + A4P2: "A4P2: PSU WRF-ARW Doppler 2011", + A1PS: "A1PS: PSU WRF-ARW 1 km (Tail Doppler Radar Assimilated)", + A1PI: "A1PI: PSU WRF-ARW 1 km (Tail Doppler Radar Assimilated)", + A1P2: "A1P2: PSU WRF-ARW 1 km (Tail Doppler Radar Assimilated)", + A4NR: "A4NR: PSU WRF-ARW 4.5 km (No Tail Doppler Radar Assimilated)", + A4NI: "A4NI: PSU WRF-ARW 4.5 km (No Tail Doppler Radar Assimilated)", + A4N2: "A4N2: PSU WRF-ARW 4.5 km (No Tail Doppler Radar Assimilated)", + A4QI: "A4QI: PSU WRF-ARW 4.5 km (Tail Doppler Radar Assimilated; GFDL Interpolator)", + A4Q2: "A4Q2: PSU WRF-ARW 4.5 km (Tail Doppler Radar Assimilated; GFDL Interpolator)", + ANPS: "ANPS: PSU WRF-ARW 3 km (No Tail Doppler Radar Assimilated)", + AHW4: "AHW4: SUNY Advanced Hurricane WRF", + AHWI: "AHWI: SUNY Advanced Hurricane WRF", + AHW2: "AHW2: SUNY Advanced Hurricane WRF", + FIM9: "FIM9: Finite-Volume Icosahedral Model (FIM9)", + FM9I: "FM9I: Finite-Volume Icosahedral Model (FIM9)", + FM92: "FM92: Finite-Volume Icosahedral Model (FIM9)", + FIMY: "FIMY: Finite-Volume Icosahedral Model (FIMY)", + FIMI: "FIMI: Finite-Volume Icosahedral Model (FIMY)", + FIM2: "FIM2: Finite-Volume Icosahedral Model (FIMY)", + H3GP: "H3GP: NCEP/AOML Hires 3-Nest HWRF", + H3GI: "H3GI: NCEP/AOML Hires 3-Nest HWRF", + H3G2: "H3G2: NCEP/AOML Hires 3-Nest HWRF", + GFDL: "GFDL: NWS/GFDL Model", + GFDI: "GFDI: NWS/GFDL Model", + GFD2: "GFD2: NWS/GFDL Model", + GHTI: "GHTI: NWS/GFDL Model No Bias Correction", + GHMI: "GHMI: NWS/GFDL Model Variable Intensity Offset", + GHM2: "GHM2: NWS/GFDL Model Variable Intensity Offset", + GFDT: "GFDT: NWS/GFDL Model (NCEP Tracker)", + GFTI: "GFTI: NWS/GFDL Model (NCEP Tracker)", + GFT2: "GFT2: NWS/GFDL Model (NCEP Tracker)", + GFDN: "GFDN: NWS/GFDL Model (Navy Version)", + GFNI: "GFNI: NWS/GFDL Model (Navy Version)", + GFN2: "GFN2: NWS/GFDL Model (Navy Version)", + GFDU: "GFDU: NWS/GFDL Model (UKMET Version)", + GFUI: "GFUI: NWS/GFDL Model (UKMET Version)", + GFU2: "GFU2: NWS/GFDL Model (UKMET Version)", + GFD5: "GFD5: NWS/GFDL Model Parallel", + GF5I: "GF5I: NWS/GFDL Model Parallel", + GF52: "GF52: NWS/GFDL Model Parallel", + GFDE: "GFDE: NWS/GFDL Model (ECMWF Fields)", + GFEI: "GFEI: NWS/GFDL Model (ECMWF Fields)", + GFE2: "GFE2: NWS/GFDL Model (ECMWF Fields)", + GFDC: "GFDC: NWS/GFDL Coupled Model", + GFCI: "GFCI: NWS/GFDL Coupled Model", + GFC2: "GFC2: NWS/GFDL Coupled Model", + GFDA: "GFDA: NWS/GFDL Model With Aviation PBL", + GP00: "GP00: GFDL Ensemble Control", + G00I: "G00I: GFDL Ensemble Control", + G002: "G002: GFDL Ensemble Control", + GPMN: "GPMN: GFDL Ensemble Mean", + GPMI: "GPMI: GFDL Ensemble Mean", + GPM2: "GPM2: GFDL Ensemble Mean", + UWN4: "UWN4: UW Madison NMS Model 4km", + UW4I: "UW4I: UW Madison NMS Model 4km", + UW42: "UW42: UW Madison NMS Model 4km", + UWN8: "UWN8: UW NMS Model 8km", + UWNI: "UWNI: UW Madison NMS Model 8km", + UWN2: "UWN2: UW Madison NMS Model 8km", + UWQI: "UWQI: UW Madison NMS Model (GFDL Interpolator)", + UWQ2: "UWQ2: UW Madison NMS Model (GFDL Interpolator)", + TV15: "TV15: HFIP Stream 1_5 Model Consensus", + FSSE: "FSSE: FSU Superensemble", + FSSI: "FSSI: FSU Superensemble", + MMSE: "MMSE: FSU Multimodel Superensemble", + SPC3: "SPC3: Statistical Prediction of Intensity", + CARQ: "CARQ: Combined ARQ Position", + XTRP: "XTRP: 12-hr Extrapolation", + KXTR: "KXTR: Parallel 12-hr Extrapolation", + "90AE": "90AE: NHC-90 test", + "90BE": "90BE: NHC-90 test", + A98E: "A98E: NHC-98 Statistical-Dynamical Model", + A67: "A67: NHC-67 Statistical-Synoptic Model", + A72: "A72: NHC-72 Statistical-Dynamical Model", + A73: "A73: NHC-73 Statistic Model", + A83: "A83: NHC-83 Statistical-Dynamical Model", + A90E: "A90E: NHC-90 (Early) Statistical-Dynamical Model", + A90L: "A90L: NHC-90 (Late) Statistical-Dynamical Model", + A9UK: "A9UK: NHC-98 (UKMET Version)", + AFW1: "AFW1: US Air Force MM5 Model", + AF1I: "AF1I: US Air Force MM5 Model", + AF12: "AF12: US Air Force MM5 Model", + MM36: "MM36: US Air Force MM5 Model", + M36I: "M36I: US Air Force MM5 Model", + M362: "M362: US Air Force MM5 Model", + BAMA: "BAMA: BAM test A", + BAMB: "BAMB: BAM test B", + BAMC: "BAMC: BAM test C", + ETA: "ETA: ETA Model", + ETAI: "ETAI: ETA Model", + ETA2: "ETA2: ETA Model", + FV5: "FV5: NASA fvGCM Model", + FVGI: "FVGI: NASA fvGCM Model", + FVG2: "FVG2: NASA fvGCM Model", + MFM: "MFM: Medium Fine Mesh Model", + MRFO: "MRFO: Medium Range Forecast (MRF) Model", + NGM: "NGM: Nested Grid Model", + NGMI: "NGMI: Nested Grid Model", + NGM2: "NGM2: Nested Grid Model", + PSS: "PSS: EP Statistic-Synoptic Model", + PSDL: "PSDL: EP Statistic-Dynamic Model", + PSDE: "PSDL: EP (Early) Statistic-Dynamic Model", + P91L: "P91L: EP NHC-91 (Late) Statistic-Dynamic Model", + P91E: "P91E: EP NHC-91 (Early) Statistic-Dynamic Model", + P9UK: "P91E: EP NHC-91 (UKMET) Statistic-Dynamic Model", + QLM: "QLM: Quasi-Lagrangian Model", + QLMI: "QLMI: Quasi-Lagrangian Model", + QLM2: "QLM2: Quasi-Lagrangian Model", + SBAR: "SBAR: SANBAR Barotropic Model", + VBAR: "VBAR: VICBAR Model", + VBRI: "VBRI: VICBAR Model", + VBR2: "VBR2: VICBAR Model", + DTOP: "DTOP: Deterministic to Probabilistic Statistical Model", + DTPE: "DTPE: Deterministic to Probabilistic Statistical Model (ECMWF Version)", + RIOB: "RIOB: Bayesian RI Model", + RIOD: "RIOD: Discriminant Analysis RI Model", + RIOL: "RIOL: Logistic Regression RI Model", + RIOC: "RIOC: Consensus of RIOB, RIOD, RIOL", + EIOB: "EIOB: Bayesian RI Model (ECMWF Version)", + EIOD: "EIOD: Discriminant Analysis RI Model (ECMWF Version)", + EIOL: "EIOL: Logistic Regression RI Model (ECMWF Version)", + EIOC: "EIOC: Consensus of EIOB, EIOD, EIOL", + GCP0: "GCP1: GFS-CAM Physics v0 (NOAA/GSL)", + GCP1: "GCP1: GFS-CAM Physics v1 (NOAA/GSL)", + GCP2: "GCP1: GFS-CAM Physics v2 (NOAA/GSL)", + BEST: "BEST: Best Track", + BCD5: "BCD5: Best Track Decay", + }; +}; + // calculates mean, stdev, and other statistics for curve data points in all apps and plot types const get_err = function (sVals, sSecs, sLevs, appParams) { /* refer to perl error_library.pl sub get_stats @@ -1797,6 +2177,8 @@ export default matsDataUtils = { consolidateContour, excludeStatFromScorecard, readableStandardRegions, + readableTCCategories, + readableAdeckModels, get_err, ctcErrorPython, checkDiffContourSignificance, From 9b90b0067c8b0049178a9d8766a22a84ab6377c7 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 4 Jun 2024 18:03:11 -0600 Subject: [PATCH 19/55] main.js supports RI stats --- .../mats-common/imports/startup/both/mats-curve-params.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js index bc848d6ba..ae89f66ed 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js @@ -132,9 +132,11 @@ export const curveParamsByApp = { "plot-type", "basin", "statistic", + "variable", "year", "storm", "truth", + "threshold", "forecast-length", "dieoff-type", "valid-time", From c4db2676a26c389be2c1458dcee189a9dcd9ca85 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Wed, 12 Jun 2024 14:12:06 -0600 Subject: [PATCH 20/55] Fixed irritating date bug --- .../mats-common/imports/startup/client/select_util.js | 10 +++++++++- .../mats-common/public/MATSReleaseNotes.html | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/imports/startup/client/select_util.js b/meteor_packages/mats-common/imports/startup/client/select_util.js index 6d5a71865..2860281a1 100644 --- a/meteor_packages/mats-common/imports/startup/client/select_util.js +++ b/meteor_packages/mats-common/imports/startup/client/select_util.js @@ -58,7 +58,15 @@ const refreshDependents = function (event, param) { selectAllbool = document.getElementById("selectAll").checked; } try { - targetElem.dispatchEvent(new CustomEvent("refresh")); + if ( + !( + Session.get("confirmPlotChange") && + targetParam.type === matsTypes.InputTypes.dateRange + ) + ) { + // don't refresh the dates if we're just changing the plot type + targetElem.dispatchEvent(new CustomEvent("refresh")); + } } catch (re) { re.message = `INFO: refreshDependents of: ${param.name} dependent: ${targetParam.name} - error: ${re.message}`; setInfo(re.message); diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 8fc3b20f5..66b2ecfb1 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -12,6 +12,9 @@

All apps v5.3.1

Changes:

* Added RI stat plotting to MET Cyclone app.

* Added ability to plot very historical RAOB data in upper air app.

+

+ * Fixed bug that caused dates to spontaneously change when plot type was changed. +

* Fixed bug in data routines where some zeros were still being erroneously filtered out as if they were NaNs. From f8dd85e9e71a0c0768f0459af71ce0f7bb3f3226 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Wed, 12 Jun 2024 15:33:46 -0600 Subject: [PATCH 21/55] Updated meteor to 2.16, and also updated meteor and NPM packages --- .../.npm/package/npm-shrinkwrap.json | 617 ++++++------------ meteor_packages/mats-common/package.js | 20 +- .../mats-common/public/MATSReleaseNotes.html | 1 + 3 files changed, 215 insertions(+), 423 deletions(-) diff --git a/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json b/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json index cfb10b2e9..2cfe6142f 100644 --- a/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json +++ b/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json @@ -2,19 +2,19 @@ "lockfileVersion": 1, "dependencies": { "@babel/runtime": { - "version": "7.24.0", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.0.tgz", - "integrity": "sha512-Chk32uHMg6TnQdvw2e9IlqPpFX/6NLuK0Ys2PqLb7/gL5uFn9mXvK715FGLlOLQrcO4qIkNHkvPGktzzXexsFw==" + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.24.7.tgz", + "integrity": "sha512-UwgBRMjJP+xv857DCngvqXI3Iq6J4v0wXmwc6sapg+zyhbwmQX67LUEFrkK5tbyJ30jGuG3ZvWpBiB9LCy1kWw==" }, "@couchbase/couchbase-darwin-x64-napi": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/@couchbase/couchbase-darwin-x64-napi/-/couchbase-darwin-x64-napi-4.2.10.tgz", - "integrity": "sha512-F5Lc1WC4A96H1iMiAquo0iod3LQp2zZIyDj/3mXQbdeDEgJb+OtpOLEeHDAEIu4JTfex/j59aV7Yr7LwEua2uw==" + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/@couchbase/couchbase-darwin-x64-napi/-/couchbase-darwin-x64-napi-4.3.1.tgz", + "integrity": "sha512-pXaz2V9f6kSIP+zsNLQ/GMpfhGx7ixZ96j41spVYhuB5XjpESA3UAr2oc+hCY58o3fe+Nixj1+0DmpEEpnW61w==" }, "@fortawesome/fontawesome-free": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-6.5.1.tgz", - "integrity": "sha512-CNy5vSwN3fsUStPRLX7fUYojyuzoEMSXPl7zSLJ8TgtRfjv24LOnOWKT2zYwaHZCJGkdyRnTmstR0P+Ah503Gw==" + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-6.5.2.tgz", + "integrity": "sha512-hRILoInAx8GNT5IMkrtIt9blOdrqHOnPBH+k70aWUAqPZPgopb9G5EQJFpaBx/S8zp2fC+mPW349Bziuk1o28Q==" }, "ansi-regex": { "version": "5.0.1", @@ -42,9 +42,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "axios": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.7.tgz", - "integrity": "sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==" + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==" }, "call-bind": { "version": "1.0.7", @@ -97,29 +97,29 @@ "integrity": "sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==" }, "couchbase": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/couchbase/-/couchbase-4.2.10.tgz", - "integrity": "sha512-PPNUTfA2cgxEQYXraX563rcCIpcNIlJv1FmCBbBxtIyNh4s4hR1rc7LQ3MMsYMKTyNEOnJQL/PPnz9D6VfGf2Q==" + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/couchbase/-/couchbase-4.3.1.tgz", + "integrity": "sha512-WHD8xBQgDlNO2/qmtMYi3owcUC+RJvIoCt1FBWG7QslXedvbY9+AMBDOkpAz3hdIHXZ/ocs2DpOXHSpHJb6J2w==" }, "csv-stringify": { - "version": "6.4.6", - "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.6.tgz", - "integrity": "sha512-h2V2XZ3uOTLilF5dPIptgUfN/o2ia/80Ie0Lly18LAnw5s8Eb7kt8rfxSUy24AztJZas9f6DPZpVlzDUtFt/ag==" + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.5.0.tgz", + "integrity": "sha512-edlXFVKcUx7r8Vx5zQucsuMg4wb/xT6qyz+Sr1vnLrdXqlLD1+UKyWNyZ9zn6mUW1ewmGxrpVwAcChGF0HQ/2Q==" }, "datatables.net": { - "version": "1.13.11", - "resolved": "https://registry.npmjs.org/datatables.net/-/datatables.net-1.13.11.tgz", - "integrity": "sha512-AE6RkMXziRaqzPcu/pl3SJXeRa6fmXQG/fVjuRESujvkzqDCYEeKTTpPMuVJSGYJpPi32WGSphVNNY1G4nSN/g==" + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/datatables.net/-/datatables.net-2.0.8.tgz", + "integrity": "sha512-4/2dYx4vl975zQqZbyoVEm0huPe61qffjBRby7K7V+y9E+ORq4R8KavkgrNMmIgO6cl85Pg4AvCbVjvPCIT1Yg==" }, "datatables.net-bs": { - "version": "1.13.11", - "resolved": "https://registry.npmjs.org/datatables.net-bs/-/datatables.net-bs-1.13.11.tgz", - "integrity": "sha512-oZeXzC2Z+pZc9Wpil6XuuwMPMfllSd+hWEauhKr8q7bTM5fGuBKzAVzSr7Tuo1OzMBZ6NkAb+HSBGCGUn5Qbhg==" + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/datatables.net-bs/-/datatables.net-bs-2.0.8.tgz", + "integrity": "sha512-GIoN0SNCaEevJci0+Dpe8YMB3BBDRuCG1RwhNjPiqBV8nBfi9n3TBlZMC2/CrsWclIo6ujH6dVR/41FeOisgVw==" }, "datatables.net-dt": { - "version": "1.13.11", - "resolved": "https://registry.npmjs.org/datatables.net-dt/-/datatables.net-dt-1.13.11.tgz", - "integrity": "sha512-4GpS4OFLwIMfhb5UdJh6bEnh0E52jIJOlx7KLKs1pSce/xpHjvcmucbUWNaEndQIpHXtIxmVPoqcDB0ZbiVB+A==" + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/datatables.net-dt/-/datatables.net-dt-2.0.8.tgz", + "integrity": "sha512-9SG5MWJXq2IQSJWuH+2DvK/9AXduZr0wI/lQbrtBBd18Ck5sO8z3EXxy5wYxxjTFZ9Z+wl0lHsO//qR8QYmWIA==" }, "daterangepicker": { "version": "3.1.0", @@ -127,9 +127,9 @@ "integrity": "sha512-DxWXvvPq4srWLCqFugqSV+6CBt/CvQ0dnpXhQ3gl0autcIDAruG1PuGG3gC7yPRNytAD1oU1AcUOzaYhOawhTw==" }, "debug": { - "version": "4.3.4", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", - "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==" + "version": "4.3.5", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", + "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==" }, "deep-extend": { "version": "0.6.0", @@ -172,9 +172,9 @@ "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==" }, "follow-redirects": { - "version": "1.15.5", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", - "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==" + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==" }, "form-data": { "version": "4.0.0", @@ -249,9 +249,9 @@ "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==" }, "hasown": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.1.tgz", - "integrity": "sha512-1/th4MHjnwncwXsIW6QMzlvYL9kG5e/CpVvLRZe4XPa8TOUNbCELqmvhDmnkNsAjwaG4+I8gJJL0JBvTTLO9qA==" + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==" }, "inherits": { "version": "2.0.4", @@ -289,9 +289,9 @@ "integrity": "sha512-m4avr8yL8kmFN8psrbFFFmB/If14iN5o9nw/NgnnM+kybDJpRsAynV2BsfpTYrTRysYUdADVD7CkUUizgkpLfg==" }, "jquery-ui": { - "version": "1.13.2", - "resolved": "https://registry.npmjs.org/jquery-ui/-/jquery-ui-1.13.2.tgz", - "integrity": "sha512-wBZPnqWs5GaYJmo1Jj0k/mrSkzdQzKDwhXNtHKcBdAcKVxMM3KNYFq+iJ2i1rwiG53Z8M4mTn3Qxrm17uH1D4Q==" + "version": "1.13.3", + "resolved": "https://registry.npmjs.org/jquery-ui/-/jquery-ui-1.13.3.tgz", + "integrity": "sha512-D2YJfswSJRh/B8M/zCowDpNFfwsDmtfnMPwjJTyvl+CBqzpYwQ+gFYIbUUlzijy/Qvoy30H1YhoSui4MNYpRwA==" }, "json-parse-helpfulerror": { "version": "1.0.3", @@ -323,564 +323,355 @@ "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-0.14.0.tgz", "integrity": "sha512-ilVJrQ+019xOJuKjPzeH10aDNL4s8aTt4qQVAta9psn0W8Mu+LxzQv1S2u6lTOg/cQL+EHkVxD6nAdaacuCL/g==" }, - "lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==" - }, "memory-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/memory-stream/-/memory-stream-1.0.0.tgz", "integrity": "sha512-Wm13VcsPIMdG96dzILfij09PvuS3APtcKNh7M28FsCA/w6+1mjR7hhPmfFNoilX9xU7wTdhsH5lJAm6XNzdtww==" }, "meteor-node-stubs": { - "version": "1.2.7", - "resolved": "https://registry.npmjs.org/meteor-node-stubs/-/meteor-node-stubs-1.2.7.tgz", - "integrity": "sha512-20bAFUhEIOD/Cos2nmvhqf2NOKpTf63WVQ+nwuaX2OFj31sU6GL4KkNylkWum8McwsH0LsMr/F+UHhduTX7KRg==", + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/meteor-node-stubs/-/meteor-node-stubs-1.2.9.tgz", + "integrity": "sha512-EKRezc1/PblYtYiK4BOT3h5geWDo9AFBBSYNamPNh8AC5msUbVCcg8kekzAa7r7JPzBX8nZWaXEQVar4t8q/Hg==", "dependencies": { + "@meteorjs/crypto-browserify": { + "version": "3.12.1", + "dependencies": { + "hash-base": { + "version": "3.0.4" + } + } + }, "asn1.js": { - "version": "5.4.1", - "resolved": "https://registry.npmjs.org/asn1.js/-/asn1.js-5.4.1.tgz", - "integrity": "sha512-+I//4cYPccV8LdmBLiX8CYvf9Sp3vQsrqu2QNXRcrbiWvcx/UdlFiqUJJzxRQxgsZmvhXhn4cSKeSmoFjVdupA==", + "version": "4.10.1", "dependencies": { "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "version": "4.12.0" } } }, "assert": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/assert/-/assert-2.0.0.tgz", - "integrity": "sha512-se5Cd+js9dXJnu6Ag2JFc00t+HmHOen+8Q+L7O9zI0PqQXr20uk2J0XQqMxZEeo5U50o8Nvmmx7dZrl+Ufr35A==" + "version": "2.1.0" }, "available-typed-arrays": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.4.tgz", - "integrity": "sha512-SA5mXJWrId1TaQjfxUYghbqQ/hYioKmLJvPJyDuYRtXXenFNMjj4hSSt1Cf1xsuXSXrtxrVC5Ot4eU6cOtBDdA==" + "version": "1.0.5" }, "base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + "version": "1.5.1" }, "bn.js": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-5.2.0.tgz", - "integrity": "sha512-D7iWRBvnZE8ecXiLj/9wbxH7Tk79fAh8IHaTNq1RWRixsS02W+5qS+iE9yq6RYl0asXx5tw0bLhmT5pIfbSquw==" + "version": "5.2.0" }, "brorand": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/brorand/-/brorand-1.1.0.tgz", - "integrity": "sha1-EsJe/kCkXjwyPrhnWgoM5XsiNx8=" + "version": "1.1.0" }, "browserify-aes": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz", - "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==" + "version": "1.2.0" }, "browserify-cipher": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/browserify-cipher/-/browserify-cipher-1.0.1.tgz", - "integrity": "sha512-sPhkz0ARKbf4rRQt2hTpAHqn47X3llLkUGn+xEJzLjwY8LRs2p0v7ljvI5EyoRO/mexrNunNECisZs+gw2zz1w==" + "version": "1.0.1" }, "browserify-des": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/browserify-des/-/browserify-des-1.0.2.tgz", - "integrity": "sha512-BioO1xf3hFwz4kc6iBhI3ieDFompMhrMlnDFC4/0/vd5MokpuAc3R+LYbwTA9A5Yc9pq9UYPqffKpW2ObuwX5A==" + "version": "1.0.2" }, "browserify-rsa": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.1.0.tgz", - "integrity": "sha512-AdEER0Hkspgno2aR97SAf6vi0y0k8NuOpGnVH3O99rcA5Q6sh8QxcngtHuJ6uXwnfAXNM4Gn1Gb7/MV1+Ymbog==" + "version": "4.1.0" }, "browserify-sign": { - "version": "4.2.1", - "resolved": "https://registry.npmjs.org/browserify-sign/-/browserify-sign-4.2.1.tgz", - "integrity": "sha512-/vrA5fguVAKKAVTNJjgSm1tRQDHUU6DbwO9IROu/0WAzC8PKhucDSh18J0RMvVeHAn5puMd+QHC2erPRNf8lmg==" + "version": "4.2.3", + "dependencies": { + "bn.js": { + "version": "5.2.1" + }, + "hash-base": { + "version": "3.0.4" + }, + "readable-stream": { + "version": "2.3.8", + "dependencies": { + "safe-buffer": { + "version": "5.1.2" + } + } + }, + "string_decoder": { + "version": "1.1.1", + "dependencies": { + "safe-buffer": { + "version": "5.1.2" + } + } + } + } }, "browserify-zlib": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/browserify-zlib/-/browserify-zlib-0.2.0.tgz", - "integrity": "sha512-Z942RysHXmJrhqk88FmKBVq/v5tqmSkDz7p54G/MGyjMnCFFnC79XWNbg+Vta8W6Wb2qtSZTSxIGkJrRpCFEiA==" + "version": "0.2.0" }, "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==" + "version": "5.7.1" }, "buffer-xor": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", - "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=" + "version": "1.0.3" }, "builtin-status-codes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/builtin-status-codes/-/builtin-status-codes-3.0.0.tgz", - "integrity": "sha1-hZgoeOIbmOHGZCXgPQF0eI9Wnug=" + "version": "3.0.0" }, "call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==" + "version": "1.0.5" }, "cipher-base": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/cipher-base/-/cipher-base-1.0.4.tgz", - "integrity": "sha512-Kkht5ye6ZGmwv40uUDZztayT2ThLQGfnj/T71N/XzeZeo3nf8foyW7zGTsPYkEya3m5f3cAypH+qe7YOrM1U2Q==" + "version": "1.0.4" }, "console-browserify": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/console-browserify/-/console-browserify-1.2.0.tgz", - "integrity": "sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==" + "version": "1.2.0" }, "constants-browserify": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/constants-browserify/-/constants-browserify-1.0.0.tgz", - "integrity": "sha1-wguW2MYXdIqvHBYCF2DNJ/y4y3U=" + "version": "1.0.0" + }, + "core-util-is": { + "version": "1.0.3" }, "create-ecdh": { "version": "4.0.4", - "resolved": "https://registry.npmjs.org/create-ecdh/-/create-ecdh-4.0.4.tgz", - "integrity": "sha512-mf+TCx8wWc9VpuxfP2ht0iSISLZnt0JgWlrOKZiNqyUZWnjIaCIVNQArMHnCZKfEYRg6IM7A+NeJoN8gf/Ws0A==", "dependencies": { "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "version": "4.12.0" } } }, "create-hash": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz", - "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==" + "version": "1.2.0" }, "create-hmac": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz", - "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==" + "version": "1.1.7" }, - "crypto-browserify": { - "version": "3.12.0", - "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", - "integrity": "sha512-fz4spIh+znjO2VjL+IdhEpRJ3YN6sMzITSBijk6FK2UvTqruSQW+/cCZTSNsMiZNvUeq0CqurF+dAbyiGOY6Wg==" + "define-data-property": { + "version": "1.1.1" }, "define-properties": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", - "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==" + "version": "1.2.1" }, "des.js": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", - "integrity": "sha512-Q0I4pfFrv2VPd34/vfLrFOoRmlYj3OV50i7fskps1jZWK1kApMWWT9G6RRUeYedLcBDIhnSDaUvJMb3AhUlaEA==" + "version": "1.0.1" }, "diffie-hellman": { "version": "5.0.3", - "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz", - "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==", "dependencies": { "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "version": "4.12.0" } } }, "domain-browser": { - "version": "4.22.0", - "resolved": "https://registry.npmjs.org/domain-browser/-/domain-browser-4.22.0.tgz", - "integrity": "sha512-IGBwjF7tNk3cwypFNH/7bfzBcgSCbaMOD3GsaY1AU/JRrnHnYgEM0+9kQt52iZxjNsjBtJYtao146V+f8jFZNw==" + "version": "4.23.0" }, "elliptic": { - "version": "6.5.4", - "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.4.tgz", - "integrity": "sha512-iLhC6ULemrljPZb+QutR5TQGB+pdW6KGD5RSegS+8sorOZT+rdQFbsQFJgvN3eRqNALqJer4oQ16YvJHlU8hzQ==", + "version": "6.5.5", "dependencies": { "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "version": "4.12.0" } } }, - "es-abstract": { - "version": "1.18.3", - "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.18.3.tgz", - "integrity": "sha512-nQIr12dxV7SSxE6r6f1l3DtAeEYdsGpps13dR0TwJg1S8gyp4ZPgy3FZcHBgbiQqnoqSTb+oC+kO4UQ0C/J8vw==" - }, - "es-to-primitive": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", - "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==" - }, - "es6-object-assign": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/es6-object-assign/-/es6-object-assign-1.1.0.tgz", - "integrity": "sha1-wsNYJlYkfDnqEHyx5mUrb58kUjw=" - }, "events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + "version": "3.3.0" }, "evp_bytestokey": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/evp_bytestokey/-/evp_bytestokey-1.0.3.tgz", - "integrity": "sha512-/f2Go4TognH/KvCISP7OUsHn85hT9nUkxxA9BEWxFn+Oj9o8ZNLm/40hdlgSLyuOimsrTKLUMEorQexp/aPQeA==" + "version": "1.0.3" }, - "foreach": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/foreach/-/foreach-2.0.5.tgz", - "integrity": "sha1-C+4AUBiusmDQo6865ljdATbsG5k=" + "for-each": { + "version": "0.3.3" }, "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2" }, "get-intrinsic": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", - "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==" + "version": "1.2.2" }, - "has": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", - "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==" + "gopd": { + "version": "1.0.1" }, - "has-bigints": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/has-bigints/-/has-bigints-1.0.1.tgz", - "integrity": "sha512-LSBS2LjbNBTf6287JEbEzvJgftkF5qFkmCo9hDRpAzKhUOlJ+hx8dd4USs00SgsUNwc4617J9ki5YtEClM2ffA==" + "has-property-descriptors": { + "version": "1.0.1" + }, + "has-proto": { + "version": "1.0.1" }, "has-symbols": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.2.tgz", - "integrity": "sha512-chXa79rL/UC2KlX17jo3vRGz0azaWEx5tGqZg5pO3NUyEJVB17dMruQlzCCOfUvElghKcm5194+BCRvi2Rv/Gw==" + "version": "1.0.3" + }, + "has-tostringtag": { + "version": "1.0.0" }, "hash-base": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.1.0.tgz", - "integrity": "sha512-1nmYp/rhMDiE7AYkDw+lLwlAzz0AntGIe51F3RfFfEqyQ3feY2eI/NcwC6umIQVOASPMsWJLJScWKSSvzL9IVA==" + "version": "3.1.0" }, "hash.js": { - "version": "1.1.7", - "resolved": "https://registry.npmjs.org/hash.js/-/hash.js-1.1.7.tgz", - "integrity": "sha512-taOaskGt4z4SOANNseOviYDvjEJinIkRgmp7LbKP2YTTmVxWBl87s/uzK9r+44BclBSp2X7K1hqeNfz9JbBeXA==" + "version": "1.1.7" + }, + "hasown": { + "version": "2.0.0" }, "hmac-drbg": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", - "integrity": "sha1-0nRXAQJabHdabFRXk+1QL8DGSaE=" + "version": "1.0.1" }, "https-browserify": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/https-browserify/-/https-browserify-1.0.0.tgz", - "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=" + "version": "1.0.0" }, "ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" + "version": "1.2.1" }, "inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + "version": "2.0.4" }, "is-arguments": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.0.tgz", - "integrity": "sha512-1Ij4lOMPl/xB5kBDn7I+b2ttPMKa8szhEIrXDuXQD/oe3HJLTLhqhgGspwgyGd6MOywBUqVvYicF72lkgDnIHg==" - }, - "is-bigint": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.2.tgz", - "integrity": "sha512-0JV5+SOCQkIdzjBK9buARcV804Ddu7A0Qet6sHi3FimE9ne6m4BGQZfRn+NZiXbBk4F4XmHfDZIipLj9pX8dSA==" - }, - "is-boolean-object": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.1.1.tgz", - "integrity": "sha512-bXdQWkECBUIAcCkeH1unwJLIpZYaa5VvuygSyS/c2lf719mTKZDU5UdDRlpd01UjADgmW8RfqaP+mRaVPdr/Ng==" + "version": "1.1.1" }, "is-callable": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.3.tgz", - "integrity": "sha512-J1DcMe8UYTBSrKezuIUTUwjXsho29693unXM2YhJUTR2txK/eG47bvNa/wipPFmZFgr/N6f1GA66dv0mEyTIyQ==" - }, - "is-date-object": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.4.tgz", - "integrity": "sha512-/b4ZVsG7Z5XVtIxs/h9W8nvfLgSAyKYdtGWQLbqy6jA1icmgjf8WCoTKgeS4wy5tYaPePouzFMANbnj94c2Z+A==" + "version": "1.2.7" }, "is-generator-function": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.9.tgz", - "integrity": "sha512-ZJ34p1uvIfptHCN7sFTjGibB9/oBg17sHqzDLfuwhvmN/qLVvIQXRQ8licZQ35WJ8KuEQt/etnnzQFI9C9Ue/A==" + "version": "1.0.10" }, "is-nan": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/is-nan/-/is-nan-1.3.2.tgz", - "integrity": "sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==" - }, - "is-negative-zero": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.1.tgz", - "integrity": "sha512-2z6JzQvZRa9A2Y7xC6dQQm4FSTSTNWjKIYYTt4246eMTJmIo0Q+ZyOsU66X8lxK1AbB92dFeglPLrhwpeRKO6w==" - }, - "is-number-object": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.5.tgz", - "integrity": "sha512-RU0lI/n95pMoUKu9v1BZP5MBcZuNSVJkMkAG2dJqC4z2GlkGUNeH68SuHuBKBD/XFe+LHZ+f9BKkLET60Niedw==" - }, - "is-regex": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.3.tgz", - "integrity": "sha512-qSVXFz28HM7y+IWX6vLCsexdlvzT1PJNFSBuaQLQ5o0IEw8UDYW6/2+eCMVyIsbM8CNLX2a/QWmSpyxYEHY7CQ==" - }, - "is-string": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.6.tgz", - "integrity": "sha512-2gdzbKUuqtQ3lYNrUTQYoClPhm7oQu4UdpSZMp1/DGgkHBT8E2Z1l0yMdb6D4zNAxwDiMv8MdulKROJGNl0Q0w==" - }, - "is-symbol": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.4.tgz", - "integrity": "sha512-C/CPBqKWnvdcxqIARxyOh4v1UUEOCHpgDa0WYgpKDFMszcrPcffg5uhwSgPCLD2WWxmq6isisz87tzT01tuGhg==" + "version": "1.3.2" }, "is-typed-array": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.5.tgz", - "integrity": "sha512-S+GRDgJlR3PyEbsX/Fobd9cqpZBuvUS+8asRqYDMLCb2qMzt1oz5m5oxQCxOgUDxiWsOVNi4yaF+/uvdlHlYug==" + "version": "1.1.12" + }, + "isarray": { + "version": "1.0.0" }, "md5.js": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.5.tgz", - "integrity": "sha512-xitP+WxNPcTTOgnTJcrhM0xvdPepipPSf3I8EIpGKeFLjt3PlJLIDG3u8EX53ZIubkb+5U2+3rELYpEhHhzdkg==" + "version": "1.3.5" }, "miller-rabin": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/miller-rabin/-/miller-rabin-4.0.1.tgz", - "integrity": "sha512-115fLhvZVqWwHPbClyntxEVfVDfl9DLLTuJvq3g2O/Oxi8AiNouAHvDSzHS0viUJc+V5vm3eq91Xwqn9dp4jRA==", "dependencies": { "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "version": "4.12.0" } } }, "minimalistic-assert": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-assert/-/minimalistic-assert-1.0.1.tgz", - "integrity": "sha512-UtJcAD4yEaGtjPezWuO9wC4nwUnVH/8/Im3yEHQP4b67cXlD/Qr9hdITCU1xDbSEXg2XKNaP8jsReV7vQd00/A==" + "version": "1.0.1" }, "minimalistic-crypto-utils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/minimalistic-crypto-utils/-/minimalistic-crypto-utils-1.0.1.tgz", - "integrity": "sha1-9sAMHAsIIkblxNmd+4x8CDsrWCo=" + "version": "1.0.1" }, "object-inspect": { - "version": "1.10.3", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.10.3.tgz", - "integrity": "sha512-e5mCJlSH7poANfC8z8S9s9S2IN5/4Zb3aZ33f5s8YqoazCFzNLloLU8r5VCG+G7WoqLvAAZoVMcy3tp/3X0Plw==" + "version": "1.13.1" }, "object-is": { - "version": "1.1.5", - "resolved": "https://registry.npmjs.org/object-is/-/object-is-1.1.5.tgz", - "integrity": "sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==" + "version": "1.1.5" }, "object-keys": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", - "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==" + "version": "1.1.1" }, "object.assign": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.2.tgz", - "integrity": "sha512-ixT2L5THXsApyiUPYKmW+2EHpXXe5Ii3M+f4e+aJFAHao5amFRW6J0OO6c/LU8Be47utCx2GL89hxGB6XSmKuQ==" + "version": "4.1.4" }, "os-browserify": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/os-browserify/-/os-browserify-0.3.0.tgz", - "integrity": "sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc=" + "version": "0.3.0" }, "pako": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz", - "integrity": "sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==" + "version": "1.0.11" }, "parse-asn1": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.6.tgz", - "integrity": "sha512-RnZRo1EPU6JBnra2vGHj0yhp6ebyjBZpmUCLHWiFhxlzvBCCpAuZ7elsBp1PVAbQN0/04VD/19rfzlBSwLstMw==" + "version": "5.1.7", + "dependencies": { + "hash-base": { + "version": "3.0.4" + } + } }, "path-browserify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", - "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==" + "version": "1.0.1" }, "pbkdf2": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.2.tgz", - "integrity": "sha512-iuh7L6jA7JEGu2WxDwtQP1ddOpaJNC4KlDEFfdQajSGgGPNi4OyDc2R7QnbY2bR9QjBVGwgvTdNJZoE7RaxUMA==" + "version": "3.1.2" }, "process": { - "version": "0.11.10", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha1-czIwDoQBYb2j5podHZGn1LwW8YI=" + "version": "0.11.10" + }, + "process-nextick-args": { + "version": "2.0.1" }, "public-encrypt": { "version": "4.0.3", - "resolved": "https://registry.npmjs.org/public-encrypt/-/public-encrypt-4.0.3.tgz", - "integrity": "sha512-zVpa8oKZSz5bTMTFClc1fQOnyyEzpl5ozpi1B5YcvBrdohMjH2rfsBtyXcuNuwjsDIXmBYlF2N5FlJYhR29t8Q==", "dependencies": { "bn.js": { - "version": "4.12.0", - "resolved": "https://registry.npmjs.org/bn.js/-/bn.js-4.12.0.tgz", - "integrity": "sha512-c98Bf3tPniI+scsdk237ku1Dc3ujXQTSgyiPUDEOe7tRkhrqridvh8klBv0HCEso1OLOYcHuCv/cS6DNxKH+ZA==" + "version": "4.12.0" } } }, "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "version": "1.4.1" }, - "querystring": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", - "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + "qs": { + "version": "6.11.2" }, "querystring-es3": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/querystring-es3/-/querystring-es3-0.2.1.tgz", - "integrity": "sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM=" + "version": "0.2.1" }, "randombytes": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", - "integrity": "sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==" + "version": "2.1.0" }, "randomfill": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/randomfill/-/randomfill-1.0.4.tgz", - "integrity": "sha512-87lcbR8+MhcWcUiQ+9e+Rwx8MyR2P7qnt15ynUlbm3TU/fjbgz4GsvfSUDTemtCCtVCqb4ZcEFlyPNTh9bBTLw==" + "version": "1.0.4" }, "readable-stream": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", - "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==" + "version": "3.6.2" }, "ripemd160": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/ripemd160/-/ripemd160-2.0.2.tgz", - "integrity": "sha512-ii4iagi25WusVoiC4B4lq7pbXfAp3D9v5CwfkY33vffw2+pkDjY1D8GaN7spsxvCSx8dkPqOZCEZyfxcmJG2IA==" + "version": "2.0.2" }, "safe-buffer": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", - "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" + "version": "5.2.1" }, - "safer-buffer": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + "set-function-length": { + "version": "1.1.1" }, "setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha1-KQy7Iy4waULX1+qbg3Mqt4VvgoU=" + "version": "1.0.5" }, "sha.js": { - "version": "2.4.11", - "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz", - "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==" + "version": "2.4.11" + }, + "side-channel": { + "version": "1.0.4" }, "stream-browserify": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-3.0.0.tgz", - "integrity": "sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==" + "version": "3.0.0" }, "stream-http": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/stream-http/-/stream-http-3.2.0.tgz", - "integrity": "sha512-Oq1bLqisTyK3TSCXpPbT4sdeYNdmyZJv1LxpEm2vu1ZhK89kSE5YXwZc3cWk0MagGaKriBh9mCFbVGtO+vY29A==" - }, - "string.prototype.trimend": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.4.tgz", - "integrity": "sha512-y9xCjw1P23Awk8EvTpcyL2NIr1j7wJ39f+k6lvRnSMz+mz9CGz9NYPelDk42kOz6+ql8xjfK8oYzy3jAP5QU5A==" - }, - "string.prototype.trimstart": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.4.tgz", - "integrity": "sha512-jh6e984OBfvxS50tdY2nRZnoC5/mLFKOREQfw8t5yytkoUsJRNxvI/E39qu1sD0OtWI3OC0XgKSmcWwziwYuZw==" + "version": "3.2.0" }, "string_decoder": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", - "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==" + "version": "1.3.0" }, "timers-browserify": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/timers-browserify/-/timers-browserify-2.0.12.tgz", - "integrity": "sha512-9phl76Cqm6FhSX9Xe1ZUAMLtm1BLkKj2Qd5ApyWkXzsMRaA7dgr81kf4wJmQf/hAvg8EEyJxDo3du/0KlhPiKQ==" + "version": "2.0.12" }, "tty-browserify": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/tty-browserify/-/tty-browserify-0.0.1.tgz", - "integrity": "sha512-C3TaO7K81YvjCgQH9Q1S3R3P3BtN3RIM8n+OvX4il1K1zgE8ZhI0op7kClgkxtutIE8hQrcrHBXvIheqKUUCxw==" - }, - "unbox-primitive": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.0.1.tgz", - "integrity": "sha512-tZU/3NqK3dA5gpE1KtyiJUrEB0lxnGkMFHptJ7q6ewdZ8s12QrODwNbhIJStmJkd1QDXa1NRA8aF2A1zk/Ypyw==" + "version": "0.0.1" }, "url": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/url/-/url-0.11.0.tgz", - "integrity": "sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE=", - "dependencies": { - "punycode": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", - "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" - } - } + "version": "0.11.3" }, "util": { - "version": "0.12.4", - "resolved": "https://registry.npmjs.org/util/-/util-0.12.4.tgz", - "integrity": "sha512-bxZ9qtSlGUWSOy9Qa9Xgk11kSslpuZwaxCg4sNIDj6FLucDab2JxnHwyNTCpHMtK1MjoQiWQ6DiUMZYbSrO+Sw==" + "version": "0.12.5" }, "util-deprecate": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", - "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + "version": "1.0.2" }, "vm-browserify": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vm-browserify/-/vm-browserify-1.1.2.tgz", - "integrity": "sha512-2ham8XPWTONajOR0ohOKOHXkm3+gaBmGut3SRuu75xLd/RRaY6vqgh8NBYYk7+RW3u5AtzPQZG8F10LHkl0lAQ==" - }, - "which-boxed-primitive": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.2.tgz", - "integrity": "sha512-bwZdv0AKLpplFY2KZRX6TvyuN7ojjr7lwkg6ml0roIy9YeuSr7JS372qlNW18UQYzgYK9ziGcerWqZOmEn9VNg==" + "version": "1.1.2" }, "which-typed-array": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.4.tgz", - "integrity": "sha512-49E0SpUe90cjpoc7BOJwyPHRqSAd12c10Qm2amdEZrJPCY2NDxaW01zHITrem+rnETY3dwrbH3UUrUwagfCYDA==" + "version": "1.1.13" }, "xtend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", - "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" + "version": "4.0.2" } } }, @@ -977,9 +768,9 @@ "integrity": "sha512-RUOOOjHLhgR1MIQrCtnEqz/HJ1RMZBIN+REnpSUrfft2bXqXy69fwJASVziWExfFXsR1bCY0TznnHooNsCo0/w==" }, "qs": { - "version": "6.11.2", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.2.tgz", - "integrity": "sha512-tDNIz22aBzCDxLtVH++VnTfzxlfeK5CbqohpSqpJgj1Wg/cQbStNAz3NuqCs5vV+pjBsK4x4pN9HlVh7rcYRiA==" + "version": "6.12.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.12.1.tgz", + "integrity": "sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==" }, "rc": { "version": "1.2.8", @@ -1007,9 +798,9 @@ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==" + "version": "7.6.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", + "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==" }, "set-blocking": { "version": "2.0.0", @@ -1017,9 +808,9 @@ "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==" }, "set-function-length": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.1.tgz", - "integrity": "sha512-j4t6ccc+VsKwYHso+kElc5neZpjtq9EnRICFZtWyBsLojhmeF/ZBd/elqm22WJh/BziDe/SBiOeAt0m2mfLD0g==" + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", + "integrity": "sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==" }, "side-channel": { "version": "1.0.6", @@ -1062,9 +853,9 @@ "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==" }, "tar": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.0.tgz", - "integrity": "sha512-/Wo7DcT0u5HUV486xg675HtjNd3BXZ6xDbzsCUZPt5iw8bTQ63bP0Raut3mvro9u+CUyq7YQd8Cx55fsZXxqLQ==" + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/tar/-/tar-6.2.1.tgz", + "integrity": "sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==" }, "universalify": { "version": "2.0.1", diff --git a/meteor_packages/mats-common/package.js b/meteor_packages/mats-common/package.js index 82466ac78..15ebd7bb5 100644 --- a/meteor_packages/mats-common/package.js +++ b/meteor_packages/mats-common/package.js @@ -4,7 +4,7 @@ Package.describe({ name: "randyp:mats-common", - version: "5.2.3", + version: "5.3.1", // Brief, one-line summary of the package. summary: "MATScommon files provides common functionality for MATS/METexpress apps", // URL to the Git repository containing the source code for this package. @@ -15,21 +15,21 @@ Package.describe({ }); Package.onUse(function (api) { - api.versionsFrom("2.15"); + api.versionsFrom("2.16"); Npm.depends({ - "@fortawesome/fontawesome-free": "6.5.1", + "@fortawesome/fontawesome-free": "6.5.2", "fs-extra": "11.2.0", - "@babel/runtime": "7.24.0", - "meteor-node-stubs": "1.2.7", + "@babel/runtime": "7.24.7", + "meteor-node-stubs": "1.2.9", url: "0.11.3", jquery: "3.7.1", - "datatables.net-bs": "1.13.11", - "datatables.net-dt": "1.13.11", - "jquery-ui": "1.13.2", - "csv-stringify": "6.4.6", + "datatables.net-bs": "2.0.8", + "datatables.net-dt": "2.0.8", + "jquery-ui": "1.13.3", + "csv-stringify": "6.5.0", "node-file-cache": "1.0.2", "python-shell": "5.0.0", - couchbase: "4.2.10", + couchbase: "4.3.1", "simpl-schema": "3.4.6", "vanillajs-datepicker": "1.3.4", daterangepicker: "3.1.0", diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 66b2ecfb1..4943a531a 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -10,6 +10,7 @@

Production build date: Current revision

All apps v5.3.1

Changes:

+

* Updated all apps to Meteor v2.16.

* Added RI stat plotting to MET Cyclone app.

* Added ability to plot very historical RAOB data in upper air app.

From 58d4391a8dda0781e234f2329b7665a9e5847e6f Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Wed, 12 Jun 2024 15:55:40 -0600 Subject: [PATCH 22/55] Updated release notes --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 4943a531a..bdc668e5a 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -11,7 +11,7 @@

All apps v5.3.1

Changes:

* Updated all apps to Meteor v2.16.

-

* Added RI stat plotting to MET Cyclone app.

+

* Added RI CTC stat plotting to MET Cyclone app.

* Added ability to plot very historical RAOB data in upper air app.

* Fixed bug that caused dates to spontaneously change when plot type was changed. From fee9ef744ed3951c249dc7848ca4e889459ccb4e Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 14 Jun 2024 13:16:28 -0600 Subject: [PATCH 23/55] Updated documentation for release --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index bdc668e5a..f50dd9dec 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -5,7 +5,7 @@

-

Production build date: Current revision

+

Production build date: 2024.06.14

All apps v5.3.1

From 0d0e5f3c7c8178baead73825ba9181c4c3da645d Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Thu, 20 Jun 2024 13:45:09 -0600 Subject: [PATCH 24/55] RESET RELEASE NOTES --- .../mats-common/public/MATSReleaseNotes.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index f50dd9dec..1bdba098c 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -3,6 +3,20 @@ style="display: block; height: 2px; margin: 1em 0; border-top: 2px solid #000000" />
+
+

+

Production build date: Current revision

+

+

PUT APP VERSIONS HERE

+

+

Changes:

+

* PUT CHANGES HERE

+
+
+
+

Production build date: 2024.06.14

From 993306835ea0ca248b7fff514ae36a5c5543cb5b Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Thu, 20 Jun 2024 18:16:46 -0600 Subject: [PATCH 25/55] More surfrad4 line plots --- .../mats-common/imports/startup/both/mats-curve-params.js | 1 + 1 file changed, 1 insertion(+) diff --git a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js index ae89f66ed..5b165afde 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js @@ -387,6 +387,7 @@ export const curveParamsByApp = { "y-variable", "scale", "forecast-length", + "dieoff-type", "valid-time", "utc-cycle-start", "average", From d031cfd652cc603f72af87e72a91edee6a342c2a Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 21 Jun 2024 16:31:18 -0600 Subject: [PATCH 26/55] Updated release notes --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 1bdba098c..cd7ab1d49 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -7,10 +7,11 @@

Production build date: Current revision

-

PUT APP VERSIONS HERE

+

All apps v5.3.2

Changes:

-

* PUT CHANGES HERE

+

* Updated surface radiation app to include dieoff plots.

+

* Updated surface radiation app to use the faster surfrad4 database.


Date: Fri, 26 Jul 2024 17:33:43 -0400 Subject: [PATCH 27/55] Updated release notes --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index cd7ab1d49..f80c4f67c 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -5,7 +5,7 @@

-

Production build date: Current revision

+

Production build date: 2024.07.26

All apps v5.3.2

From 435b31e9cc47ef61e61eda33818b7a3c2fbe501c Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 6 Aug 2024 12:19:36 -0600 Subject: [PATCH 28/55] RESET RELEASE NOTES --- .../mats-common/public/MATSReleaseNotes.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index f80c4f67c..8161ec258 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -3,6 +3,20 @@ style="display: block; height: 2px; margin: 1em 0; border-top: 2px solid #000000" />
+
+

+

Production build date: Current revision

+

+

PUT APP VERSIONS HERE

+

+

Changes:

+

* PUT CHANGES HERE

+
+
+
+

Production build date: 2024.07.26

From 17fa1fd2e5a94b3953e57b759acb9077fe1ca409 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 9 Aug 2024 13:31:51 -0600 Subject: [PATCH 29/55] Fixed "undefined appName" client-side bug --- .../mats-common/templates/topnav/top_nav.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/meteor_packages/mats-common/templates/topnav/top_nav.js b/meteor_packages/mats-common/templates/topnav/top_nav.js index 9de907284..d9da986bf 100644 --- a/meteor_packages/mats-common/templates/topnav/top_nav.js +++ b/meteor_packages/mats-common/templates/topnav/top_nav.js @@ -1,6 +1,8 @@ /* * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ +import { Meteor } from "meteor/meteor"; +import { matsCollections, matsTypes } from "meteor/randyp:mats-common"; import matsMethods from "../../imports/startup/api/matsMethods"; const getRunEnvironment = function () { @@ -20,16 +22,21 @@ const getRunEnvironment = function () { Template.topNav.helpers({ transparentGif() { + const urlComponents = document.location.href.split("/"); const baseURL = Meteor.settings.public.home === undefined - ? `https://${document.location.href.split("/")[2]}` + ? `https://${urlComponents[2]}` : Meteor.settings.public.home; + const appName = + matsCollections.Settings === undefined || + matsCollections.Settings.findOne({}) === undefined || + matsCollections.Settings.findOne({}).appName === undefined + ? `${urlComponents[urlComponents.length - 1]}` + : matsCollections.Settings.findOne({}).appName; if (baseURL.includes("localhost")) { return `${baseURL}/packages/randyp_mats-common/public/img/noaa_transparent.png`; } - return `${baseURL}/${ - matsCollections.Settings.findOne({}).appName - }/packages/randyp_mats-common/public/img/noaa_transparent.png`; + return `${baseURL}/${appName}/packages/randyp_mats-common/public/img/noaa_transparent.png`; }, emailText() { switch (getRunEnvironment()) { From 9cb984c4c1d7e338381cd9204e828a82f49a0d78 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 9 Aug 2024 13:36:14 -0600 Subject: [PATCH 30/55] Cleanup --- meteor_packages/mats-common/templates/topnav/top_nav.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/meteor_packages/mats-common/templates/topnav/top_nav.js b/meteor_packages/mats-common/templates/topnav/top_nav.js index d9da986bf..a037fe994 100644 --- a/meteor_packages/mats-common/templates/topnav/top_nav.js +++ b/meteor_packages/mats-common/templates/topnav/top_nav.js @@ -42,7 +42,6 @@ Template.topNav.helpers({ switch (getRunEnvironment()) { case "metexpress": return "METexpress"; - break; default: if ( matsCollections.Settings.findOne({}) !== undefined && @@ -58,7 +57,6 @@ Template.topNav.helpers({ switch (getRunEnvironment()) { case "metexpress": return "National Weather Service"; - break; default: return "Global Systems Laboratory"; } @@ -67,7 +65,6 @@ Template.topNav.helpers({ switch (getRunEnvironment()) { case "metexpress": return "https://www.weather.gov/"; - break; default: return "http://gsl.noaa.gov/"; } @@ -76,7 +73,6 @@ Template.topNav.helpers({ switch (getRunEnvironment()) { case "metexpress": return "METexpress"; - break; default: if ( matsCollections.Settings.findOne({}) !== undefined && From 25806d0b2943121416edafe15cd05e5241e663d6 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 9 Aug 2024 14:58:40 -0600 Subject: [PATCH 31/55] Fixed broken csv-stringify --- .../imports/startup/api/matsMethods.js | 77 +++++++------------ 1 file changed, 29 insertions(+), 48 deletions(-) diff --git a/meteor_packages/mats-common/imports/startup/api/matsMethods.js b/meteor_packages/mats-common/imports/startup/api/matsMethods.js index d267f24ed..307535ad8 100644 --- a/meteor_packages/mats-common/imports/startup/api/matsMethods.js +++ b/meteor_packages/mats-common/imports/startup/api/matsMethods.js @@ -1324,6 +1324,32 @@ const _getDates = function (params, req, res, next) { } }; +// helper function for _getCSV +const stringifyCurveData = function (stringify, dataArray, res) { + const thisDataArray = dataArray[0]; + stringify.stringify( + thisDataArray, + { + header: true, + }, + function (err, output) { + if (err) { + console.log("error in _getCSV:", err); + res.write(`error,${err.toLocaleString()}`); + res.end(`

_getCSV Error! ${err.toLocaleString()}

`); + return; + } + res.write(output); + if (dataArray.length > 1) { + const newDataArray = dataArray.slice(1); + stringifyCurveData(stringify, newDataArray, res); + } else { + res.end(); + } + } + ); +}; + // private middleware for _getCSV route const _getCSV = function (params, req, res, next) { if (Meteor.isServer) { @@ -1333,42 +1359,12 @@ const _getCSV = function (params, req, res, next) { const result = _getFlattenedResultData(params.key, 0, -1000); const statArray = Object.values(result.stats); const dataArray = Object.values(result.data); - const statResultArray = []; - const dataResultArray = []; - for (let si = 0; si < statArray.length; si++) { - statResultArray.push(Object.keys(statArray[si])); // push the stat header for this curve(keys) - statResultArray.push( - statArray[si].n === 0 ? [statArray[si].label] : Object.values(statArray[si]) - ); // push the stats for this curve - } - for (let di = 0; di < dataArray.length; di++) { - const dataSubArray = Object.values(dataArray[di]); - const dataHeader = - dataSubArray[0] === undefined - ? statArray[di].label - : Object.keys(dataSubArray[0]); - // dataHeader[0] = 'label'; - dataHeader[0] = - dataSubArray[0] === undefined - ? "NO DATA" - : Object.keys(dataSubArray[0]).filter( - (key) => key.indexOf("Curve") !== -1 - )[0]; - dataResultArray.push(dataHeader); // push this curve data header (keys) - if (dataSubArray[0] === undefined) { - continue; - } - for (let dsi = 0; dsi < dataSubArray.length; dsi++) { - // push this curves data - dataResultArray.push(Object.values(dataSubArray[dsi])); - } - } const fileName = `matsplot-${moment.utc().format("YYYYMMDD-HH.mm.ss")}.csv`; res.setHeader("Content-disposition", `attachment; filename=${fileName}`); res.setHeader("Content-Type", "attachment.ContentType"); - stringify( - statResultArray, + stringify.stringify( + statArray, { header: true, }, @@ -1380,22 +1376,7 @@ const _getCSV = function (params, req, res, next) { return; } res.write(output); - stringify( - dataResultArray, - { - header: true, - }, - function (err, output) { - if (err) { - console.log("error in _getCSV:", err); - res.write(`error,${err.toLocaleString()}`); - res.end(`

_getCSV Error! ${err.toLocaleString()}

`); - return; - } - res.write(output); - res.end(); - } - ); + stringifyCurveData(stringify, dataArray, res); } ); } catch (e) { From 8d2d4345ca0afe8a04a6914e367403069088662e Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 9 Aug 2024 17:22:10 -0600 Subject: [PATCH 32/55] Updated release notes --- meteor_packages/mats-common/public/MATSReleaseNotes.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/meteor_packages/mats-common/public/MATSReleaseNotes.html b/meteor_packages/mats-common/public/MATSReleaseNotes.html index 8161ec258..c626ebc11 100755 --- a/meteor_packages/mats-common/public/MATSReleaseNotes.html +++ b/meteor_packages/mats-common/public/MATSReleaseNotes.html @@ -7,10 +7,11 @@

Production build date: Current revision

-

PUT APP VERSIONS HERE

+

All apps v5.3.3

Changes:

-

* PUT CHANGES HERE

+

* Fixed broken CSV export.

+

* Surface app now gets realtime/retro model status out of metadata.


Date: Tue, 13 Aug 2024 13:58:43 -0600 Subject: [PATCH 33/55] Linted data_query_util.js --- .../imports/startup/api/matsMethods.js | 14 +- .../imports/startup/server/data_diff_util.js | 50 +- .../startup/server/data_process_util.js | 88 +- .../imports/startup/server/data_query_util.js | 7160 +++++++++-------- .../startup/server/matsMiddle_common.js | 4 +- .../server/matsMiddle_dailyModelCycle.js | 12 +- .../startup/server/matsMiddle_dieoff.js | 8 +- .../imports/startup/server/matsMiddle_map.js | 20 +- .../startup/server/matsMiddle_timeSeries.js | 8 +- .../startup/server/matsMiddle_validTime.js | 8 +- .../public/python/python_query_util.py | 78 +- .../templates/textOutput/textOutput.js | 36 +- 12 files changed, 3764 insertions(+), 3722 deletions(-) diff --git a/meteor_packages/mats-common/imports/startup/api/matsMethods.js b/meteor_packages/mats-common/imports/startup/api/matsMethods.js index 307535ad8..f1b806a49 100644 --- a/meteor_packages/mats-common/imports/startup/api/matsMethods.js +++ b/meteor_packages/mats-common/imports/startup/api/matsMethods.js @@ -1485,7 +1485,7 @@ const _getFlattenedResultData = function (rk, p, np) { data[ci] !== undefined && data[ci].stats !== undefined && data[ci].stats[0] !== undefined && - data[ci].stats[0].n_forecast !== undefined; + data[ci].stats[0].nForecast !== undefined; // if the curve label is a reserved word do not process the curve (its a zero or max curve) var reservedWords = Object.values(matsTypes.ReservedWords); if (reservedWords.indexOf(data[ci].label) >= 0) { @@ -1525,10 +1525,10 @@ const _getFlattenedResultData = function (rk, p, np) { curveDataElement.cn = data[ci].stats[cdi].cn; } else if (isModeSingle) { curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.n_forecast = data[ci].stats[cdi].n_forecast; - curveDataElement.n_matched = data[ci].stats[cdi].n_matched; - curveDataElement.n_simple = data[ci].stats[cdi].n_simple; - curveDataElement.n_total = data[ci].stats[cdi].n_total; + curveDataElement.nForecast = data[ci].stats[cdi].nForecast; + curveDataElement.nMatched = data[ci].stats[cdi].nMatched; + curveDataElement.nSimple = data[ci].stats[cdi].nSimple; + curveDataElement.nTotal = data[ci].stats[cdi].nTotal; } else if (isModePairs) { curveDataElement.stat = data[ci].stats[cdi].stat; curveDataElement.n = data[ci].stats[cdi].n; @@ -1676,7 +1676,7 @@ const _getFlattenedResultData = function (rk, p, np) { var stats = {}; stats.label = data[0].label; stats["total number of obs"] = data[0].stats.reduce(function (prev, curr) { - return prev + curr.N_times; + return prev + curr.nTimes; }, 0); stats["mean difference"] = matsDataUtils.average(data[0].queryVal); stats["standard deviation"] = matsDataUtils.stdev(data[0].queryVal); @@ -1704,7 +1704,7 @@ const _getFlattenedResultData = function (rk, p, np) { for (var si = 0; si < data[0].siteName.length; si++) { var curveDataElement = {}; curveDataElement["site name"] = data[0].siteName[si]; - curveDataElement["number of times"] = data[0].stats[si].N_times; + curveDataElement["number of times"] = data[0].stats[si].nTimes; if (isCTC) { curveDataElement.stat = data[0].queryVal[si]; curveDataElement.hit = data[0].stats[si].hit; diff --git a/meteor_packages/mats-common/imports/startup/server/data_diff_util.js b/meteor_packages/mats-common/imports/startup/server/data_diff_util.js index 798c161eb..361785637 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_diff_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_diff_util.js @@ -62,10 +62,10 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes subLevs: [], stats: [], text: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], glob_stats: {}, xmin: Number.MAX_VALUE, xmax: Number.MIN_VALUE, @@ -134,10 +134,10 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes glob_max: null, glob_min: null, }, - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], xmin: Number.MAX_VALUE, xmax: Number.MIN_VALUE, ymin: Number.MAX_VALUE, @@ -312,13 +312,13 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes minuendDataSubObsSum = minuendData.subObsSum[minuendIndex]; minuendDataSubAbsSum = minuendData.subAbsSum[minuendIndex]; } else if ( - minuendData.n_total.length > 0 && - subtrahendData.n_total.length + minuendData.nTotal.length > 0 && + subtrahendData.nTotal.length ) { - minuendDataNForecast = minuendData.n_forecast[minuendIndex]; - minuendDataNMatched = minuendData.n_matched[minuendIndex]; - minuendDataNSimple = minuendData.n_simple[minuendIndex]; - minuendDataNTotal = minuendData.n_total[minuendIndex]; + minuendDataNForecast = minuendData.nForecast[minuendIndex]; + minuendDataNMatched = minuendData.nMatched[minuendIndex]; + minuendDataNSimple = minuendData.nSimple[minuendIndex]; + minuendDataNTotal = minuendData.nTotal[minuendIndex]; } minuendDataSubValues = minuendData.subVals[minuendIndex]; minuendDataSubSeconds = minuendData.subSecs[minuendIndex]; @@ -340,13 +340,13 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes subtrahendDataSubObsSum = subtrahendData.subObsSum[subtrahendIndex]; subtrahendDataSubAbsSum = subtrahendData.subAbsSum[subtrahendIndex]; } else if ( - minuendData.n_total.length > 0 && - subtrahendData.n_total.length + minuendData.nTotal.length > 0 && + subtrahendData.nTotal.length ) { - subtrahendDataNForecast = subtrahendData.n_forecast[subtrahendIndex]; - subtrahendDataNMatched = subtrahendData.n_matched[subtrahendIndex]; - subtrahendDataNSimple = subtrahendData.n_simple[subtrahendIndex]; - subtrahendDataNTotal = subtrahendData.n_total[subtrahendIndex]; + subtrahendDataNForecast = subtrahendData.nForecast[subtrahendIndex]; + subtrahendDataNMatched = subtrahendData.nMatched[subtrahendIndex]; + subtrahendDataNSimple = subtrahendData.nSimple[subtrahendIndex]; + subtrahendDataNTotal = subtrahendData.nTotal[subtrahendIndex]; } subtrahendDataSubValues = subtrahendData.subVals[subtrahendIndex]; subtrahendDataSubSeconds = subtrahendData.subSecs[subtrahendIndex]; @@ -438,11 +438,11 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes if (hasLevels) { d.subLevs.push(tempSubLevsArray); } - if (minuendData.n_total.length > 0 && subtrahendData.n_total.length) { - d.n_forecast.push(minuendDataNForecast - subtrahendDataNForecast); - d.n_matched.push(minuendDataNMatched - subtrahendDataNMatched); - d.n_simple.push(minuendDataNSimple - subtrahendDataNSimple); - d.n_total.push(minuendDataNTotal - subtrahendDataNTotal); + if (minuendData.nTotal.length > 0 && subtrahendData.nTotal.length) { + d.nForecast.push(minuendDataNForecast - subtrahendDataNForecast); + d.nMatched.push(minuendDataNMatched - subtrahendDataNMatched); + d.nSimple.push(minuendDataNSimple - subtrahendDataNSimple); + d.nTotal.push(minuendDataNTotal - subtrahendDataNTotal); } d.sum += d[independentVarName][largeIntervalCurveIndex]; } else { diff --git a/meteor_packages/mats-common/imports/startup/server/data_process_util.js b/meteor_packages/mats-common/imports/startup/server/data_process_util.js index 265f3b8aa..132b38e40 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_process_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_process_util.js @@ -272,21 +272,21 @@ const processDataXYCurve = function ( } else if (statType === "met-mode_single") { data.stats[di] = { stat: data.y[di], - n_forecast: data.n_forecast[di], - n_matched: data.n_matched[di], - n_simple: data.n_simple[di], - n_total: data.n_total[di], + nForecast: data.nForecast[di], + nMatched: data.nMatched[di], + nSimple: data.nSimple[di], + nTotal: data.nTotal[di], }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
Forecast objects: ${ - data.n_forecast[di] === null ? null : data.n_forecast[di].toString() + data.nForecast[di] === null ? null : data.nForecast[di].toString() }
Matched objects: ${ - data.n_matched[di] === null ? null : data.n_matched[di].toString() + data.nMatched[di] === null ? null : data.nMatched[di].toString() }
Simple objects: ${ - data.n_simple[di] === null ? null : data.n_simple[di].toString() + data.nSimple[di] === null ? null : data.nSimple[di].toString() }
Total objects: ${ - data.n_total[di] === null ? null : data.n_total[di].toString() + data.nTotal[di] === null ? null : data.nTotal[di].toString() }`; } else { data.stats[di] = { @@ -411,10 +411,10 @@ const processDataXYCurve = function ( data.subInterest = []; data.subData = []; data.subHeaders = []; - data.n_forecast = []; - data.n_matched = []; - data.n_simple = []; - data.n_total = []; + data.nForecast = []; + data.nMatched = []; + data.nSimple = []; + data.nTotal = []; data.subVals = []; data.subSecs = []; data.subLevs = []; @@ -727,21 +727,21 @@ const processDataProfile = function ( } else if (statType === "met-mode_single") { data.stats[di] = { stat: data.x[di], - n_forecast: data.n_forecast[di], - n_matched: data.n_matched[di], - n_simple: data.n_simple[di], - n_total: data.n_total[di], + nForecast: data.nForecast[di], + nMatched: data.nMatched[di], + nSimple: data.nSimple[di], + nTotal: data.nTotal[di], }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
Forecast objects: ${ - data.n_forecast[di] === null ? null : data.n_forecast[di].toString() + data.nForecast[di] === null ? null : data.nForecast[di].toString() }
Matched objects: ${ - data.n_matched[di] === null ? null : data.n_matched[di].toString() + data.nMatched[di] === null ? null : data.nMatched[di].toString() }
Simple objects: ${ - data.n_simple[di] === null ? null : data.n_simple[di].toString() + data.nSimple[di] === null ? null : data.nSimple[di].toString() }
Total objects: ${ - data.n_total[di] === null ? null : data.n_total[di].toString() + data.nTotal[di] === null ? null : data.nTotal[di].toString() }`; } else { data.stats[di] = { @@ -863,10 +863,10 @@ const processDataProfile = function ( data.subInterest = []; data.subData = []; data.subHeaders = []; - data.n_forecast = []; - data.n_matched = []; - data.n_simple = []; - data.n_total = []; + data.nForecast = []; + data.nMatched = []; + data.nSimple = []; + data.nTotal = []; data.subVals = []; data.subSecs = []; data.subLevs = []; @@ -1422,10 +1422,10 @@ const processDataPerformanceDiagram = function ( data.subInterest = []; data.subData = []; data.subHeaders = []; - data.n_forecast = []; - data.n_matched = []; - data.n_simple = []; - data.n_total = []; + data.nForecast = []; + data.nMatched = []; + data.nSimple = []; + data.nTotal = []; data.subVals = []; data.subSecs = []; data.subLevs = []; @@ -1520,10 +1520,10 @@ const processDataGridScaleProb = function ( data.subInterest = []; data.subData = []; data.subHeaders = []; - data.n_forecast = []; - data.n_matched = []; - data.n_simple = []; - data.n_total = []; + data.nForecast = []; + data.nMatched = []; + data.nSimple = []; + data.nTotal = []; data.subVals = []; data.subSecs = []; data.subLevs = []; @@ -1846,10 +1846,10 @@ const processDataHistogram = function ( subLevs: [], stats: [], text: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], glob_stats: {}, bin_stats: [], xmin: Number.MAX_VALUE, @@ -2016,10 +2016,10 @@ const processDataHistogram = function ( data.subInterest = []; data.subData = []; data.subHeaders = []; - data.n_forecast = []; - data.n_matched = []; - data.n_simple = []; - data.n_total = []; + data.nForecast = []; + data.nMatched = []; + data.nSimple = []; + data.nTotal = []; data.subVals = []; data.subSecs = []; data.subLevs = []; @@ -2121,10 +2121,10 @@ const processDataContour = function ( data.subAbsSum = []; data.subData = []; data.subHeaders = []; - data.n_forecast = []; - data.n_matched = []; - data.n_simple = []; - data.n_total = []; + data.nForecast = []; + data.nMatched = []; + data.nSimple = []; + data.nTotal = []; data.subVals = []; data.subSecs = []; data.subLevs = []; diff --git a/meteor_packages/mats-common/imports/startup/server/data_query_util.js b/meteor_packages/mats-common/imports/startup/server/data_query_util.js index a1f2574ac..a5244335a 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_query_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_query_util.js @@ -4,6 +4,34 @@ import { matsDataUtils, matsTypes, matsCollections } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; +import { _ } from "meteor/underscore"; + +/* global Assets */ +/* eslint-disable global-require */ + +// utility for querying the DB +const simplePoolQueryWrapSynchronous = function (pool, statement) { + /* + simple synchronous query of statement to the specified pool. + params : + pool - a predefined db pool (usually defined in main.js). i.e. wfip2Pool = mysql.createPool(wfip2Settings); + statement - String - a valid sql statement + actions - queries database and will wait until query returns. + return: rowset - an array of rows + throws: error + */ + if (Meteor.isServer) { + // eslint-disable-next-line global-require + const Future = require("fibers/future"); + const queryWrap = Future.wrap(function (wrappedPool, wrappedStatement, callback) { + wrappedPool.query(wrappedStatement, function (err, rows) { + return callback(err, rows); + }); + }); + return queryWrap(pool, statement).wait(); + } + return null; +}; // utility to get the cadence for a particular model, so that the query function // knows where to include null points for missing data. @@ -13,7 +41,7 @@ const getModelCadence = async function (pool, dataSource, startDate, endDate) { try { // this query should only return data if the model cadence is irregular. // otherwise, the cadence will be calculated later by the query function. - let cycles_raw; + let cyclesRaw; if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { /* we have to call the couchbase utilities as async functions but this @@ -22,7 +50,7 @@ const getModelCadence = async function (pool, dataSource, startDate, endDate) { */ const doc = await pool.getCb("MD:matsAux:COMMON:V01"); const newModel = doc.standardizedModleList[dataSource]; - cycles_raw = doc.primaryModelOrders[newModel] + cyclesRaw = doc.primaryModelOrders[newModel] ? doc.primaryModelOrders[newModel].cycleSecnds : undefined; } else { @@ -34,15 +62,13 @@ const getModelCadence = async function (pool, dataSource, startDate, endDate) { `where model = ` + `(select new_model as display_text from mats_common.standardized_model_list where old_model = '${dataSource}');` ); - cycles_raw = rows[0].cycle_seconds - ? JSON.parse(rows[0].cycle_seconds) - : undefined; + cyclesRaw = rows[0].cycle_seconds ? JSON.parse(rows[0].cycle_seconds) : undefined; } - const cycles_keys = cycles_raw ? Object.keys(cycles_raw).sort() : undefined; + const cyclesKeys = cyclesRaw ? Object.keys(cyclesRaw).sort() : undefined; // there can be difference cadences for different time periods (each time period is a key in cycles_keys, // with the cadences for that period represented as values in cycles_raw), so this section identifies all // time periods relevant to the requested date range, and returns the union of their cadences. - if (cycles_keys.length !== 0) { + if (cyclesKeys.length !== 0) { let newTime; let chosenStartTime; let chosenEndTime; @@ -50,8 +76,8 @@ const getModelCadence = async function (pool, dataSource, startDate, endDate) { let chosenEndIdx; let foundStart = false; let foundEnd = false; - for (var ti = cycles_keys.length - 1; ti >= 0; ti--) { - newTime = cycles_keys[ti]; + for (let ti = cyclesKeys.length - 1; ti >= 0; ti -= 1) { + newTime = cyclesKeys[ti]; if (startDate >= Number(newTime) && !foundStart) { chosenStartTime = newTime; chosenStartIdx = ti; @@ -68,20 +94,20 @@ const getModelCadence = async function (pool, dataSource, startDate, endDate) { } if (chosenStartTime !== undefined && chosenEndTime !== undefined) { if (Number(chosenStartTime) === Number(chosenEndTime)) { - cycles = cycles_raw[chosenStartTime]; + cycles = cyclesRaw[chosenStartTime]; } else if (chosenEndIdx - chosenStartIdx === 1) { - const startCycles = cycles_raw[chosenStartTime]; - const endCycles = cycles_raw[chosenEndTime]; + const startCycles = cyclesRaw[chosenStartTime]; + const endCycles = cyclesRaw[chosenEndTime]; cycles = _.union(startCycles, endCycles); } else { let middleCycles = []; let currCycles; - for (ti = chosenStartIdx + 1; ti < chosenEndIdx; ti++) { - currCycles = cycles_raw[cycles_keys[ti]]; + for (let ti = chosenStartIdx + 1; ti < chosenEndIdx; ti += 1) { + currCycles = cyclesRaw[cyclesKeys[ti]]; middleCycles = _.union(middleCycles, currCycles); } - const startCycles = cycles_raw[chosenStartTime]; - const endCycles = cycles_raw[chosenEndTime]; + const startCycles = cyclesRaw[chosenStartTime]; + const endCycles = cyclesRaw[chosenEndTime]; cycles = _.union(startCycles, endCycles, middleCycles); } cycles.sort(function (a, b) { @@ -94,8 +120,8 @@ const getModelCadence = async function (pool, dataSource, startDate, endDate) { // if there isn't a cycles_per_model entry, it just means that the model has a regular cadence } if (cycles !== null && cycles !== undefined && cycles.length > 0) { - for (let c = 0; c < cycles.length; c++) { - cycles[c] = cycles[c] * 1000; // convert to milliseconds + for (let c = 0; c < cycles.length; c += 1) { + cycles[c] *= 1000; // convert to milliseconds } } else { cycles = []; // regular cadence model--cycles will be calculated later by the query function @@ -103,28 +129,6 @@ const getModelCadence = async function (pool, dataSource, startDate, endDate) { return cycles; }; -// utility for querying the DB -const simplePoolQueryWrapSynchronous = function (pool, statement) { - /* - simple synchronous query of statement to the specified pool. - params : - pool - a predefined db pool (usually defined in main.js). i.e. wfip2Pool = mysql.createPool(wfip2Settings); - statement - String - a valid sql statement - actions - queries database and will wait until query returns. - return: rowset - an array of rows - throws: error - */ - if (Meteor.isServer) { - const Future = require("fibers/future"); - const queryWrap = Future.wrap(function (pool, statement, callback) { - pool.query(statement, function (err, rows) { - return callback(err, rows); - }); - }); - return queryWrap(pool, statement).wait(); - } -}; - // get stations in a predefined region const getStationsInCouchbaseRegion = function (pool, region) { if (Meteor.isServer) { @@ -144,6 +148,7 @@ const getStationsInCouchbaseRegion = function (pool, region) { dFuture.wait(); return sitesList; } + return null; }; // utility for querying the DB via Python @@ -177,15 +182,15 @@ const queryDBPython = function (pool, queryArray) { JSON.stringify(queryArray), ], }; - const pyShell = require("python-shell"); - const Future = require("fibers/future"); + const Future = require("fibers/future"); const future = new Future(); let d = []; let error = ""; - let N0 = []; - let N_times = []; + let n0 = []; + let nTimes = []; + const pyShell = require("python-shell"); pyShell.PythonShell.run("python_query_util.py", pyOptions) .then((results) => { // query callback - build the curve data from the results - or set an error @@ -196,8 +201,8 @@ const queryDBPython = function (pool, queryArray) { // get the data back from the query const parsedData = JSON.parse(results); d = parsedData.data; - N0 = parsedData.N0; - N_times = parsedData.N_times; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; error = parsedData.error; } // done waiting - have results @@ -211,8 +216,8 @@ const queryDBPython = function (pool, queryArray) { // wait for future to finish future.wait(); // check for nulls in output, since JSON only passes strings - for (let idx = 0; idx < d.length; idx++) { - for (let didx = 0; didx < d[idx].y.length; didx++) { + for (let idx = 0; idx < d.length; idx += 1) { + for (let didx = 0; didx < d[idx].y.length; didx += 1) { if (d[idx].y[didx] === "null") { d[idx].y[didx] = null; if (d[idx].subVals.length > 0) { @@ -226,11 +231,11 @@ const queryDBPython = function (pool, queryArray) { d[idx].subCn[didx] = NaN; } else if (queryArray[idx].statLineType === "mode_pair") { d[idx].subInterest[didx] = NaN; - } else if (queryArray[idx].statLineType === "mode_pair") { - d[idx].n_forecast[didx] = 0; - d[idx].n_matched[didx] = 0; - d[idx].n_simple[didx] = 0; - d[idx].n_total[didx] = 0; + } else if (queryArray[idx].statLineType === "mode_single") { + d[idx].nForecast[didx] = 0; + d[idx].nMatched[didx] = 0; + d[idx].nSimple[didx] = 0; + d[idx].nTotal[didx] = 0; } } d[idx].subSecs[didx] = NaN; @@ -248,11 +253,11 @@ const queryDBPython = function (pool, queryArray) { d[idx].subCn[didx] = NaN; } else if (queryArray[idx].statLineType === "mode_pair") { d[idx].subInterest[didx] = NaN; - } else if (queryArray[idx].statLineType === "mode_pair") { - d[idx].n_forecast[didx] = 0; - d[idx].n_matched[didx] = 0; - d[idx].n_simple[didx] = 0; - d[idx].n_total[didx] = 0; + } else if (queryArray[idx].statLineType === "mode_single") { + d[idx].nForecast[didx] = 0; + d[idx].nMatched[didx] = 0; + d[idx].nSimple[didx] = 0; + d[idx].nTotal[didx] = 0; } } d[idx].subSecs[didx] = NaN; @@ -263,391 +268,648 @@ const queryDBPython = function (pool, queryArray) { return { data: d, error, - N0, - N_times, + n0, + nTimes, }; } + return null; }; -// this method queries the database for timeseries plots -const queryDBTimeSeries = function ( - pool, - statementOrMwRows, - dataSource, - forecastOffset, - startDate, - endDate, - averageStr, - statisticStr, - validTimes, +// this method parses the returned query data for xy curves +const parseQueryDataXYCurve = function ( + rows, + d, appParams, - forceRegularCadence + statisticStr, + forecastOffset, + cycles, + regular ) { - if (Meteor.isServer) { - // upper air is only verified at 00Z and 12Z, so you need to force irregular models to verify at that regular cadence - let cycles = getModelCadence(pool, dataSource, startDate, endDate); // if irregular model cadence, get cycle times. If regular, get empty array. - if (validTimes.length > 0 && validTimes !== matsTypes.InputTypes.unused) { - if (typeof validTimes === "string" || validTimes instanceof String) { - validTimes = validTimes.split(","); - } - let vtCycles = validTimes.map(function (x) { - return (Number(x) - forecastOffset) * 3600 * 1000; - }); // selecting validTimes makes the cadence irregular - vtCycles = vtCycles.map(function (x) { - return x < 0 ? x + 24 * 3600 * 1000 : x; - }); // make sure no cycles are negative - vtCycles = vtCycles.sort(function (a, b) { - return Number(a) - Number(b); - }); // sort 'em - cycles = cycles.length > 0 ? _.intersection(cycles, vtCycles) : vtCycles; // if we already had cycles get the ones that correspond to valid times - } - const regular = - forceRegularCadence || - averageStr !== "None" || - !(cycles !== null && cycles.length > 0); // If curves have averaging, the cadence is always regular, i.e. it's the cadence of the average + /* + var d = { // d will contain the curve data + x: [], + y: [], + error_x: [], + error_y: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subSquareDiffSum: [], + subNSum: [], + subObsModelDiffSum: [], + subModelSum: [], + subObsSum: [], + subAbsSum: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + glob_stats: {}, + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + sum: 0 + }; + */ + const returnD = d; + const { plotType } = appParams; + const { hasLevels } = appParams; + const completenessQCParam = Number(appParams.completeness) / 100; + const outlierQCParam = + appParams.outliers !== "all" ? Number(appParams.outliers) : appParams.outliers; - let d = { - // d will contain the curve data - x: [], - y: [], - error_x: [], - error_y: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subSquareDiffSum: [], - subNSum: [], - subObsModelDiffSum: [], - subModelSum: [], - subObsSum: [], - subAbsSum: [], - subData: [], - subHeaders: [], - subVals: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], - glob_stats: {}, - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - sum: 0, - }; - let error = ""; - let N0 = []; - let N_times = []; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); + let isCTC = false; + let isScalar = false; + const { hideGaps } = appParams; - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBTimeSeries' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - let rows = null; - if (Array.isArray(statementOrMwRows)) { - rows = statementOrMwRows; - dFuture.return(); - } else { - (async () => { - rows = await pool.queryCB(statementOrMwRows); - // done waiting - have results - dFuture.return(); - })(); - } - dFuture.wait(); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; + // initialize local variables + const n0 = []; + const nTimes = []; + const curveIndependentVars = []; + const curveStats = []; + const subHit = []; + const subFa = []; + const subMiss = []; + const subCn = []; + const subSquareDiffSum = []; + const subNSum = []; + const subObsModelDiffSum = []; + const subModelSum = []; + const subObsSum = []; + const subAbsSum = []; + const subVals = []; + const subSecs = []; + const subLevs = []; + let timeInterval; + + for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + let independentVar; + switch (plotType) { + case matsTypes.PlotTypes.validtime: + independentVar = Number(rows[rowIndex].hr_of_day); + break; + case matsTypes.PlotTypes.gridscale: + independentVar = Number(rows[rowIndex].gridscale); + break; + case matsTypes.PlotTypes.gridscaleProb: + independentVar = Number(rows[rowIndex].binValue); + break; + case matsTypes.PlotTypes.profile: + independentVar = Number(rows[rowIndex].avVal.toString().replace("P", "")); + break; + case matsTypes.PlotTypes.timeSeries: + // default the time interval to an hour. It won't matter since it won't be used unless there's more than one data point. + timeInterval = + rows.length > 1 ? Number(rows[1].avtime) - Number(rows[0].avtime) : 3600; + independentVar = Number(rows[rowIndex].avtime) * 1000; + break; + case matsTypes.PlotTypes.dailyModelCycle: + independentVar = Number(rows[rowIndex].avtime) * 1000; + break; + case matsTypes.PlotTypes.dieoff: + independentVar = Number(rows[rowIndex].fcst_lead); + break; + case matsTypes.PlotTypes.threshold: + independentVar = Number(rows[rowIndex].thresh); + break; + default: + independentVar = Number(rows[rowIndex].avtime); + } + let stat; + if (rows[rowIndex].stat === undefined && rows[rowIndex].hit !== undefined) { + // this is a contingency table plot + isCTC = true; + const hit = Number(rows[rowIndex].hit); + const fa = Number(rows[rowIndex].fa); + const miss = Number(rows[rowIndex].miss); + const cn = Number(rows[rowIndex].cn); + const n = rows[rowIndex].sub_data.toString().split(",").length; + if (hit + fa + miss + cn > 0) { + stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); + stat = Number.isNaN(Number(stat)) ? null : stat; } else { - parsedData = parseQueryDataXYCurve( - rows, - d, - appParams, - statisticStr, - forecastOffset, - cycles, - regular - ); - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; + stat = null; } - } else { - // if this app isn't couchbase, use mysql - pool.query(statementOrMwRows, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else { - parsedData = parseQueryDataXYCurve( - rows, - d, - appParams, - statisticStr, - forecastOffset, - cycles, - regular - ); - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; - } - // done waiting - have results - dFuture.return(); - }); + } else if ( + rows[rowIndex].stat === undefined && + rows[rowIndex].square_diff_sum !== undefined + ) { + // this is a scalar partial sums plot + isScalar = true; + const squareDiffSum = Number(rows[rowIndex].square_diff_sum); + const NSum = Number(rows[rowIndex].N_sum); + const obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); + const modelSum = Number(rows[rowIndex].model_sum); + const obsSum = Number(rows[rowIndex].obs_sum); + const absSum = Number(rows[rowIndex].abs_sum); + if (NSum > 0) { + stat = matsDataUtils.calculateStatScalar( + squareDiffSum, + NSum, + obsModelDiffSum, + modelSum, + obsSum, + absSum, + statisticStr + ); + stat = Number.isNaN(Number(stat)) ? null : stat; + } else { + stat = null; + } + } else { + // not a contingency table plot or a scalar partial sums plot + stat = rows[rowIndex].stat === "NULL" ? null : rows[rowIndex].stat; } - // wait for the future to finish - sort of like 'back to the future' ;) - dFuture.wait(); - - return { - data: d, - error, - N0, - N_times, - }; - } -}; - -// this method queries the database for specialty curves such as profiles, dieoffs, threshold plots, valid time plots, grid scale plots, and histograms -const queryDBSpecialtyCurve = function ( - pool, - statementOrMwRows, - appParams, - statisticStr -) { - if (Meteor.isServer) { - let d = { - // d will contain the curve data - x: [], - y: [], - error_x: [], - error_y: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subSquareDiffSum: [], - subNSum: [], - subObsModelDiffSum: [], - subModelSum: [], - subObsSum: [], - subAbsSum: [], - subData: [], - subHeaders: [], - subVals: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], - glob_stats: {}, - bin_stats: [], - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - sum: 0, - }; - let error = ""; - let N0 = []; - let N_times = []; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); + n0.push(rows[rowIndex].n0); // number of values that go into a point on the graph + nTimes.push(rows[rowIndex].nTimes); // number of times that go into a point on the graph - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBSpecialtyCurve' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - let rows = null; - if (Array.isArray(statementOrMwRows)) { - rows = statementOrMwRows; - dFuture.return(); - } else { - (async () => { - rows = await pool.queryCB(statementOrMwRows); - dFuture.return(); - })(); + if (plotType === matsTypes.PlotTypes.timeSeries) { + // Find the minimum time_interval to be sure we don't accidentally go past the next data point. + if (rowIndex < rows.length - 1) { + const timeDiff = + Number(rows[rowIndex + 1].avtime) - Number(rows[rowIndex].avtime); + if (timeDiff < timeInterval) { + timeInterval = timeDiff; + } } - dFuture.wait(); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; - } else { - if (appParams.plotType !== matsTypes.PlotTypes.histogram) { - parsedData = parseQueryDataXYCurve( - rows, - d, - appParams, - statisticStr, - null, - null, - null + } + + // store sub values that will later be used for calculating error bar statistics. + let thisSubHit = []; + let thisSubFa = []; + let thisSubMiss = []; + let thisSubCn = []; + let thisSubSquareDiffSum = []; + let thisSubNSum = []; + let thisSubObsModelDiffSum = []; + let thisSubModelSum = []; + let thisSumObsSum = []; + let thisSumAbsSum = []; + let thisSubValues = []; + let thisSubSecs = []; + let thisSubLevs = []; + let thisSubStdev = 0; + let thisSubMean = 0; + let sdLimit = 0; + if ( + stat !== null && + rows[rowIndex].sub_data !== undefined && + rows[rowIndex].sub_data !== null + ) { + // parse the sub-data + try { + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + if (isCTC) { + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubHit.push(Number(currSubData[2])); + thisSubFa.push(Number(currSubData[3])); + thisSubMiss.push(Number(currSubData[4])); + thisSubCn.push(Number(currSubData[5])); + thisSubValues.push( + matsDataUtils.calculateStatCTC( + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + currSubData.length, + statisticStr + ) + ); + } else { + thisSubHit.push(Number(currSubData[1])); + thisSubFa.push(Number(currSubData[2])); + thisSubMiss.push(Number(currSubData[3])); + thisSubCn.push(Number(currSubData[4])); + thisSubValues.push( + matsDataUtils.calculateStatCTC( + Number(currSubData[1]), + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + currSubData.length, + statisticStr + ) + ); + } + } else if (isScalar) { + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubSquareDiffSum.push(Number(currSubData[2])); + thisSubNSum.push(Number(currSubData[3])); + thisSubObsModelDiffSum.push(Number(currSubData[4])); + thisSubModelSum.push(Number(currSubData[5])); + thisSumObsSum.push(Number(currSubData[6])); + thisSumAbsSum.push(Number(currSubData[7])); + thisSubValues.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + Number(currSubData[7]), + statisticStr + ) + ); + } else { + thisSubSquareDiffSum.push(Number(currSubData[1])); + thisSubNSum.push(Number(currSubData[2])); + thisSubObsModelDiffSum.push(Number(currSubData[3])); + thisSubModelSum.push(Number(currSubData[4])); + thisSumObsSum.push(Number(currSubData[5])); + thisSumAbsSum.push(Number(currSubData[6])); + thisSubValues.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[1]), + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + statisticStr + ) + ); + } + } else { + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubValues.push(Number(currSubData[2])); + } else { + thisSubValues.push(Number(currSubData[1])); + } + } + } + // Now that we have all the sub-values, we can get the standard deviation and remove the ones that exceed it + if (outlierQCParam !== "all") { + thisSubStdev = matsDataUtils.stdev(thisSubValues); + thisSubMean = matsDataUtils.average(thisSubValues); + sdLimit = outlierQCParam * thisSubStdev; + for (let svIdx = thisSubValues.length - 1; svIdx >= 0; svIdx -= 1) { + if (Math.abs(thisSubValues[svIdx] - thisSubMean) > sdLimit) { + if (isCTC) { + thisSubHit.splice(svIdx, 1); + thisSubFa.splice(svIdx, 1); + thisSubMiss.splice(svIdx, 1); + thisSubCn.splice(svIdx, 1); + } else if (isScalar) { + thisSubSquareDiffSum.splice(svIdx, 1); + thisSubNSum.splice(svIdx, 1); + thisSubObsModelDiffSum.splice(svIdx, 1); + thisSubModelSum.splice(svIdx, 1); + thisSumObsSum.splice(svIdx, 1); + thisSumAbsSum.splice(svIdx, 1); + } + thisSubValues.splice(svIdx, 1); + thisSubSecs.splice(svIdx, 1); + if (hasLevels) { + thisSubLevs.splice(svIdx, 1); + } + } + } + } + if (isCTC) { + const hit = matsDataUtils.sum(thisSubHit); + const fa = matsDataUtils.sum(thisSubFa); + const miss = matsDataUtils.sum(thisSubMiss); + const cn = matsDataUtils.sum(thisSubCn); + stat = matsDataUtils.calculateStatCTC( + hit, + fa, + miss, + cn, + thisSubHit.length, + statisticStr + ); + } else if (isScalar) { + const squareDiffSum = matsDataUtils.sum(thisSubSquareDiffSum); + const NSum = matsDataUtils.sum(thisSubNSum); + const obsModelDiffSum = matsDataUtils.sum(thisSubObsModelDiffSum); + const modelSum = matsDataUtils.sum(thisSubModelSum); + const obsSum = matsDataUtils.sum(thisSumObsSum); + const absSum = matsDataUtils.sum(thisSumAbsSum); + stat = matsDataUtils.calculateStatScalar( + squareDiffSum, + NSum, + obsModelDiffSum, + modelSum, + obsSum, + absSum, + statisticStr ); + } else if (statisticStr.toLowerCase().includes("count")) { + stat = matsDataUtils.sum(thisSubValues); } else { - parsedData = parseQueryDataHistogram(rows, d, appParams, statisticStr); + stat = matsDataUtils.average(thisSubValues); } - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in parseQueryDataXYCurve. The expected fields don't seem to be present in the results cache: ${e.message}`; + throw new Error(e.message); } } else { - // if this app isn't couchbase, use mysql - pool.query(statementOrMwRows, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else { - if (appParams.plotType !== matsTypes.PlotTypes.histogram) { - parsedData = parseQueryDataXYCurve( - rows, - d, - appParams, - statisticStr, - null, - null, - null - ); - } else { - parsedData = parseQueryDataHistogram(rows, d, appParams, statisticStr); - } - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; - } - // done waiting - have results - dFuture.return(); - }); + if (isCTC) { + thisSubHit = NaN; + thisSubFa = NaN; + thisSubMiss = NaN; + thisSubCn = NaN; + } else if (isScalar) { + thisSubSquareDiffSum = NaN; + thisSubNSum = NaN; + thisSubObsModelDiffSum = NaN; + thisSubModelSum = NaN; + thisSumObsSum = NaN; + thisSumAbsSum = NaN; + thisSubValues = NaN; + } + thisSubValues = NaN; + thisSubSecs = NaN; + if (hasLevels) { + thisSubLevs = NaN; + } } - // wait for the future to finish - sort of like 'back to the future' ;) - dFuture.wait(); - - return { - data: d, - error, - N0, - N_times, - }; - } -}; -// this method queries the database for performance diagrams -const queryDBReliability = function (pool, statement, appParams, kernel) { - if (Meteor.isServer) { - let d = { - // d will contain the curve data - x: [], - y: [], - binVals: [], - hitCount: [], - fcstCount: [], - fcstRawCount: [], - sample_climo: 0, - n: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subData: [], - subHeaders: [], - subRelHit: [], - subRelRawCount: [], - subRelCount: [], - subVals: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - }; - let error = ""; - const N0 = []; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); + // deal with missing forecast cycles for dailyModelCycle plot type + if ( + plotType === matsTypes.PlotTypes.dailyModelCycle && + rowIndex > 0 && + Number(independentVar) - Number(rows[rowIndex - 1].avtime * 1000) > + 3600 * 24 * 1000 && + !hideGaps + ) { + const cyclesMissing = + Math.ceil( + (Number(independentVar) - Number(rows[rowIndex - 1].avtime * 1000)) / + (3600 * 24 * 1000) + ) - 1; + const offsetFromMidnight = Math.floor( + (Number(independentVar) % (24 * 3600 * 1000)) / (3600 * 1000) + ); + for (let missingIdx = cyclesMissing; missingIdx > 0; missingIdx -= 1) { + curveIndependentVars.push( + independentVar - + 3600 * 24 * 1000 * missingIdx - + 3600 * offsetFromMidnight * 1000 + ); + curveStats.push(null); + if (isCTC) { + subHit.push(NaN); + subFa.push(NaN); + subMiss.push(NaN); + subCn.push(NaN); + subSquareDiffSum.push([]); + subNSum.push([]); + subObsModelDiffSum.push([]); + subModelSum.push([]); + subObsSum.push([]); + subAbsSum.push([]); + } else if (isScalar) { + subHit.push([]); + subFa.push([]); + subMiss.push([]); + subCn.push([]); + subSquareDiffSum.push(NaN); + subNSum.push(NaN); + subObsModelDiffSum.push(NaN); + subModelSum.push(NaN); + subObsSum.push(NaN); + subAbsSum.push(NaN); + } else { + subHit.push([]); + subFa.push([]); + subMiss.push([]); + subCn.push([]); + subSquareDiffSum.push([]); + subNSum.push([]); + subObsModelDiffSum.push([]); + subModelSum.push([]); + subObsSum.push([]); + subAbsSum.push([]); + } + subVals.push(NaN); + subSecs.push(NaN); + if (hasLevels) { + subLevs.push(NaN); + } + } + } + curveIndependentVars.push(independentVar); + curveStats.push(stat); + subHit.push(thisSubHit); + subFa.push(thisSubFa); + subMiss.push(thisSubMiss); + subCn.push(thisSubCn); + subSquareDiffSum.push(thisSubSquareDiffSum); + subNSum.push(thisSubNSum); + subObsModelDiffSum.push(thisSubObsModelDiffSum); + subModelSum.push(thisSubModelSum); + subObsSum.push(thisSumObsSum); + subAbsSum.push(thisSumAbsSum); + subVals.push(thisSubValues); + subSecs.push(thisSubSecs); + if (hasLevels) { + subLevs.push(thisSubLevs); + } + } - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBReliability' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - (async () => { - const rows = await pool.queryCB(statement); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; + const nTimesMax = Math.max(...nTimes); + let sum = 0; + let indVarMin = Number.MAX_VALUE; + let indVarMax = -1 * Number.MAX_VALUE; + let depVarMin = Number.MAX_VALUE; + let depVarMax = -1 * Number.MAX_VALUE; + let dIdx; + + for (dIdx = 0; dIdx < curveIndependentVars.length; dIdx += 1) { + const thisNTimes = nTimes[dIdx]; + // Make sure that we don't have any points with a smaller completeness value than specified by the user. + if (curveStats[dIdx] === null || thisNTimes < completenessQCParam * nTimesMax) { + if (!hideGaps) { + if (plotType === matsTypes.PlotTypes.profile) { + // profile has the stat first, and then the independent var. The others have independent var and then stat. + // this is in the pattern of x-plotted-variable, y-plotted-variable. + returnD.x.push(null); + returnD.y.push(curveIndependentVars[dIdx]); + returnD.error_x.push(null); // placeholder } else { - parsedData = parseQueryDataReliability(rows, d, appParams, kernel); - d = parsedData.d; + returnD.x.push(curveIndependentVars[dIdx]); + returnD.y.push(null); + returnD.error_y.push(null); // placeholder } - dFuture.return(); - })(); + returnD.subHit.push(NaN); + returnD.subFa.push(NaN); + returnD.subMiss.push(NaN); + returnD.subCn.push(NaN); + returnD.subSquareDiffSum.push(NaN); + returnD.subNSum.push(NaN); + returnD.subObsModelDiffSum.push(NaN); + returnD.subModelSum.push(NaN); + returnD.subObsSum.push(NaN); + returnD.subAbsSum.push(NaN); + returnD.subVals.push(NaN); + returnD.subSecs.push(NaN); + if (hasLevels) { + returnD.subLevs.push(NaN); + } + } } else { - // if this app isn't couchbase, use mysql - pool.query(statement, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; + // there's valid data at this point, so store it + sum += curveStats[dIdx]; + if (plotType === matsTypes.PlotTypes.profile) { + // profile has the stat first, and then the independent var. The others have independent var and then stat. + // this is in the pattern of x-plotted-variable, y-plotted-variable. + returnD.x.push(curveStats[dIdx]); + returnD.y.push(curveIndependentVars[dIdx]); + returnD.error_x.push(null); // placeholder + } else { + returnD.x.push(curveIndependentVars[dIdx]); + returnD.y.push(curveStats[dIdx]); + returnD.error_y.push(null); // placeholder + } + returnD.subHit.push(subHit[dIdx]); + returnD.subFa.push(subFa[dIdx]); + returnD.subMiss.push(subMiss[dIdx]); + returnD.subCn.push(subCn[dIdx]); + returnD.subSquareDiffSum.push(subSquareDiffSum[dIdx]); + returnD.subNSum.push(subNSum[dIdx]); + returnD.subObsModelDiffSum.push(subObsModelDiffSum[dIdx]); + returnD.subModelSum.push(subModelSum[dIdx]); + returnD.subObsSum.push(subObsSum[dIdx]); + returnD.subAbsSum.push(subAbsSum[dIdx]); + returnD.subVals.push(subVals[dIdx]); + returnD.subSecs.push(subSecs[dIdx]); + if (hasLevels) { + returnD.subLevs.push(subLevs[dIdx]); + } + indVarMin = + curveIndependentVars[dIdx] < indVarMin ? curveIndependentVars[dIdx] : indVarMin; + indVarMax = + curveIndependentVars[dIdx] > indVarMax ? curveIndependentVars[dIdx] : indVarMax; + depVarMin = curveStats[dIdx] < depVarMin ? curveStats[dIdx] : depVarMin; + depVarMax = curveStats[dIdx] > depVarMax ? curveStats[dIdx] : depVarMax; + } + } + + // add in any missing times in the time series + if (plotType === matsTypes.PlotTypes.timeSeries && !hideGaps) { + timeInterval *= 1000; + const dayInMilliSeconds = 24 * 3600 * 1000; + let lowerIndependentVar; + let upperIndependentVar; + let newTime; + let thisCadence; + let numberOfDaysBack; + for (dIdx = curveIndependentVars.length - 2; dIdx >= 0; dIdx -= 1) { + lowerIndependentVar = curveIndependentVars[dIdx]; + upperIndependentVar = curveIndependentVars[dIdx + 1]; + const cyclesMissing = + Math.ceil( + (Number(upperIndependentVar) - Number(lowerIndependentVar)) / timeInterval + ) - 1; + for (let missingIdx = cyclesMissing; missingIdx > 0; missingIdx -= 1) { + newTime = lowerIndependentVar + missingIdx * timeInterval; + if (!regular) { + // if it's not a regular model, we only want to add a null point if this is an init time that should have had a forecast. + thisCadence = newTime % dayInMilliSeconds; // current hour of day (valid time) + if (Number(thisCadence) - Number(forecastOffset) * 3600 * 1000 < 0) { + // check to see if cycle time was on a previous day -- if so, need to wrap around 00Z to get current hour of day (cycle time) + numberOfDaysBack = Math.ceil( + (-1 * (Number(thisCadence) - Number(forecastOffset) * 3600 * 1000)) / + dayInMilliSeconds + ); + thisCadence = + Number(thisCadence) - + Number(forecastOffset) * 3600 * 1000 + + numberOfDaysBack * dayInMilliSeconds; // current hour of day (cycle time) + } else { + thisCadence = Number(thisCadence) - Number(forecastOffset) * 3600 * 1000; // current hour of day (cycle time) + } + if (cycles.indexOf(thisCadence) !== -1) { + matsDataUtils.addNullPoint( + returnD, + dIdx + 1, + matsTypes.PlotTypes.timeSeries, + "x", + newTime, + "y", + isCTC, + isScalar, + hasLevels + ); + } } else { - parsedData = parseQueryDataReliability(rows, d, appParams, kernel); - d = parsedData.d; + matsDataUtils.addNullPoint( + returnD, + dIdx + 1, + matsTypes.PlotTypes.timeSeries, + "x", + newTime, + "y", + isCTC, + isScalar, + hasLevels + ); } - // done waiting - have results - dFuture.return(); - }); + } } - // wait for the future to finish - sort of like 'back to the future' ;) - dFuture.wait(); + } - return { - data: d, - error, - }; + if (plotType === matsTypes.PlotTypes.profile) { + returnD.xmin = depVarMin; + returnD.xmax = depVarMax; + returnD.ymin = indVarMin; + returnD.ymax = indVarMax; + } else { + returnD.xmin = indVarMin; + returnD.xmax = indVarMax; + returnD.ymin = depVarMin; + returnD.ymax = depVarMax; } + + returnD.sum = sum; + + return { + d: returnD, + n0, + nTimes, + }; }; -// this method queries the database for performance diagrams -const queryDBPerformanceDiagram = function (pool, statement, appParams) { - if (Meteor.isServer) { +// this method parses the returned query data for performance diagrams +const parseQueryDataReliability = function (rows, d, appParams, kernel) { + /* let d = { // d will contain the curve data x: [], y: [], binVals: [], - oy_all: [], - on_all: [], + hitCount: [], + fcstCount: [], + fcstRawCount: [], + sample_climo: 0, n: [], subHit: [], subFa: [], @@ -655,3399 +917,3179 @@ const queryDBPerformanceDiagram = function (pool, statement, appParams) { subCn: [], subData: [], subHeaders: [], + subRelHit: [], + subRelRawCount: [], + subRelCount: [], subVals: [], subSecs: [], subLevs: [], stats: [], text: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], xmin: Number.MAX_VALUE, xmax: Number.MIN_VALUE, ymin: Number.MAX_VALUE, ymax: Number.MIN_VALUE, }; - let error = ""; - let N0 = []; - let N_times = []; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); + */ - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBSPerformanceDiagram' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - (async () => { - const rows = await pool.queryCB(statement); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; - } else { - parsedData = parseQueryDataPerformanceDiagram(rows, d, appParams); - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; - } - dFuture.return(); - })(); - } else { - // if this app isn't couchbase, use mysql - pool.query(statement, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else { - parsedData = parseQueryDataPerformanceDiagram(rows, d, appParams); - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; + const returnD = d; + const { hasLevels } = appParams; + + // initialize local variables + const binVals = []; + const hitCounts = []; + const fcstCounts = []; + const fcstRawCounts = []; + const observedFreqs = []; + let totalForecastCount = 0; + const subRelCount = []; + const subRelRawCount = []; + const subRelHit = []; + const subVals = []; + const subSecs = []; + const subLevs = []; + for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + if ( + Number(rows[rowIndex].kernel) === 0 && + rows[rowIndex].rawfcstcount !== undefined && + rows[rowIndex].rawfcstcount !== "NULL" + ) { + totalForecastCount += Number(rows[rowIndex].rawfcstcount); + let subRawCounts = []; // actually raw counts but I'm re-using fields + // parse the sub-data + if (rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null) { + try { + const subData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < subData.length; sdIdx += 1) { + currSubData = subData[sdIdx].split(";"); + if (hasLevels) { + subRawCounts.push(Number(currSubData[3])); + } else { + subRawCounts.push(Number(currSubData[2])); + } + } + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in parseQueryDataReliability. The expected fields don't seem to be present in the results cache: ${e.message}`; + throw new Error(e.message); } - // done waiting - have results - dFuture.return(); - }); + } else { + subRawCounts = NaN; + } + subRelRawCount.push(subRawCounts); } - // wait for the future to finish - sort of like 'back to the future' ;) - dFuture.wait(); - - return { - data: d, - error, - N0, - N_times, - }; - } -}; - -// this method queries the database for performance diagrams -const queryDBSimpleScatter = function ( - pool, - statement, - appParams, - statisticXStr, - statisticYStr -) { - if (Meteor.isServer) { - let d = { - // d will contain the curve data - x: [], - y: [], - binVals: [], - subSquareDiffSumX: [], - subNSumX: [], - subObsModelDiffSumX: [], - subModelSumX: [], - subObsSumX: [], - subAbsSumX: [], - subSquareDiffSumY: [], - subNSumY: [], - subObsModelDiffSumY: [], - subModelSumY: [], - subObsSumY: [], - subAbsSumY: [], - subValsX: [], - subValsY: [], - subSecsX: [], - subSecsY: [], - subSecs: [], - subLevsX: [], - subLevsY: [], - subLevs: [], - stats: [], - text: [], - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - sum: 0, - }; - let error = ""; - let N0 = []; - let N_times = []; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); + if (Number(rows[rowIndex].kernel) === Number(kernel)) { + const binVal = Number(rows[rowIndex].binValue); + let hitCount; + let fcstCount; + let observedFreq; + if ( + rows[rowIndex].fcstcount !== undefined && + rows[rowIndex].hitcount !== undefined + ) { + hitCount = + rows[rowIndex].hitcount === "NULL" ? null : Number(rows[rowIndex].hitcount); + fcstCount = + rows[rowIndex].fcstcount === "NULL" ? null : Number(rows[rowIndex].fcstcount); + observedFreq = hitCount / fcstCount; + } else { + hitCount = null; + fcstCount = null; + } + binVals.push(binVal); + hitCounts.push(hitCount); + fcstCounts.push(fcstCount); + observedFreqs.push(observedFreq); - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBSPerformanceDiagram' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - (async () => { - const rows = await pool.queryCB(statement); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; - } else { - parsedData = parseQueryDataSimpleScatter( - rows, - d, - appParams, - statisticXStr, - statisticYStr - ); - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; + let thisSubRelHit = []; + let subRelCounts = []; + const thisSubValues = []; + let thisSubSecs = []; + let thisSubLevs = []; + if ( + hitCount !== null && + rows[rowIndex].sub_data !== undefined && + rows[rowIndex].sub_data !== null + ) { + // parse the sub-data + try { + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + subRelCounts.push(Number(currSubData[2])); + thisSubRelHit.push(Number(currSubData[4])); + // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. + thisSubValues.push(0); + } else { + subRelCounts.push(Number(currSubData[1])); + thisSubRelHit.push(Number(currSubData[3])); + // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. + thisSubValues.push(0); + } + } + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in parseQueryDataReliability. The expected fields don't seem to be present in the results cache: ${e.message}`; + throw new Error(e.message); } - dFuture.return(); - })(); - } else { - // if this app isn't couchbase, use mysql - pool.query(statement, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else { - parsedData = parseQueryDataSimpleScatter( - rows, - d, - appParams, - statisticXStr, - statisticYStr - ); - d = parsedData.d; - N0 = parsedData.N0; - N_times = parsedData.N_times; + } else { + subRelCounts = NaN; + thisSubRelHit = NaN; + thisSubSecs = NaN; + if (hasLevels) { + thisSubLevs = NaN; } - // done waiting - have results - dFuture.return(); - }); + } + subRelCount.push(subRelCounts); + subRelHit.push(thisSubRelHit); + subVals.push(thisSubValues); + subSecs.push(thisSubSecs); + if (hasLevels) { + subLevs.push(thisSubLevs); + } } - // wait for the future to finish - sort of like 'back to the future' ;) - dFuture.wait(); + } - return { - data: d, - error, - N0, - N_times, - }; + const sampleClimo = matsDataUtils.sum(hitCounts) / totalForecastCount; + + returnD.x = + binVals[binVals.length - 1] === 100 + ? binVals.map((bin) => bin / 100) + : binVals.map((bin) => bin / 10); + returnD.y = observedFreqs; + returnD.binVals = binVals; + returnD.hitCount = hitCounts; + returnD.fcstCount = fcstCounts; + returnD.fcstRawCount = fcstRawCounts; + returnD.sample_climo = sampleClimo; + returnD.subRelHit = subRelHit; + returnD.subRelCount = subRelCount; + returnD.subRelRawCount = subRelRawCount; + returnD.subVals = subVals; + returnD.subSecs = subSecs; + returnD.subLevs = subLevs; + + let xMin = Number.MAX_VALUE; + let xMax = -1 * Number.MAX_VALUE; + let yMin = Number.MAX_VALUE; + let yMax = -1 * Number.MAX_VALUE; + + for (let didx = 0; didx < binVals.length; didx += 1) { + xMin = returnD.x[didx] !== null && returnD.x[didx] < xMin ? returnD.x[didx] : xMin; + xMax = returnD.x[didx] !== null && returnD.x[didx] > xMax ? returnD.x[didx] : xMax; + yMin = returnD.y[didx] !== null && returnD.y[didx] < yMin ? returnD.y[didx] : yMin; + yMax = returnD.y[didx] !== null && returnD.y[didx] > yMax ? returnD.y[didx] : yMax; } + + returnD.xmin = xMin; + returnD.xmax = xMax; + returnD.ymin = yMin; + returnD.ymax = yMax; + return { + d: returnD, + }; }; -// this method queries the database for map plots -const queryDBMapScalar = function ( - pool, - statementOrMwRows, - dataSource, - statistic, - variable, - varUnits, - siteMap, - appParams -) { - if (Meteor.isServer) { - // d will contain the curve data - let d = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - color: [], - stats: [], - text: [], - }; - let dLowest = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "", - }; - let dLow = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "", - }; - let dModerate = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "", - }; - let dHigh = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "", - }; - let dHighest = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "", - }; - let valueLimits = {}; +// this method parses the returned query data for performance diagrams +const parseQueryDataPerformanceDiagram = function (rows, d, appParams) { + /* + var d = { // d will contain the curve data + x: [], + y: [], + binVals: [], + oy_all: [], + on_all: [], + n: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + }; + */ - let error = ""; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); + const returnD = d; + const { hasLevels } = appParams; - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBSpecialtyCurve' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - let rows = null; - if (Array.isArray(statementOrMwRows)) { - rows = statementOrMwRows; - dFuture.return(); - } else { - (async () => { - rows = await pool.queryCB(statementOrMwRows); - dFuture.return(); - })(); - } - dFuture.wait(); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; - } else { - parsedData = parseQueryDataMapScalar( - rows, - d, - dLowest, - dLow, - dModerate, - dHigh, - dHighest, - dataSource, - siteMap, - statistic, - variable, - varUnits, - appParams, - true - ); - d = parsedData.d; - dLowest = parsedData.dLowest; - dLow = parsedData.dLow; - dModerate = parsedData.dModerate; - dHigh = parsedData.dHigh; - dHighest = parsedData.dHighest; - valueLimits = parsedData.valueLimits; - } + // initialize local variables + const n0 = []; + const nTimes = []; + const successes = []; + const pods = []; + const binVals = []; + const oyAll = []; + const onAll = []; + const subHit = []; + const subFa = []; + const subMiss = []; + const subCn = []; + const subVals = []; + const subSecs = []; + const subLevs = []; + for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + const binVal = Number(rows[rowIndex].binVal); + let pod; + let success; + let oy; + let on; + if (rows[rowIndex].pod !== undefined && rows[rowIndex].far !== undefined) { + pod = rows[rowIndex].pod === "NULL" ? null : Number(rows[rowIndex].pod); + success = rows[rowIndex].far === "NULL" ? null : 1 - Number(rows[rowIndex].far); + oy = rows[rowIndex].oy === "NULL" ? null : Number(rows[rowIndex].oy_all); + on = rows[rowIndex].on === "NULL" ? null : Number(rows[rowIndex].on_all); } else { - // if this app isn't couchbase, use mysql - pool.query(statementOrMwRows, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else { - parsedData = parseQueryDataMapScalar( - rows, - d, - dLowest, - dLow, - dModerate, - dHigh, - dHighest, - dataSource, - siteMap, - statistic, - variable, - varUnits, - appParams, - false - ); - d = parsedData.d; - dLowest = parsedData.dLowest; - dLow = parsedData.dLow; - dModerate = parsedData.dModerate; - dHigh = parsedData.dHigh; - dHighest = parsedData.dHighest; - valueLimits = parsedData.valueLimits; + pod = null; + success = null; + oy = null; + on = null; + } + n0.push(rows[rowIndex].n0); // number of values that go into a point on the graph + nTimes.push(rows[rowIndex].nTimes); // number of times that go into a point on the graph + successes.push(success); + pods.push(pod); + binVals.push(binVal); + oyAll.push(oy); + onAll.push(on); + + let thisSubHit = []; + let thisSubFa = []; + let thisSubMiss = []; + let thisSubCn = []; + const thisSubValues = []; + let thisSubSecs = []; + let thisSubLevs = []; + if ( + pod !== null && + rows[rowIndex].sub_data !== undefined && + rows[rowIndex].sub_data !== null + ) { + // parse the sub-data + try { + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubHit.push(Number(currSubData[2])); + thisSubFa.push(Number(currSubData[3])); + thisSubMiss.push(Number(currSubData[4])); + thisSubCn.push(Number(currSubData[5])); + // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. + thisSubValues.push(0); + } else { + thisSubHit.push(Number(currSubData[1])); + thisSubFa.push(Number(currSubData[2])); + thisSubMiss.push(Number(currSubData[3])); + thisSubCn.push(Number(currSubData[4])); + // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. + thisSubValues.push(0); + } } - // done waiting - have results - dFuture.return(); - }); - // wait for future to finish - dFuture.wait(); + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in parseQueryDataPerformanceDiagram. The expected fields don't seem to be present in the results cache: ${e.message}`; + throw new Error(e.message); + } + } else { + thisSubHit = NaN; + thisSubFa = NaN; + thisSubMiss = NaN; + thisSubCn = NaN; + thisSubSecs = NaN; + if (hasLevels) { + thisSubLevs = NaN; + } + } + subHit.push(thisSubHit); + subFa.push(thisSubFa); + subMiss.push(thisSubMiss); + subCn.push(thisSubCn); + subVals.push(thisSubValues); + subSecs.push(thisSubSecs); + if (hasLevels) { + subLevs.push(thisSubLevs); } + } - return { - data: d, - dataLowest: dLowest, - dataLow: dLow, - dataModerate: dModerate, - dataHigh: dHigh, - dataHighest: dHighest, - valueLimits, - error, - }; + returnD.x = successes; + returnD.y = pods; + returnD.binVals = binVals; + returnD.oy_all = oyAll; + returnD.on_all = onAll; + returnD.subHit = subHit; + returnD.subFa = subFa; + returnD.subMiss = subMiss; + returnD.subCn = subCn; + returnD.subVals = subVals; + returnD.subSecs = subSecs; + returnD.subLevs = subLevs; + returnD.n = n0; + + let successMin = Number.MAX_VALUE; + let successMax = -1 * Number.MAX_VALUE; + let podMin = Number.MAX_VALUE; + let podMax = -1 * Number.MAX_VALUE; + + for (let dIdx = 0; dIdx < binVals.length; dIdx += 1) { + successMin = + successes[dIdx] !== null && successes[dIdx] < successMin + ? successes[dIdx] + : successMin; + successMax = + successes[dIdx] !== null && successes[dIdx] > successMax + ? successes[dIdx] + : successMax; + podMin = podMin[dIdx] !== null && pods[dIdx] < podMin ? pods[dIdx] : podMin; + podMax = podMin[dIdx] !== null && pods[dIdx] > podMax ? pods[dIdx] : podMax; } + + returnD.xmin = successMin; + returnD.xmax = successMax; + returnD.ymin = podMin; + returnD.ymax = podMax; + + return { + d: returnD, + n0, + nTimes, + }; }; -// this method queries the database for map plots in CTC apps -const queryDBMapCTC = function ( - pool, - statementOrMwRows, - dataSource, - statistic, - siteMap, - appParams +// this method parses the returned query data for simple scatter plots +const parseQueryDataSimpleScatter = function ( + rows, + d, + appParams, + statisticXStr, + statisticYStr ) { - if (Meteor.isServer) { - // d will contain the curve data - let d = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - color: [], - stats: [], - text: [], - }; - // for skill scores <= 10% - let dPurple = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(128,0,255)", - }; - // for skill scores <= 20% - let dPurpleBlue = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(64,0,255)", - }; - // for skill scores <= 30% - let dBlue = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(0,0,255)", - }; - // for skill scores <= 40% - let dBlueGreen = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(64,128,128)", - }; - // for skill scores <= 50% - let dGreen = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(128,255,0)", - }; - // for skill scores <= 60% - let dGreenYellow = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(160,224,0)", - }; - // for skill scores <= 70% - let dYellow = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(192,192,0)", - }; - // for skill scores <= 80% - let dOrange = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(255,128,0)", - }; - // for skill scores <= 90% - let dOrangeRed = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(255,64,0)", - }; - // for skill scores <= 100% - let dRed = { - siteName: [], - queryVal: [], - lat: [], - lon: [], - stats: [], - text: [], - color: "rgb(255,0,0)", - }; - let valueLimits = {}; - - let error = ""; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); - - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBSpecialtyCurve' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - let rows = null; - if (Array.isArray(statementOrMwRows)) { - rows = statementOrMwRows; - dFuture.return(); - } else { - (async () => { - rows = await pool.queryCB(statementOrMwRows); - dFuture.return(); - })(); - } - dFuture.wait(); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; - } else { - parsedData = parseQueryDataMapCTC( - rows, - d, - dPurple, - dPurpleBlue, - dBlue, - dBlueGreen, - dGreen, - dGreenYellow, - dYellow, - dOrange, - dOrangeRed, - dRed, - dataSource, - siteMap, - statistic, - appParams, - true - ); - d = parsedData.d; - dPurple = parsedData.dPurple; - dPurpleBlue = parsedData.dPurpleBlue; - dBlue = parsedData.dBlue; - dBlueGreen = parsedData.dBlueGreen; - dGreen = parsedData.dGreen; - dGreenYellow = parsedData.dGreenYellow; - dYellow = parsedData.dYellow; - dOrange = parsedData.dOrange; - dOrangeRed = parsedData.dOrangeRed; - dRed = parsedData.dRed; - valueLimits = parsedData.valueLimits; - } - } else { - // if this app isn't couchbase, use mysql - pool.query(statementOrMwRows, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else { - parsedData = parseQueryDataMapCTC( - rows, - d, - dPurple, - dPurpleBlue, - dBlue, - dBlueGreen, - dGreen, - dGreenYellow, - dYellow, - dOrange, - dOrangeRed, - dRed, - dataSource, - siteMap, - statistic, - appParams, - false - ); - d = parsedData.d; - dPurple = parsedData.dPurple; - dPurpleBlue = parsedData.dPurpleBlue; - dBlue = parsedData.dBlue; - dBlueGreen = parsedData.dBlueGreen; - dGreen = parsedData.dGreen; - dGreenYellow = parsedData.dGreenYellow; - dYellow = parsedData.dYellow; - dOrange = parsedData.dOrange; - dOrangeRed = parsedData.dOrangeRed; - dRed = parsedData.dRed; - valueLimits = parsedData.valueLimits; - } - // done waiting - have results - dFuture.return(); - }); - // wait for future to finish - dFuture.wait(); - } - - return { - data: d, - dataPurple: dPurple, - dataPurpleBlue: dPurpleBlue, - dataBlue: dBlue, - dataBlueGreen: dBlueGreen, - dataGreen: dGreen, - dataGreenYellow: dGreenYellow, - dataYellow: dYellow, - dataOrange: dOrange, - dataOrangeRed: dOrangeRed, - dataRed: dRed, - valueLimits, - error, - }; - } -}; - -// this method queries the database for contour plots -const queryDBContour = function (pool, statement, appParams, statisticStr) { - if (Meteor.isServer) { - let d = { - // d will contain the curve data - x: [], - y: [], - z: [], - n: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subSquareDiffSum: [], - subNSum: [], - subObsModelDiffSum: [], - subModelSum: [], - subObsSum: [], - subAbsSum: [], - subData: [], - subHeaders: [], - subVals: [], - subSecs: [], - subLevs: [], - text: [], - xTextOutput: [], - yTextOutput: [], - zTextOutput: [], - nTextOutput: [], - hitTextOutput: [], - faTextOutput: [], - missTextOutput: [], - cnTextOutput: [], - squareDiffSumTextOutput: [], - NSumTextOutput: [], - obsModelDiffSumTextOutput: [], - modelSumTextOutput: [], - obsSumTextOutput: [], - absSumTextOutput: [], - minDateTextOutput: [], - maxDateTextOutput: [], - stdev: [], - stats: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], - glob_stats: {}, - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - zmin: Number.MAX_VALUE, - zmax: Number.MIN_VALUE, - sum: 0, - }; - let error = ""; - let parsedData; - const Future = require("fibers/future"); - const dFuture = new Future(); - - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - /* - we have to call the couchbase utilities as async functions but this - routine 'queryDBContour' cannot itself be async because the graph page needs to wait - for its result, so we use an anonymous async() function here to wrap the queryCB call - */ - (async () => { - const rows = await pool.queryCB(statement); - if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else if (rows.includes("queryCB ERROR: ")) { - error = rows; - } else { - parsedData = parseQueryDataContour(rows, d, appParams, statisticStr); - d = parsedData.d; - } - dFuture.return(); - })(); - } else { - // if this app isn't couchbase, use mysql - pool.query(statement, function (err, rows) { - // query callback - build the curve data from the results - or set an error - if (err !== undefined && err !== null) { - error = err.message; - } else if (rows === undefined || rows === null || rows.length === 0) { - error = matsTypes.Messages.NO_DATA_FOUND; - } else { - parsedData = parseQueryDataContour(rows, d, appParams, statisticStr); - d = parsedData.d; - } - // done waiting - have results - dFuture.return(); - }); - } - // wait for the future to finish - sort of like 'back to the future' ;) - dFuture.wait(); - - return { - data: d, - error, - }; - } -}; - -// this method parses the returned query data for xy curves -const parseQueryDataXYCurve = function ( - rows, - d, - appParams, - statisticStr, - forecastOffset, - cycles, - regular -) { - /* - var d = { // d will contain the curve data - x: [], - y: [], - error_x: [], - error_y: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subSquareDiffSum: [], - subNSum: [], - subObsModelDiffSum: [], - subModelSum: [], - subObsSum: [], - subAbsSum: [], - subData: [], - subHeaders: [], - subVals: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - glob_stats: {}, - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - sum: 0 - }; - */ - const { plotType } = appParams; - const { hasLevels } = appParams; - const completenessQCParam = Number(appParams.completeness) / 100; - const outlierQCParam = - appParams.outliers !== "all" ? Number(appParams.outliers) : appParams.outliers; - - let isCTC = false; - let isScalar = false; - const { hideGaps } = appParams; - - // initialize local variables - const N0 = []; - const N_times = []; - const curveIndependentVars = []; - const curveStats = []; - const subHit = []; - const subFa = []; - const subMiss = []; - const subCn = []; - const subSquareDiffSum = []; - const subNSum = []; - const subObsModelDiffSum = []; - const subModelSum = []; - const subObsSum = []; - const subAbsSum = []; - const subVals = []; - const subSecs = []; - const subLevs = []; - let time_interval; - - for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) { - var independentVar; - switch (plotType) { - case matsTypes.PlotTypes.validtime: - independentVar = Number(rows[rowIndex].hr_of_day); - break; - case matsTypes.PlotTypes.gridscale: - independentVar = Number(rows[rowIndex].gridscale); - break; - case matsTypes.PlotTypes.gridscaleProb: - independentVar = Number(rows[rowIndex].binValue); - break; - case matsTypes.PlotTypes.profile: - independentVar = Number(rows[rowIndex].avVal.toString().replace("P", "")); - break; - case matsTypes.PlotTypes.timeSeries: - // default the time interval to an hour. It won't matter since it won't be used unless there's more than one data point. - time_interval = - rows.length > 1 ? Number(rows[1].avtime) - Number(rows[0].avtime) : 3600; - independentVar = Number(rows[rowIndex].avtime) * 1000; - break; - case matsTypes.PlotTypes.dailyModelCycle: - independentVar = Number(rows[rowIndex].avtime) * 1000; - break; - case matsTypes.PlotTypes.dieoff: - independentVar = Number(rows[rowIndex].fcst_lead); - break; - case matsTypes.PlotTypes.threshold: - independentVar = Number(rows[rowIndex].thresh); - break; - default: - independentVar = Number(rows[rowIndex].avtime); - } - var stat; - if (rows[rowIndex].stat === undefined && rows[rowIndex].hit !== undefined) { - // this is a contingency table plot - isCTC = true; - const hit = Number(rows[rowIndex].hit); - const fa = Number(rows[rowIndex].fa); - const miss = Number(rows[rowIndex].miss); - const cn = Number(rows[rowIndex].cn); - const n = rows[rowIndex].sub_data.toString().split(",").length; - if (hit + fa + miss + cn > 0) { - stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = isNaN(Number(stat)) ? null : stat; - } else { - stat = null; - } - } else if ( - rows[rowIndex].stat === undefined && - rows[rowIndex].square_diff_sum !== undefined - ) { - // this is a scalar partial sums plot - isScalar = true; - const squareDiffSum = Number(rows[rowIndex].square_diff_sum); - const NSum = Number(rows[rowIndex].N_sum); - const obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); - const modelSum = Number(rows[rowIndex].model_sum); - const obsSum = Number(rows[rowIndex].obs_sum); - const absSum = Number(rows[rowIndex].abs_sum); - if (NSum > 0) { - stat = matsDataUtils.calculateStatScalar( - squareDiffSum, - NSum, - obsModelDiffSum, - modelSum, - obsSum, - absSum, - statisticStr - ); - stat = isNaN(Number(stat)) ? null : stat; - } else { - stat = null; - } - } else { - // not a contingency table plot or a scalar partial sums plot - stat = rows[rowIndex].stat === "NULL" ? null : rows[rowIndex].stat; - } - N0.push(rows[rowIndex].N0); // number of values that go into a point on the graph - N_times.push(rows[rowIndex].N_times); // number of times that go into a point on the graph - - if (plotType === matsTypes.PlotTypes.timeSeries) { - // Find the minimum time_interval to be sure we don't accidentally go past the next data point. - if (rowIndex < rows.length - 1) { - const time_diff = - Number(rows[rowIndex + 1].avtime) - Number(rows[rowIndex].avtime); - if (time_diff < time_interval) { - time_interval = time_diff; - } - } - } - - // store sub values that will later be used for calculating error bar statistics. - let sub_hit = []; - let sub_fa = []; - let sub_miss = []; - let sub_cn = []; - let sub_square_diff_sum = []; - let sub_N_sum = []; - let sub_obs_model_diff_sum = []; - let sub_model_sum = []; - let sub_obs_sum = []; - let sub_abs_sum = []; - let sub_values = []; - let sub_secs = []; - let sub_levs = []; - let sub_stdev = 0; - let sub_mean = 0; - let sd_limit = 0; - if ( - stat !== null && - rows[rowIndex].sub_data !== undefined && - rows[rowIndex].sub_data !== null - ) { - // parse the sub-data - try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - var curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - if (isCTC) { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_hit.push(Number(curr_sub_data[2])); - sub_fa.push(Number(curr_sub_data[3])); - sub_miss.push(Number(curr_sub_data[4])); - sub_cn.push(Number(curr_sub_data[5])); - sub_values.push( - matsDataUtils.calculateStatCTC( - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - curr_sub_data.length, - statisticStr - ) - ); - } else { - sub_hit.push(Number(curr_sub_data[1])); - sub_fa.push(Number(curr_sub_data[2])); - sub_miss.push(Number(curr_sub_data[3])); - sub_cn.push(Number(curr_sub_data[4])); - sub_values.push( - matsDataUtils.calculateStatCTC( - Number(curr_sub_data[1]), - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - curr_sub_data.length, - statisticStr - ) - ); - } - } else if (isScalar) { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_square_diff_sum.push(Number(curr_sub_data[2])); - sub_N_sum.push(Number(curr_sub_data[3])); - sub_obs_model_diff_sum.push(Number(curr_sub_data[4])); - sub_model_sum.push(Number(curr_sub_data[5])); - sub_obs_sum.push(Number(curr_sub_data[6])); - sub_abs_sum.push(Number(curr_sub_data[7])); - sub_values.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - Number(curr_sub_data[7]), - statisticStr - ) - ); - } else { - sub_square_diff_sum.push(Number(curr_sub_data[1])); - sub_N_sum.push(Number(curr_sub_data[2])); - sub_obs_model_diff_sum.push(Number(curr_sub_data[3])); - sub_model_sum.push(Number(curr_sub_data[4])); - sub_obs_sum.push(Number(curr_sub_data[5])); - sub_abs_sum.push(Number(curr_sub_data[6])); - sub_values.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[1]), - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - statisticStr - ) - ); - } - } else { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_values.push(Number(curr_sub_data[2])); - } else { - sub_values.push(Number(curr_sub_data[1])); - } - } - } - // Now that we have all the sub-values, we can get the standard deviation and remove the ones that exceed it - if (outlierQCParam !== "all") { - sub_stdev = matsDataUtils.stdev(sub_values); - sub_mean = matsDataUtils.average(sub_values); - sd_limit = outlierQCParam * sub_stdev; - for (let svIdx = sub_values.length - 1; svIdx >= 0; svIdx--) { - if (Math.abs(sub_values[svIdx] - sub_mean) > sd_limit) { - if (isCTC) { - sub_hit.splice(svIdx, 1); - sub_fa.splice(svIdx, 1); - sub_miss.splice(svIdx, 1); - sub_cn.splice(svIdx, 1); - } else if (isScalar) { - sub_square_diff_sum.splice(svIdx, 1); - sub_N_sum.splice(svIdx, 1); - sub_obs_model_diff_sum.splice(svIdx, 1); - sub_model_sum.splice(svIdx, 1); - sub_obs_sum.splice(svIdx, 1); - sub_abs_sum.splice(svIdx, 1); - } - sub_values.splice(svIdx, 1); - sub_secs.splice(svIdx, 1); - if (hasLevels) { - sub_levs.splice(svIdx, 1); - } - } - } - } - if (isCTC) { - const hit = matsDataUtils.sum(sub_hit); - const fa = matsDataUtils.sum(sub_fa); - const miss = matsDataUtils.sum(sub_miss); - const cn = matsDataUtils.sum(sub_cn); - stat = matsDataUtils.calculateStatCTC( - hit, - fa, - miss, - cn, - sub_hit.length, - statisticStr - ); - } else if (isScalar) { - const squareDiffSum = matsDataUtils.sum(sub_square_diff_sum); - const NSum = matsDataUtils.sum(sub_N_sum); - const obsModelDiffSum = matsDataUtils.sum(sub_obs_model_diff_sum); - const modelSum = matsDataUtils.sum(sub_model_sum); - const obsSum = matsDataUtils.sum(sub_obs_sum); - const absSum = matsDataUtils.sum(sub_abs_sum); - stat = matsDataUtils.calculateStatScalar( - squareDiffSum, - NSum, - obsModelDiffSum, - modelSum, - obsSum, - absSum, - statisticStr - ); - } else if (statisticStr.toLowerCase().includes("count")) { - stat = matsDataUtils.sum(sub_values); - } else { - stat = matsDataUtils.average(sub_values); - } - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataXYCurve. The expected fields don't seem to be present in the results cache: ${e.message}`; - throw new Error(e.message); - } - } else { - if (isCTC) { - sub_hit = NaN; - sub_fa = NaN; - sub_miss = NaN; - sub_cn = NaN; - } else if (isScalar) { - sub_square_diff_sum = NaN; - sub_N_sum = NaN; - sub_obs_model_diff_sum = NaN; - sub_model_sum = NaN; - sub_obs_sum = NaN; - sub_abs_sum = NaN; - sub_values = NaN; - } - sub_values = NaN; - sub_secs = NaN; - if (hasLevels) { - sub_levs = NaN; - } - } - - // deal with missing forecast cycles for dailyModelCycle plot type - if ( - plotType === matsTypes.PlotTypes.dailyModelCycle && - rowIndex > 0 && - Number(independentVar) - Number(rows[rowIndex - 1].avtime * 1000) > - 3600 * 24 * 1000 && - !hideGaps - ) { - const cycles_missing = - Math.ceil( - (Number(independentVar) - Number(rows[rowIndex - 1].avtime * 1000)) / - (3600 * 24 * 1000) - ) - 1; - const offsetFromMidnight = Math.floor( - (Number(independentVar) % (24 * 3600 * 1000)) / (3600 * 1000) - ); - for (var missingIdx = cycles_missing; missingIdx > 0; missingIdx--) { - curveIndependentVars.push( - independentVar - - 3600 * 24 * 1000 * missingIdx - - 3600 * offsetFromMidnight * 1000 - ); - curveStats.push(null); - if (isCTC) { - subHit.push(NaN); - subFa.push(NaN); - subMiss.push(NaN); - subCn.push(NaN); - subSquareDiffSum.push([]); - subNSum.push([]); - subObsModelDiffSum.push([]); - subModelSum.push([]); - subObsSum.push([]); - subAbsSum.push([]); - } else if (isScalar) { - subHit.push([]); - subFa.push([]); - subMiss.push([]); - subCn.push([]); - subSquareDiffSum.push(NaN); - subNSum.push(NaN); - subObsModelDiffSum.push(NaN); - subModelSum.push(NaN); - subObsSum.push(NaN); - subAbsSum.push(NaN); - } else { - subHit.push([]); - subFa.push([]); - subMiss.push([]); - subCn.push([]); - subSquareDiffSum.push([]); - subNSum.push([]); - subObsModelDiffSum.push([]); - subModelSum.push([]); - subObsSum.push([]); - subAbsSum.push([]); - } - subVals.push(NaN); - subSecs.push(NaN); - if (hasLevels) { - subLevs.push(NaN); - } - } - } - curveIndependentVars.push(independentVar); - curveStats.push(stat); - subHit.push(sub_hit); - subFa.push(sub_fa); - subMiss.push(sub_miss); - subCn.push(sub_cn); - subSquareDiffSum.push(sub_square_diff_sum); - subNSum.push(sub_N_sum); - subObsModelDiffSum.push(sub_obs_model_diff_sum); - subModelSum.push(sub_model_sum); - subObsSum.push(sub_obs_sum); - subAbsSum.push(sub_abs_sum); - subVals.push(sub_values); - subSecs.push(sub_secs); - if (hasLevels) { - subLevs.push(sub_levs); - } - } - - const N0_max = Math.max(...N0); - const N_times_max = Math.max(...N_times); - let sum = 0; - let indVarMin = Number.MAX_VALUE; - let indVarMax = -1 * Number.MAX_VALUE; - let depVarMin = Number.MAX_VALUE; - let depVarMax = -1 * Number.MAX_VALUE; - let d_idx; - - for (d_idx = 0; d_idx < curveIndependentVars.length; d_idx++) { - const this_N0 = N0[d_idx]; - const this_N_times = N_times[d_idx]; - // Make sure that we don't have any points with a smaller completeness value than specified by the user. - if ( - curveStats[d_idx] === null || - this_N_times < completenessQCParam * N_times_max - ) { - if (!hideGaps) { - if (plotType === matsTypes.PlotTypes.profile) { - // profile has the stat first, and then the independent var. The others have independent var and then stat. - // this is in the pattern of x-plotted-variable, y-plotted-variable. - d.x.push(null); - d.y.push(curveIndependentVars[d_idx]); - d.error_x.push(null); // placeholder - } else { - d.x.push(curveIndependentVars[d_idx]); - d.y.push(null); - d.error_y.push(null); // placeholder - } - d.subHit.push(NaN); - d.subFa.push(NaN); - d.subMiss.push(NaN); - d.subCn.push(NaN); - d.subSquareDiffSum.push(NaN); - d.subNSum.push(NaN); - d.subObsModelDiffSum.push(NaN); - d.subModelSum.push(NaN); - d.subObsSum.push(NaN); - d.subAbsSum.push(NaN); - d.subVals.push(NaN); - d.subSecs.push(NaN); - if (hasLevels) { - d.subLevs.push(NaN); - } - } - } else { - // there's valid data at this point, so store it - sum += curveStats[d_idx]; - if (plotType === matsTypes.PlotTypes.profile) { - // profile has the stat first, and then the independent var. The others have independent var and then stat. - // this is in the pattern of x-plotted-variable, y-plotted-variable. - d.x.push(curveStats[d_idx]); - d.y.push(curveIndependentVars[d_idx]); - d.error_x.push(null); // placeholder - } else { - d.x.push(curveIndependentVars[d_idx]); - d.y.push(curveStats[d_idx]); - d.error_y.push(null); // placeholder - } - d.subHit.push(subHit[d_idx]); - d.subFa.push(subFa[d_idx]); - d.subMiss.push(subMiss[d_idx]); - d.subCn.push(subCn[d_idx]); - d.subSquareDiffSum.push(subSquareDiffSum[d_idx]); - d.subNSum.push(subNSum[d_idx]); - d.subObsModelDiffSum.push(subObsModelDiffSum[d_idx]); - d.subModelSum.push(subModelSum[d_idx]); - d.subObsSum.push(subObsSum[d_idx]); - d.subAbsSum.push(subAbsSum[d_idx]); - d.subVals.push(subVals[d_idx]); - d.subSecs.push(subSecs[d_idx]); - if (hasLevels) { - d.subLevs.push(subLevs[d_idx]); - } - indVarMin = - curveIndependentVars[d_idx] < indVarMin - ? curveIndependentVars[d_idx] - : indVarMin; - indVarMax = - curveIndependentVars[d_idx] > indVarMax - ? curveIndependentVars[d_idx] - : indVarMax; - depVarMin = curveStats[d_idx] < depVarMin ? curveStats[d_idx] : depVarMin; - depVarMax = curveStats[d_idx] > depVarMax ? curveStats[d_idx] : depVarMax; - } - } - - // add in any missing times in the time series - if (plotType === matsTypes.PlotTypes.timeSeries && !hideGaps) { - time_interval *= 1000; - const dayInMilliSeconds = 24 * 3600 * 1000; - let lowerIndependentVar; - let upperIndependentVar; - let newTime; - let thisCadence; - let numberOfDaysBack; - for (d_idx = curveIndependentVars.length - 2; d_idx >= 0; d_idx--) { - lowerIndependentVar = curveIndependentVars[d_idx]; - upperIndependentVar = curveIndependentVars[d_idx + 1]; - const cycles_missing = - Math.ceil( - (Number(upperIndependentVar) - Number(lowerIndependentVar)) / time_interval - ) - 1; - for (missingIdx = cycles_missing; missingIdx > 0; missingIdx--) { - newTime = lowerIndependentVar + missingIdx * time_interval; - if (!regular) { - // if it's not a regular model, we only want to add a null point if this is an init time that should have had a forecast. - thisCadence = newTime % dayInMilliSeconds; // current hour of day (valid time) - if (Number(thisCadence) - Number(forecastOffset) * 3600 * 1000 < 0) { - // check to see if cycle time was on a previous day -- if so, need to wrap around 00Z to get current hour of day (cycle time) - numberOfDaysBack = Math.ceil( - (-1 * (Number(thisCadence) - Number(forecastOffset) * 3600 * 1000)) / - dayInMilliSeconds - ); - thisCadence = - Number(thisCadence) - - Number(forecastOffset) * 3600 * 1000 + - numberOfDaysBack * dayInMilliSeconds; // current hour of day (cycle time) - } else { - thisCadence = Number(thisCadence) - Number(forecastOffset) * 3600 * 1000; // current hour of day (cycle time) - } - if (cycles.indexOf(thisCadence) !== -1) { - matsDataUtils.addNullPoint( - d, - d_idx + 1, - matsTypes.PlotTypes.timeSeries, - "x", - newTime, - "y", - isCTC, - isScalar, - hasLevels - ); - } - } else { - matsDataUtils.addNullPoint( - d, - d_idx + 1, - matsTypes.PlotTypes.timeSeries, - "x", - newTime, - "y", - isCTC, - isScalar, - hasLevels - ); - } - } - } - } - - if (plotType === matsTypes.PlotTypes.profile) { - d.xmin = depVarMin; - d.xmax = depVarMax; - d.ymin = indVarMin; - d.ymax = indVarMax; - } else { - d.xmin = indVarMin; - d.xmax = indVarMax; - d.ymin = depVarMin; - d.ymax = depVarMax; - } - - d.sum = sum; - - return { - d, - N0, - N_times, - }; -}; - -// this method parses the returned query data for performance diagrams -const parseQueryDataReliability = function (rows, d, appParams, kernel) { /* - let d = { - // d will contain the curve data - x: [], - y: [], - binVals: [], - hitCount: [], - fcstCount: [], - fcstRawCount: [], - sample_climo: 0, - n: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subData: [], - subHeaders: [], - subRelHit: [], - subRelRawCount: [], - subRelCount: [], - subVals: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - n_forecast: [], - n_matched: [], - n_simple: [], - n_total: [], - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - }; + var d = { // d will contain the curve data + x: [], + y: [], + binVals: [], + subSquareDiffSumX: [], + subNSumX: [], + subObsModelDiffSumX: [], + subModelSumX: [], + subObsSumX: [], + subAbsSumX: [], + subValsX: [], + subSquareDiffSumY: [], + subNSumY: [], + subObsModelDiffSumY: [], + subModelSumY: [], + subObsSumY: [], + subAbsSumY: [], + subValsY: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + sum: 0 + }; */ + const returnD = d; const { hasLevels } = appParams; // initialize local variables + const n0 = []; + const nTimes = []; + const xStats = []; + const yStats = []; const binVals = []; - const hitCounts = []; - const fcstCounts = []; - const fcstRawCounts = []; - const observedFreqs = []; - let totalForecastCount = 0; - const subRelCount = []; - const subRelRawCount = []; - const subRelHit = []; - const subVals = []; + const subSquareDiffSumX = []; + const subNSumX = []; + const subObsModelDiffSumX = []; + const subModelSumX = []; + const subObsSumX = []; + const subAbsSumX = []; + const subSquareDiffSumY = []; + const subNSumY = []; + const subObsModelDiffSumY = []; + const subModelSumY = []; + const subObsSumY = []; + const subAbsSumY = []; + const subValsX = []; + const subValsY = []; const subSecs = []; const subLevs = []; for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + const binVal = Number(rows[rowIndex].binVal); + let xStat; + let yStat; if ( - Number(rows[rowIndex].kernel) === 0 && - rows[rowIndex].rawfcstcount !== undefined && - rows[rowIndex].rawfcstcount !== "NULL" - ) { - totalForecastCount += Number(rows[rowIndex].rawfcstcount); - const fcstRawCount = - rows[rowIndex].rawfcstcount === "NULL" - ? null - : Number(rows[rowIndex].rawfcstcount); - let sub_raw_counts = []; // actually raw counts but I'm re-using fields - // parse the sub-data - if (rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null) { - try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - let curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - if (hasLevels) { - sub_raw_counts.push(Number(curr_sub_data[3])); - } else { - sub_raw_counts.push(Number(curr_sub_data[2])); - } - } - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataReliability. The expected fields don't seem to be present in the results cache: ${e.message}`; - throw new Error(e.message); - } + rows[rowIndex].square_diff_sumX !== undefined && + rows[rowIndex].square_diff_sumY !== undefined + ) { + // this is a scalar partial sums plot + const squareDiffSumX = Number(rows[rowIndex].square_diff_sumX); + const NSumX = Number(rows[rowIndex].N_sumX); + const obsModelDiffSumX = Number(rows[rowIndex].obs_model_diff_sumX); + const modelSumX = Number(rows[rowIndex].model_sumX); + const obsSumX = Number(rows[rowIndex].obs_sumX); + const absSumX = Number(rows[rowIndex].abs_sumX); + const squareDiffSumY = Number(rows[rowIndex].square_diff_sumY); + const NSumY = Number(rows[rowIndex].N_sumY); + const obsModelDiffSumY = Number(rows[rowIndex].obs_model_diff_sumY); + const modelSumY = Number(rows[rowIndex].model_sumY); + const obsSumY = Number(rows[rowIndex].obs_sumY); + const absSumY = Number(rows[rowIndex].abs_sumY); + if (NSumX > 0 && NSumY > 0) { + xStat = matsDataUtils.calculateStatScalar( + squareDiffSumX, + NSumX, + obsModelDiffSumX, + modelSumX, + obsSumX, + absSumX, + statisticXStr + ); + xStat = Number.isNaN(Number(xStat)) ? null : xStat; + yStat = matsDataUtils.calculateStatScalar( + squareDiffSumY, + NSumY, + obsModelDiffSumY, + modelSumY, + obsSumY, + absSumY, + statisticYStr + ); + yStat = Number.isNaN(Number(yStat)) ? null : yStat; } else { - sub_raw_counts = NaN; + xStat = null; + yStat = null; } - subRelRawCount.push(sub_raw_counts); } - if (Number(rows[rowIndex].kernel) === Number(kernel)) { - const binVal = Number(rows[rowIndex].binValue); - let hitCount; - let fcstCount; - let observedFreq; - if ( - rows[rowIndex].fcstcount !== undefined && - rows[rowIndex].hitcount !== undefined - ) { - hitCount = - rows[rowIndex].hitcount === "NULL" ? null : Number(rows[rowIndex].hitcount); - fcstCount = - rows[rowIndex].fcstcount === "NULL" ? null : Number(rows[rowIndex].fcstcount); - observedFreq = hitCount / fcstCount; - } else { - hitCount = null; - fcstCount = null; - } - binVals.push(binVal); - hitCounts.push(hitCount); - fcstCounts.push(fcstCount); - observedFreqs.push(observedFreq); + n0.push(rows[rowIndex].n0); // number of values that go into a point on the graph + nTimes.push(rows[rowIndex].nTimes); // number of times that go into a point on the graph + binVals.push(binVal); - let sub_rel_hit = []; - let sub_rel_counts = []; - const sub_values = []; - let sub_secs = []; - let sub_levs = []; - if ( - hitCount !== null && - rows[rowIndex].sub_data !== undefined && - rows[rowIndex].sub_data !== null - ) { - // parse the sub-data - try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - let curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_rel_counts.push(Number(curr_sub_data[2])); - sub_rel_hit.push(Number(curr_sub_data[4])); - // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. - sub_values.push(0); + let thisSubSquareDiffSumX = []; + let thisSubNSumX = []; + let thisSubObsModelDiffSumX = []; + let thisSubModelSumX = []; + let thisSubObsSumX = []; + let thisSubAbsSumX = []; + let thisSubValuesX = []; + let thisSubSquareDiffSumY = []; + let thisSubNSumY = []; + let thisSubObsModelDiffSumY = []; + let thisSubModelSumY = []; + let thisSubObsSumY = []; + let thisSubAbsSumY = []; + let thisSubValuesY = []; + let thisSubSecs = []; + let thisSubLevs = []; + if ( + xStat !== null && + yStat !== null && + rows[rowIndex].sub_data !== undefined && + rows[rowIndex].sub_data !== null + ) { + // parse the sub-data + try { + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); } else { - sub_rel_counts.push(Number(curr_sub_data[1])); - sub_rel_hit.push(Number(curr_sub_data[3])); - // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. - sub_values.push(0); + thisSubLevs.push(currSubData[1]); } + thisSubSquareDiffSumX.push(Number(currSubData[2])); + thisSubNSumX.push(Number(currSubData[3])); + thisSubObsModelDiffSumX.push(Number(currSubData[4])); + thisSubModelSumX.push(Number(currSubData[5])); + thisSubObsSumX.push(Number(currSubData[6])); + thisSubAbsSumX.push(Number(currSubData[7])); + thisSubValuesX.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + Number(currSubData[7]), + statisticXStr + ) + ); + thisSubSquareDiffSumY.push(Number(currSubData[8])); + thisSubNSumY.push(Number(currSubData[9])); + thisSubObsModelDiffSumY.push(Number(currSubData[10])); + thisSubModelSumY.push(Number(currSubData[11])); + thisSubObsSumY.push(Number(currSubData[12])); + thisSubAbsSumY.push(Number(currSubData[13])); + thisSubValuesY.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[8]), + Number(currSubData[9]), + Number(currSubData[10]), + Number(currSubData[11]), + Number(currSubData[12]), + Number(currSubData[13]), + statisticYStr + ) + ); + } else { + thisSubSquareDiffSumX.push(Number(currSubData[1])); + thisSubNSumX.push(Number(currSubData[2])); + thisSubObsModelDiffSumX.push(Number(currSubData[3])); + thisSubModelSumX.push(Number(currSubData[4])); + thisSubObsSumX.push(Number(currSubData[5])); + thisSubAbsSumX.push(Number(currSubData[6])); + thisSubValuesX.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[1]), + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + statisticXStr + ) + ); + thisSubSquareDiffSumY.push(Number(currSubData[7])); + thisSubNSumY.push(Number(currSubData[8])); + thisSubObsModelDiffSumY.push(Number(currSubData[9])); + thisSubModelSumY.push(Number(currSubData[10])); + thisSubObsSumY.push(Number(currSubData[11])); + thisSubAbsSumY.push(Number(currSubData[12])); + thisSubValuesY.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[7]), + Number(currSubData[8]), + Number(currSubData[9]), + Number(currSubData[10]), + Number(currSubData[11]), + Number(currSubData[12]), + statisticXStr + ) + ); } - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataReliability. The expected fields don't seem to be present in the results cache: ${e.message}`; - throw new Error(e.message); - } - } else { - sub_rel_counts = NaN; - sub_rel_hit = NaN; - sub_secs = NaN; - if (hasLevels) { - sub_levs = NaN; } + const squareDiffSumX = matsDataUtils.sum(thisSubSquareDiffSumX); + const NSumX = matsDataUtils.sum(thisSubNSumX); + const obsModelDiffSumX = matsDataUtils.sum(thisSubObsModelDiffSumX); + const modelSumX = matsDataUtils.sum(thisSubModelSumX); + const obsSumX = matsDataUtils.sum(thisSubObsSumX); + const absSumX = matsDataUtils.sum(thisSubAbsSumX); + xStat = matsDataUtils.calculateStatScalar( + squareDiffSumX, + NSumX, + obsModelDiffSumX, + modelSumX, + obsSumX, + absSumX, + statisticXStr + ); + const squareDiffSumY = matsDataUtils.sum(thisSubSquareDiffSumY); + const NSumY = matsDataUtils.sum(thisSubNSumY); + const obsModelDiffSumY = matsDataUtils.sum(thisSubObsModelDiffSumY); + const modelSumY = matsDataUtils.sum(thisSubModelSumY); + const obsSumY = matsDataUtils.sum(thisSubObsSumY); + const absSumY = matsDataUtils.sum(thisSubAbsSumY); + yStat = matsDataUtils.calculateStatScalar( + squareDiffSumY, + NSumY, + obsModelDiffSumY, + modelSumY, + obsSumY, + absSumY, + statisticYStr + ); + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in parseQueryDataXYCurve. The expected fields don't seem to be present in the results cache: ${e.message}`; + throw new Error(e.message); } - subRelCount.push(sub_rel_counts); - subRelHit.push(sub_rel_hit); - subVals.push(sub_values); - subSecs.push(sub_secs); + } else { + thisSubSquareDiffSumX = NaN; + thisSubNSumX = NaN; + thisSubObsModelDiffSumX = NaN; + thisSubModelSumX = NaN; + thisSubObsSumX = NaN; + thisSubAbsSumX = NaN; + thisSubValuesX = NaN; + thisSubSquareDiffSumY = NaN; + thisSubNSumY = NaN; + thisSubObsModelDiffSumY = NaN; + thisSubModelSumY = NaN; + thisSubObsSumY = NaN; + thisSubAbsSumY = NaN; + thisSubValuesY = NaN; + thisSubSecs = NaN; if (hasLevels) { - subLevs.push(sub_levs); + thisSubLevs = NaN; } } - } - const sampleClimo = matsDataUtils.sum(hitCounts) / totalForecastCount; + xStats.push(xStat); + yStats.push(yStat); + subSquareDiffSumX.push(thisSubSquareDiffSumX); + subNSumX.push(thisSubNSumX); + subObsModelDiffSumX.push(thisSubObsModelDiffSumX); + subModelSumX.push(thisSubModelSumX); + subObsSumX.push(thisSubObsSumX); + subAbsSumX.push(thisSubAbsSumX); + subValsX.push(thisSubValuesX); + subSquareDiffSumY.push(thisSubSquareDiffSumY); + subNSumY.push(thisSubNSumY); + subObsModelDiffSumY.push(thisSubObsModelDiffSumY); + subModelSumY.push(thisSubModelSumY); + subObsSumY.push(thisSubObsSumY); + subAbsSumY.push(thisSubAbsSumY); + subValsY.push(thisSubValuesY); + subSecs.push(thisSubSecs); + if (hasLevels) { + subLevs.push(thisSubLevs); + } + } - d.x = - binVals[binVals.length - 1] === 100 - ? binVals.map((bin) => bin / 100) - : binVals.map((bin) => bin / 10); - d.y = observedFreqs; - d.binVals = binVals; - d.hitCount = hitCounts; - d.fcstCount = fcstCounts; - d.fcstRawCount = fcstRawCounts; - d.sample_climo = sampleClimo; - d.subRelHit = subRelHit; - d.subRelCount = subRelCount; - d.subRelRawCount = subRelRawCount; - d.subVals = subVals; - d.subSecs = subSecs; - d.subLevs = subLevs; + returnD.x = xStats; + returnD.y = yStats; + returnD.binVals = binVals; + returnD.subSquareDiffSumX = subSquareDiffSumX; + returnD.subNSumX = subNSumX; + returnD.subObsModelDiffSumX = subObsModelDiffSumX; + returnD.subModelSumX = subModelSumX; + returnD.subObsSumX = subObsSumX; + returnD.subAbsSumX = subAbsSumX; + returnD.subValsX = subValsX; + returnD.subSquareDiffSumY = subSquareDiffSumY; + returnD.subNSumY = subNSumY; + returnD.subObsModelDiffSumY = subObsModelDiffSumY; + returnD.subModelSumY = subModelSumY; + returnD.subObsSumY = subObsSumY; + returnD.subAbsSumY = subAbsSumY; + returnD.subValsY = subValsY; + returnD.subSecs = subSecs; + returnD.subLevs = subLevs; + returnD.n = n0; - let xMin = Number.MAX_VALUE; - let xMax = -1 * Number.MAX_VALUE; - let yMin = Number.MAX_VALUE; - let yMax = -1 * Number.MAX_VALUE; + let xmin = Number.MAX_VALUE; + let xmax = -1 * Number.MAX_VALUE; + let ymin = Number.MAX_VALUE; + let ymax = -1 * Number.MAX_VALUE; - for (let didx = 0; didx < binVals.length; didx += 1) { - xMin = d.x[didx] !== null && d.x[didx] < xMin ? d.x[didx] : xMin; - xMax = d.x[didx] !== null && d.x[didx] > xMax ? d.x[didx] : xMax; - yMin = d.y[didx] !== null && d.y[didx] < yMin ? d.y[didx] : yMin; - yMax = d.y[didx] !== null && d.y[didx] > yMax ? d.y[didx] : yMax; + for (let dIdx = 0; dIdx < binVals.length; dIdx += 1) { + xmin = xStats[dIdx] !== null && xStats[dIdx] < xmin ? xStats[dIdx] : xmin; + xmax = xStats[dIdx] !== null && xStats[dIdx] > xmax ? xStats[dIdx] : xmax; + ymin = yStats[dIdx] !== null && yStats[dIdx] < ymin ? yStats[dIdx] : ymin; + ymax = yStats[dIdx] !== null && yStats[dIdx] > ymax ? yStats[dIdx] : ymax; } - d.xmin = xMin; - d.xmax = xMax; - d.ymin = yMin; - d.ymax = yMax; + returnD.xmin = xmin; + returnD.xmax = xmax; + returnD.ymin = ymin; + returnD.ymax = ymax; + return { - d, + d: returnD, + n0, + nTimes, }; }; -// this method parses the returned query data for performance diagrams -const parseQueryDataPerformanceDiagram = function (rows, d, appParams) { - /* - var d = { // d will contain the curve data - x: [], - y: [], - binVals: [], - oy_all: [], - on_all: [], - n: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subData: [], - subHeaders: [], - subVals: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - }; - */ +// this method parses the returned query data for maps +const parseQueryDataMapScalar = function ( + rows, + d, + dLowest, + dLow, + dModerate, + dHigh, + dHighest, + dataSource, + siteMap, + statistic, + variable, + varUnits, + appParams, + isCouchbase +) { + const returnD = d; + const returnDLowest = dLowest; + const returnDLow = dLow; + const returnDModerate = dModerate; + const returnDHigh = dHigh; + const returnDHighest = dHighest; const { hasLevels } = appParams; + let highLimit = 10; + let lowLimit = -10; + const outlierQCParam = + appParams.outliers !== "all" ? Number(appParams.outliers) : appParams.outliers; - // initialize local variables - const N0 = []; - const N_times = []; - const successes = []; - const pods = []; - const binVals = []; - const oy_all = []; - const on_all = []; - const subHit = []; - const subFa = []; - const subMiss = []; - const subCn = []; - const subVals = []; - const subSecs = []; - const subLevs = []; - for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) { - const binVal = Number(rows[rowIndex].binVal); - var pod; - var success; - var oy; - var on; - if (rows[rowIndex].pod !== undefined && rows[rowIndex].far !== undefined) { - pod = rows[rowIndex].pod === "NULL" ? null : Number(rows[rowIndex].pod); - success = rows[rowIndex].far === "NULL" ? null : 1 - Number(rows[rowIndex].far); - oy = rows[rowIndex].oy === "NULL" ? null : Number(rows[rowIndex].oy_all); - on = rows[rowIndex].on === "NULL" ? null : Number(rows[rowIndex].on_all); + // determine which colormap will be used for this plot + let colorLowest = ""; + let colorLow = ""; + let colorModerate = ""; + let colorHigh = ""; + let colorHighest = ""; + if (statistic.includes("Bias")) { + if ( + variable.toLowerCase().includes("rh") || + variable.toLowerCase().includes("relative humidity") || + variable.toLowerCase().includes("dewpoint") || + variable.toLowerCase().includes("dpt") || + variable.toLowerCase().includes("td") + ) { + colorLowest = "rgb(140,81,00)"; + colorLow = "rgb(191,129,45)"; + colorModerate = "rgb(125,125,125)"; + colorHigh = "rgb(53,151,143)"; + colorHighest = "rgb(1,102,95)"; + } else if (variable.toLowerCase().includes("temp")) { + colorLowest = "rgb(24,28,247)"; + colorLow = "rgb(67,147,195)"; + colorModerate = "rgb(125,125,125)"; + colorHigh = "rgb(255,120,86)"; + colorHighest = "rgb(216,21,47)"; + } else { + colorLowest = "rgb(0,134,0)"; + colorLow = "rgb(80,255,80)"; + colorModerate = "rgb(125,125,125)"; + colorHigh = "rgb(255,80,255)"; + colorHighest = "rgb(134,0,134)"; + } + } else { + colorLowest = "rgb(125,125,125)"; + colorLow = "rgb(196,179,139)"; + colorModerate = "rgb(243,164,96)"; + colorHigh = "rgb(210,105,30)"; + colorHighest = "rgb(237,0,0)"; + } + returnDLowest.color = colorLowest; + returnDLow.color = colorLow; + returnDModerate.color = colorModerate; + returnDHigh.color = colorHigh; + returnDHighest.color = colorHighest; + + let queryVal; + for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + const site = rows[rowIndex].sta_id; + const squareDiffSum = Number(rows[rowIndex].square_diff_sum); + const NSum = Number(rows[rowIndex].N_sum); + const obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); + const modelSum = Number(rows[rowIndex].model_sum); + const obsSum = Number(rows[rowIndex].obs_sum); + const absSum = Number(rows[rowIndex].abs_sum); + if (NSum > 0) { + queryVal = matsDataUtils.calculateStatScalar( + squareDiffSum, + NSum, + obsModelDiffSum, + modelSum, + obsSum, + absSum, + `${statistic}_${variable}` + ); + queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; } else { - pod = null; - success = null; - oy = null; - on = null; + queryVal = null; } - N0.push(rows[rowIndex].N0); // number of values that go into a point on the graph - N_times.push(rows[rowIndex].N_times); // number of times that go into a point on the graph - successes.push(success); - pods.push(pod); - binVals.push(binVal); - oy_all.push(oy); - on_all.push(on); - - let sub_hit = []; - let sub_fa = []; - let sub_miss = []; - let sub_cn = []; - const sub_values = []; - let sub_secs = []; - let sub_levs = []; + // store sub values to test them for stdev. + const thisSubSquareDiffSum = []; + const thisSubNSum = []; + const thisSubObsModelDiffSum = []; + const thisSubModelSum = []; + const thisSubObsSum = []; + const thisSubAbsSum = []; + const thisSubValues = []; + const thisSubSecs = []; + const thisSubLevs = []; + let thisSubStdev = 0; + let thisSubMean = 0; + let sdLimit = 0; if ( - pod !== null && + queryVal !== null && rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null ) { // parse the sub-data try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - var curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - sub_secs.push(Number(curr_sub_data[0])); + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); } else { - sub_levs.push(curr_sub_data[1]); + thisSubLevs.push(currSubData[1]); } - sub_hit.push(Number(curr_sub_data[2])); - sub_fa.push(Number(curr_sub_data[3])); - sub_miss.push(Number(curr_sub_data[4])); - sub_cn.push(Number(curr_sub_data[5])); - // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. - sub_values.push(0); + thisSubSquareDiffSum.push(Number(currSubData[2])); + thisSubNSum.push(Number(currSubData[3])); + thisSubObsModelDiffSum.push(Number(currSubData[4])); + thisSubModelSum.push(Number(currSubData[5])); + thisSubObsSum.push(Number(currSubData[6])); + thisSubAbsSum.push(Number(currSubData[7])); + thisSubValues.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + Number(currSubData[7]), + `${statistic}_${variable}` + ) + ); } else { - sub_hit.push(Number(curr_sub_data[1])); - sub_fa.push(Number(curr_sub_data[2])); - sub_miss.push(Number(curr_sub_data[3])); - sub_cn.push(Number(curr_sub_data[4])); - // this is a dummy to fit the expectations of common functions that xy line curves have a populated sub_values array. It isn't used for anything. - sub_values.push(0); + thisSubSquareDiffSum.push(Number(currSubData[1])); + thisSubNSum.push(Number(currSubData[2])); + thisSubObsModelDiffSum.push(Number(currSubData[3])); + thisSubModelSum.push(Number(currSubData[4])); + thisSubObsSum.push(Number(currSubData[5])); + thisSubAbsSum.push(Number(currSubData[6])); + thisSubValues.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[1]), + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + `${statistic}_${variable}` + ) + ); + } + } + // Now that we have all the sub-values, we can get the standard deviation and remove the ones that exceed it + if (outlierQCParam !== "all") { + thisSubStdev = matsDataUtils.stdev(thisSubValues); + thisSubMean = matsDataUtils.average(thisSubValues); + sdLimit = outlierQCParam * thisSubStdev; + for (let svIdx = thisSubValues.length - 1; svIdx >= 0; svIdx -= 1) { + if (Math.abs(thisSubValues[svIdx] - thisSubMean) > sdLimit) { + thisSubSquareDiffSum.splice(svIdx, 1); + thisSubNSum.splice(svIdx, 1); + thisSubObsModelDiffSum.splice(svIdx, 1); + thisSubModelSum.splice(svIdx, 1); + thisSubObsSum.splice(svIdx, 1); + thisSubAbsSum.splice(svIdx, 1); + thisSubValues.splice(svIdx, 1); + thisSubSecs.splice(svIdx, 1); + if (hasLevels) { + thisSubLevs.splice(svIdx, 1); + } + } } } + const squareDiffSumSum = matsDataUtils.sum(thisSubSquareDiffSum); + const NSumSum = matsDataUtils.sum(thisSubNSum); + const obsModelDiffSumSum = matsDataUtils.sum(thisSubObsModelDiffSum); + const modelSumSum = matsDataUtils.sum(thisSubModelSum); + const obsSumSum = matsDataUtils.sum(thisSubObsSum); + const absSumSum = matsDataUtils.sum(thisSubAbsSum); + queryVal = matsDataUtils.calculateStatScalar( + squareDiffSumSum, + NSumSum, + obsModelDiffSumSum, + modelSumSum, + obsSumSum, + absSumSum, + `${statistic}_${variable}` + ); + queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataPerformanceDiagram. The expected fields don't seem to be present in the results cache: ${e.message}`; + e.message = `Error in parseQueryDataMapScalar. The expected fields don't seem to be present in the results cache: ${e.message}`; throw new Error(e.message); } - } else { - sub_hit = NaN; - sub_fa = NaN; - sub_miss = NaN; - sub_cn = NaN; - sub_secs = NaN; - if (hasLevels) { - sub_levs = NaN; + returnD.queryVal.push(queryVal); + returnD.stats.push({ + nTimes: rows[rowIndex].nTimes, + min_time: rows[rowIndex].min_secs, + max_time: rows[rowIndex].max_secs, + }); + + let thisSite; + if (isCouchbase) { + thisSite = siteMap.find((obj) => obj.name === site); + } else { + thisSite = siteMap.find((obj) => obj.options.id === site); } - } - subHit.push(sub_hit); - subFa.push(sub_fa); - subMiss.push(sub_miss); - subCn.push(sub_cn); - subVals.push(sub_values); - subSecs.push(sub_secs); - if (hasLevels) { - subLevs.push(sub_levs); + + const tooltips = + `${thisSite.origName}
${variable} ${statistic}
` + + `model: ${dataSource}
` + + `stat: ${queryVal} ${varUnits}
` + + `n: ${rows[rowIndex].n0}`; + returnD.text.push(tooltips); + returnD.siteName.push(thisSite.origName); + returnD.lat.push(thisSite.point[0]); + returnD.lon.push(thisSite.point[1]); + returnD.color.push("rgb(125,125,125)"); // dummy } } - d.x = successes; - d.y = pods; - d.binVals = binVals; - d.oy_all = oy_all; - d.on_all = on_all; - d.subHit = subHit; - d.subFa = subFa; - d.subMiss = subMiss; - d.subCn = subCn; - d.subVals = subVals; - d.subSecs = subSecs; - d.subLevs = subLevs; - d.n = N0; + // get range of values for colorscale, eliminating the highest and lowest as outliers + let filteredValues = returnD.queryVal.filter((x) => x || x === 0); + filteredValues = filteredValues.sort(function (a, b) { + return Number(a) - Number(b); + }); + highLimit = filteredValues[Math.floor(filteredValues.length * 0.98)]; + lowLimit = filteredValues[Math.floor(filteredValues.length * 0.02)]; - let successMin = Number.MAX_VALUE; - let successMax = -1 * Number.MAX_VALUE; - let podMin = Number.MAX_VALUE; - let podMax = -1 * Number.MAX_VALUE; + const maxValue = + Math.abs(highLimit) > Math.abs(lowLimit) ? Math.abs(highLimit) : Math.abs(lowLimit); + if (statistic === "Bias (Model - Obs)") { + // bias colorscale needs to be symmetrical around 0 + highLimit = maxValue; + lowLimit = -1 * maxValue; + } - for (let d_idx = 0; d_idx < binVals.length; d_idx++) { - successMin = - successes[d_idx] !== null && successes[d_idx] < successMin - ? successes[d_idx] - : successMin; - successMax = - successes[d_idx] !== null && successes[d_idx] > successMax - ? successes[d_idx] - : successMax; - podMin = podMin[d_idx] !== null && pods[d_idx] < podMin ? pods[d_idx] : podMin; - podMax = podMin[d_idx] !== null && pods[d_idx] > podMax ? pods[d_idx] : podMax; + // get stdev threshold at which to exclude entire points + const allMean = matsDataUtils.average(filteredValues); + const allStdev = matsDataUtils.stdev(filteredValues); + let allSdLimit; + if (outlierQCParam !== "all") { + allSdLimit = outlierQCParam * allStdev; } - d.xmin = successMin; - d.xmax = successMax; - d.ymin = podMin; - d.ymax = podMax; + for (let didx = returnD.queryVal.length - 1; didx >= 0; didx -= 1) { + queryVal = returnD.queryVal[didx]; + if (!(outlierQCParam !== "all" && Math.abs(queryVal - allMean) > allSdLimit)) { + // this point is too far from the mean. Exclude it. + returnD.queryVal.splice(didx, 1); + returnD.stats.splice(didx, 1); + returnD.text.splice(didx, 1); + returnD.siteName.splice(didx, 1); + returnD.lat.splice(didx, 1); + returnD.lon.splice(didx, 1); + returnD.color.splice(didx, 1); + let textMarker; + if (variable.includes("2m") || variable.includes("10m")) { + textMarker = queryVal === null ? "" : queryVal.toFixed(0); + } else { + textMarker = queryVal === null ? "" : queryVal.toFixed(1); + } + // sort the data by the color it will appear on the map + if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.2) { + returnD.color[didx] = colorLowest; + returnDLowest.siteName.push(returnD.siteName[didx]); + returnDLowest.queryVal.push(queryVal); + returnDLowest.text.push(textMarker); + returnDLowest.lat.push(returnD.lat[didx]); + returnDLowest.lon.push(returnD.lon[didx]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.4) { + returnD.color[didx] = colorLow; + returnDLow.siteName.push(returnD.siteName[didx]); + returnDLow.queryVal.push(queryVal); + returnDLow.text.push(textMarker); + returnDLow.lat.push(returnD.lat[didx]); + returnDLow.lon.push(returnD.lon[didx]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.6) { + returnD.color[didx] = colorModerate; + returnDModerate.siteName.push(returnD.siteName[didx]); + returnDModerate.queryVal.push(queryVal); + returnDModerate.text.push(textMarker); + returnDModerate.lat.push(returnD.lat[didx]); + returnDModerate.lon.push(returnD.lon[didx]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.8) { + returnD.color[didx] = colorHigh; + returnDHigh.siteName.push(returnD.siteName[didx]); + returnDHigh.queryVal.push(queryVal); + returnDHigh.text.push(textMarker); + returnDHigh.lat.push(returnD.lat[didx]); + returnDHigh.lon.push(returnD.lon[didx]); + } else { + returnD.color[didx] = colorHighest; + returnDHighest.siteName.push(returnD.siteName[didx]); + returnDHighest.queryVal.push(queryVal); + returnDHighest.text.push(textMarker); + returnDHighest.lat.push(returnD.lat[didx]); + returnDHighest.lon.push(returnD.lon[didx]); + } + } + } // end of loop row return { - d, - N0, - N_times, + d: returnD, + dLowest: returnDLowest, + dLow: returnDLow, + dModerate: returnDModerate, + dHigh: returnDHigh, + dHighest: returnDHighest, + valueLimits: { + maxValue, + highLimit, + lowLimit, + }, }; }; -// this method parses the returned query data for simple scatter plots -const parseQueryDataSimpleScatter = function ( +// this method parses the returned query data for maps in CTC apps +const parseQueryDataMapCTC = function ( rows, d, + dPurple, + dPurpleBlue, + dBlue, + dBlueGreen, + dGreen, + dGreenYellow, + dYellow, + dOrange, + dOrangeRed, + dRed, + dataSource, + siteMap, + statistic, appParams, - statisticXStr, - statisticYStr + isCouchbase ) { - /* - var d = { // d will contain the curve data - x: [], - y: [], - binVals: [], - subSquareDiffSumX: [], - subNSumX: [], - subObsModelDiffSumX: [], - subModelSumX: [], - subObsSumX: [], - subAbsSumX: [], - subValsX: [], - subSquareDiffSumY: [], - subNSumY: [], - subObsModelDiffSumY: [], - subModelSumY: [], - subObsSumY: [], - subAbsSumY: [], - subValsY: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - sum: 0 - }; - */ + const returnD = d; + const returnDPurple = dPurple; + const returnDPurpleBlue = dPurpleBlue; + const returnDBlue = dBlue; + const returnDBlueGreen = dBlueGreen; + const returnDGreen = dGreen; + const returnDGreenYellow = dGreenYellow; + const returnDYellow = dYellow; + const returnDOrange = dOrange; + const returnDOrangeRed = dOrangeRed; + const returnDRed = dRed; const { hasLevels } = appParams; + let highLimit = 100; + let lowLimit = -100; + const outlierQCParam = + appParams.outliers !== "all" ? Number(appParams.outliers) : appParams.outliers; - // initialize local variables - const N0 = []; - const N_times = []; - const xStats = []; - const yStats = []; - const binVals = []; - const subSquareDiffSumX = []; - const subNSumX = []; - const subObsModelDiffSumX = []; - const subModelSumX = []; - const subObsSumX = []; - const subAbsSumX = []; - const subSquareDiffSumY = []; - const subNSumY = []; - const subObsModelDiffSumY = []; - const subModelSumY = []; - const subObsSumY = []; - const subAbsSumY = []; - const subValsX = []; - const subValsY = []; - const subSecs = []; - const subLevs = []; - for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) { - const binVal = Number(rows[rowIndex].binVal); - var xStat; - var yStat; - if ( - rows[rowIndex].square_diff_sumX !== undefined && - rows[rowIndex].square_diff_sumY !== undefined - ) { - // this is a scalar partial sums plot - const squareDiffSumX = Number(rows[rowIndex].square_diff_sumX); - const NSumX = Number(rows[rowIndex].N_sumX); - const obsModelDiffSumX = Number(rows[rowIndex].obs_model_diff_sumX); - const modelSumX = Number(rows[rowIndex].model_sumX); - const obsSumX = Number(rows[rowIndex].obs_sumX); - const absSumX = Number(rows[rowIndex].abs_sumX); - const squareDiffSumY = Number(rows[rowIndex].square_diff_sumY); - const NSumY = Number(rows[rowIndex].N_sumY); - const obsModelDiffSumY = Number(rows[rowIndex].obs_model_diff_sumY); - const modelSumY = Number(rows[rowIndex].model_sumY); - const obsSumY = Number(rows[rowIndex].obs_sumY); - const absSumY = Number(rows[rowIndex].abs_sumY); - if (NSumX > 0 && NSumY > 0) { - xStat = matsDataUtils.calculateStatScalar( - squareDiffSumX, - NSumX, - obsModelDiffSumX, - modelSumX, - obsSumX, - absSumX, - statisticXStr - ); - xStat = isNaN(Number(xStat)) ? null : xStat; - yStat = matsDataUtils.calculateStatScalar( - squareDiffSumY, - NSumY, - obsModelDiffSumY, - modelSumY, - obsSumY, - absSumY, - statisticYStr - ); - yStat = isNaN(Number(yStat)) ? null : yStat; - } else { - xStat = null; - yStat = null; + let queryVal; + for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + const site = rows[rowIndex].sta_id; + const hit = Number(rows[rowIndex].hit); + const fa = Number(rows[rowIndex].fa); + const miss = Number(rows[rowIndex].miss); + const cn = Number(rows[rowIndex].cn); + const n = rows[rowIndex].nTimes; + if (hit + fa + miss + cn > 0) { + queryVal = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statistic); + queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + switch (statistic) { + case "PODy (POD of value < threshold)": + case "PODy (POD of value > threshold)": + case "PODn (POD of value > threshold)": + case "PODn (POD of value < threshold)": + case "FAR (False Alarm Ratio)": + case "CSI (Critical Success Index)": + lowLimit = 0; + highLimit = 100; + break; + case "Bias (forecast/actual)": + lowLimit = 0; + highLimit = 2; + break; + case "ETS (Equitable Threat Score)": + lowLimit = -100 / 3; + highLimit = 100; + break; + case "Ratio Nlow / Ntot ((hit + miss)/(hit + miss + fa + cn))": + case "Ratio Nhigh / Ntot ((hit + miss)/(hit + miss + fa + cn))": + case "Ratio Nlow / Ntot ((fa + cn)/(hit + miss + fa + cn))": + case "Ratio Nhigh / Ntot ((fa + cn)/(hit + miss + fa + cn))": + lowLimit = 0; + highLimit = 1; + break; + case "TSS (True Skill Score)": + case "HSS (Heidke Skill Score)": + default: + lowLimit = -100; + highLimit = 100; + break; } + } else { + queryVal = null; } - N0.push(rows[rowIndex].N0); // number of values that go into a point on the graph - N_times.push(rows[rowIndex].N_times); // number of times that go into a point on the graph - binVals.push(binVal); - let sub_square_diff_sumX = []; - let sub_N_sumX = []; - let sub_obs_model_diff_sumX = []; - let sub_model_sumX = []; - let sub_obs_sumX = []; - let sub_abs_sumX = []; - let sub_valuesX = []; - let sub_square_diff_sumY = []; - let sub_N_sumY = []; - let sub_obs_model_diff_sumY = []; - let sub_model_sumY = []; - let sub_obs_sumY = []; - let sub_abs_sumY = []; - let sub_valuesY = []; - let sub_secs = []; - let sub_levs = []; + // store sub values to test them for stdev. + const thisSubHit = []; + const thisSubFa = []; + const thisSubMiss = []; + const thisSubCn = []; + const thisSubValues = []; + const thisSubSecs = []; + const thisSubLevs = []; + let thisSubStdev = 0; + let thisSubMean = 0; + let sdLimit = 0; if ( - xStat !== null && - yStat !== null && + queryVal !== null && rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null ) { // parse the sub-data try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - var curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - sub_secs.push(Number(curr_sub_data[0])); + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); } else { - sub_levs.push(curr_sub_data[1]); + thisSubLevs.push(currSubData[1]); } - sub_square_diff_sumX.push(Number(curr_sub_data[2])); - sub_N_sumX.push(Number(curr_sub_data[3])); - sub_obs_model_diff_sumX.push(Number(curr_sub_data[4])); - sub_model_sumX.push(Number(curr_sub_data[5])); - sub_obs_sumX.push(Number(curr_sub_data[6])); - sub_abs_sumX.push(Number(curr_sub_data[7])); - sub_valuesX.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - Number(curr_sub_data[7]), - statisticXStr - ) - ); - sub_square_diff_sumY.push(Number(curr_sub_data[8])); - sub_N_sumY.push(Number(curr_sub_data[9])); - sub_obs_model_diff_sumY.push(Number(curr_sub_data[10])); - sub_model_sumY.push(Number(curr_sub_data[11])); - sub_obs_sumY.push(Number(curr_sub_data[12])); - sub_abs_sumY.push(Number(curr_sub_data[13])); - sub_valuesY.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[8]), - Number(curr_sub_data[9]), - Number(curr_sub_data[10]), - Number(curr_sub_data[11]), - Number(curr_sub_data[12]), - Number(curr_sub_data[13]), - statisticYStr + thisSubHit.push(Number(currSubData[2])); + thisSubFa.push(Number(currSubData[3])); + thisSubMiss.push(Number(currSubData[4])); + thisSubCn.push(Number(currSubData[5])); + thisSubValues.push( + matsDataUtils.calculateStatCTC( + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + currSubData.length, + statistic ) ); } else { - sub_square_diff_sumX.push(Number(curr_sub_data[1])); - sub_N_sumX.push(Number(curr_sub_data[2])); - sub_obs_model_diff_sumX.push(Number(curr_sub_data[3])); - sub_model_sumX.push(Number(curr_sub_data[4])); - sub_obs_sumX.push(Number(curr_sub_data[5])); - sub_abs_sumX.push(Number(curr_sub_data[6])); - sub_valuesX.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[1]), - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - statisticXStr - ) - ); - sub_square_diff_sumY.push(Number(curr_sub_data[7])); - sub_N_sumY.push(Number(curr_sub_data[8])); - sub_obs_model_diff_sumY.push(Number(curr_sub_data[9])); - sub_model_sumY.push(Number(curr_sub_data[10])); - sub_obs_sumY.push(Number(curr_sub_data[11])); - sub_abs_sumY.push(Number(curr_sub_data[12])); - sub_valuesY.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[7]), - Number(curr_sub_data[8]), - Number(curr_sub_data[9]), - Number(curr_sub_data[10]), - Number(curr_sub_data[11]), - Number(curr_sub_data[12]), - statisticXStr + thisSubHit.push(Number(currSubData[1])); + thisSubFa.push(Number(currSubData[2])); + thisSubMiss.push(Number(currSubData[3])); + thisSubCn.push(Number(currSubData[4])); + thisSubValues.push( + matsDataUtils.calculateStatCTC( + Number(currSubData[1]), + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + currSubData.length, + statistic ) ); } } - const squareDiffSumX = matsDataUtils.sum(sub_square_diff_sumX); - const NSumX = matsDataUtils.sum(sub_N_sumX); - const obsModelDiffSumX = matsDataUtils.sum(sub_obs_model_diff_sumX); - const modelSumX = matsDataUtils.sum(sub_model_sumX); - const obsSumX = matsDataUtils.sum(sub_obs_sumX); - const absSumX = matsDataUtils.sum(sub_abs_sumX); - xStat = matsDataUtils.calculateStatScalar( - squareDiffSumX, - NSumX, - obsModelDiffSumX, - modelSumX, - obsSumX, - absSumX, - statisticXStr - ); - const squareDiffSumY = matsDataUtils.sum(sub_square_diff_sumY); - const NSumY = matsDataUtils.sum(sub_N_sumY); - const obsModelDiffSumY = matsDataUtils.sum(sub_obs_model_diff_sumY); - const modelSumY = matsDataUtils.sum(sub_model_sumY); - const obsSumY = matsDataUtils.sum(sub_obs_sumY); - const absSumY = matsDataUtils.sum(sub_abs_sumY); - yStat = matsDataUtils.calculateStatScalar( - squareDiffSumY, - NSumY, - obsModelDiffSumY, - modelSumY, - obsSumY, - absSumY, - statisticYStr + // Now that we have all the sub-values, we can get the standard deviation and remove the ones that exceed it + if (outlierQCParam !== "all") { + thisSubStdev = matsDataUtils.stdev(thisSubValues); + thisSubMean = matsDataUtils.average(thisSubValues); + sdLimit = outlierQCParam * thisSubStdev; + for (let svIdx = thisSubValues.length - 1; svIdx >= 0; svIdx -= 1) { + if (Math.abs(thisSubValues[svIdx] - thisSubMean) > sdLimit) { + thisSubHit.splice(svIdx, 1); + thisSubFa.splice(svIdx, 1); + thisSubMiss.splice(svIdx, 1); + thisSubCn.splice(svIdx, 1); + thisSubValues.splice(svIdx, 1); + thisSubSecs.splice(svIdx, 1); + if (hasLevels) { + thisSubLevs.splice(svIdx, 1); + } + } + } + } + const hitSum = matsDataUtils.sum(thisSubHit); + const faSum = matsDataUtils.sum(thisSubFa); + const missSum = matsDataUtils.sum(thisSubMiss); + const cnSum = matsDataUtils.sum(thisSubCn); + queryVal = matsDataUtils.calculateStatCTC( + hitSum, + faSum, + missSum, + cnSum, + thisSubHit.length, + statistic ); + queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataXYCurve. The expected fields don't seem to be present in the results cache: ${e.message}`; + e.message = `Error in parseQueryDataMapCTC. The expected fields don't seem to be present in the results cache: ${e.message}`; throw new Error(e.message); } - } else { - sub_square_diff_sumX = NaN; - sub_N_sumX = NaN; - sub_obs_model_diff_sumX = NaN; - sub_model_sumX = NaN; - sub_obs_sumX = NaN; - sub_abs_sumX = NaN; - sub_valuesX = NaN; - sub_square_diff_sumY = NaN; - sub_N_sumY = NaN; - sub_obs_model_diff_sumY = NaN; - sub_model_sumY = NaN; - sub_obs_sumY = NaN; - sub_abs_sumY = NaN; - sub_valuesY = NaN; - sub_secs = NaN; - if (hasLevels) { - sub_levs = NaN; - } - } - - xStats.push(xStat); - yStats.push(yStat); - subSquareDiffSumX.push(sub_square_diff_sumX); - subNSumX.push(sub_N_sumX); - subObsModelDiffSumX.push(sub_obs_model_diff_sumX); - subModelSumX.push(sub_model_sumX); - subObsSumX.push(sub_obs_sumX); - subAbsSumX.push(sub_abs_sumX); - subValsX.push(sub_valuesX); - subSquareDiffSumY.push(sub_square_diff_sumY); - subNSumY.push(sub_N_sumY); - subObsModelDiffSumY.push(sub_obs_model_diff_sumY); - subModelSumY.push(sub_model_sumY); - subObsSumY.push(sub_obs_sumY); - subAbsSumY.push(sub_abs_sumY); - subValsY.push(sub_valuesY); - subSecs.push(sub_secs); - if (hasLevels) { - subLevs.push(sub_levs); - } - } - - d.x = xStats; - d.y = yStats; - d.binVals = binVals; - d.subSquareDiffSumX = subSquareDiffSumX; - d.subNSumX = subNSumX; - d.subObsModelDiffSumX = subObsModelDiffSumX; - d.subModelSumX = subModelSumX; - d.subObsSumX = subObsSumX; - d.subAbsSumX = subAbsSumX; - d.subValsX = subValsX; - d.subSquareDiffSumY = subSquareDiffSumY; - d.subNSumY = subNSumY; - d.subObsModelDiffSumY = subObsModelDiffSumY; - d.subModelSumY = subModelSumY; - d.subObsSumY = subObsSumY; - d.subAbsSumY = subAbsSumY; - d.subValsY = subValsY; - d.subSecs = subSecs; - d.subLevs = subLevs; - d.n = N0; + returnD.queryVal.push(queryVal); + returnD.stats.push({ + nTimes: rows[rowIndex].nTimes, + min_time: rows[rowIndex].min_secs, + max_time: rows[rowIndex].max_secs, + hit: rows[rowIndex].hit, + fa: rows[rowIndex].fa, + miss: rows[rowIndex].miss, + cn: rows[rowIndex].cn, + }); - let xmin = Number.MAX_VALUE; - let xmax = -1 * Number.MAX_VALUE; - let ymin = Number.MAX_VALUE; - let ymax = -1 * Number.MAX_VALUE; + let thisSite; + if (isCouchbase) { + thisSite = siteMap.find((obj) => obj.name === site); + } else { + thisSite = siteMap.find((obj) => obj.options.id === site); + } - for (let d_idx = 0; d_idx < binVals.length; d_idx++) { - xmin = xStats[d_idx] !== null && xStats[d_idx] < xmin ? xStats[d_idx] : xmin; - xmax = xStats[d_idx] !== null && xStats[d_idx] > xmax ? xStats[d_idx] : xmax; - ymin = yStats[d_idx] !== null && yStats[d_idx] < ymin ? yStats[d_idx] : ymin; - ymax = yStats[d_idx] !== null && yStats[d_idx] > ymax ? yStats[d_idx] : ymax; - } + const tooltips = + `${thisSite.origName}
` + + `model: ${dataSource}
${statistic}: ${queryVal}
` + + `n: ${rows[rowIndex].nTimes}
` + + `hits: ${rows[rowIndex].hit}
` + + `false alarms: ${rows[rowIndex].fa}
` + + `misses: ${rows[rowIndex].miss}
` + + `correct nulls: ${rows[rowIndex].cn}`; + returnD.text.push(tooltips); + returnD.siteName.push(thisSite.origName); + returnD.lat.push(thisSite.point[0]); + returnD.lon.push(thisSite.point[1]); - d.xmin = xmin; - d.xmax = xmax; - d.ymin = ymin; - d.ymax = ymax; + // sort the data by the color it will appear on the map + const textMarker = queryVal === null ? "" : queryVal.toFixed(0); + if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.1) { + returnD.color.push("rgb(128,0,255)"); + returnDPurple.siteName.push(thisSite.origName); + returnDPurple.queryVal.push(queryVal); + returnDPurple.text.push(textMarker); + returnDPurple.lat.push(thisSite.point[0]); + returnDPurple.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.2) { + returnD.color.push("rgb(64,0,255)"); + returnDPurpleBlue.siteName.push(thisSite.origName); + returnDPurpleBlue.queryVal.push(queryVal); + returnDPurpleBlue.text.push(textMarker); + returnDPurpleBlue.lat.push(thisSite.point[0]); + returnDPurpleBlue.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.3) { + returnD.color.push("rgb(0,0,255)"); + returnDBlue.siteName.push(thisSite.origName); + returnDBlue.queryVal.push(queryVal); + returnDBlue.text.push(textMarker); + returnDBlue.lat.push(thisSite.point[0]); + returnDBlue.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.4) { + returnD.color.push("rgb(64,128,128)"); + returnDBlueGreen.siteName.push(thisSite.origName); + returnDBlueGreen.queryVal.push(queryVal); + returnDBlueGreen.text.push(textMarker); + returnDBlueGreen.lat.push(thisSite.point[0]); + returnDBlueGreen.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.5) { + returnD.color.push("rgb(128,255,0)"); + returnDGreen.siteName.push(thisSite.origName); + returnDGreen.queryVal.push(queryVal); + returnDGreen.text.push(textMarker); + returnDGreen.lat.push(thisSite.point[0]); + returnDGreen.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.6) { + returnD.color.push("rgb(160,224,0)"); + returnDGreenYellow.siteName.push(thisSite.origName); + returnDGreenYellow.queryVal.push(queryVal); + returnDGreenYellow.text.push(textMarker); + returnDGreenYellow.lat.push(thisSite.point[0]); + returnDGreenYellow.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.7) { + returnD.color.push("rgb(192,192,0)"); + returnDYellow.siteName.push(thisSite.origName); + returnDYellow.queryVal.push(queryVal); + returnDYellow.text.push(textMarker); + returnDYellow.lat.push(thisSite.point[0]); + returnDYellow.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.8) { + returnD.color.push("rgb(255,128,0)"); + returnDOrange.siteName.push(thisSite.origName); + returnDOrange.queryVal.push(queryVal); + returnDOrange.text.push(textMarker); + returnDOrange.lat.push(thisSite.point[0]); + returnDOrange.lon.push(thisSite.point[1]); + } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.9) { + returnD.color.push("rgb(255,64,0)"); + returnDOrangeRed.siteName.push(thisSite.origName); + returnDOrangeRed.queryVal.push(queryVal); + returnDOrangeRed.text.push(textMarker); + returnDOrangeRed.lat.push(thisSite.point[0]); + returnDOrangeRed.lon.push(thisSite.point[1]); + } else { + returnD.color.push("rgb(255,0,0)"); + returnDRed.siteName.push(thisSite.origName); + returnDRed.queryVal.push(queryVal); + returnDRed.text.push(textMarker); + returnDRed.lat.push(thisSite.point[0]); + returnDRed.lon.push(thisSite.point[1]); + } + } + } // end of loop row return { - d, - N0, - N_times, + d: returnD, + dPurple: returnDPurple, + dPurpleBlue: returnDPurpleBlue, + dBlue: returnDBlue, + dBlueGreen: returnDBlue, + dGreen: returnDGreen, + dGreenYellow: returnDGreenYellow, + dYellow: returnDYellow, + dOrange: returnDOrange, + dOrangeRed: returnDOrangeRed, + dRed: returnDRed, + valueLimits: { + maxValue: highLimit, + highLimit, + lowLimit, + }, }; }; -// this method parses the returned query data for maps -const parseQueryDataMapScalar = function ( - rows, - d, - dLowest, - dLow, - dModerate, - dHigh, - dHighest, - dataSource, - siteMap, - statistic, - variable, - varUnits, - appParams, - isCouchbase -) { +// this method parses the returned query data for histograms +const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { + /* + var d = { // d will contain the curve data + x: [], + y: [], + error_x: [], + error_y: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subSquareDiffSum: [], + subNSum: [], + subObsModelDiffSum: [], + subModelSum: [], + subObsSum: [], + subAbsSum: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + glob_stats: {}, + bin_stats: [], + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + sum: 0 + }; + */ + const returnD = d; const { hasLevels } = appParams; - let highLimit = 10; - let lowLimit = -10; - const outlierQCParam = - appParams.outliers !== "all" ? Number(appParams.outliers) : appParams.outliers; + let isCTC = false; + let isScalar = false; - // determine which colormap will be used for this plot - let colorLowest = ""; - let colorLow = ""; - let colorModerate = ""; - let colorHigh = ""; - let colorHighest = ""; - if (statistic.includes("Bias")) { - if ( - variable.toLowerCase().includes("rh") || - variable.toLowerCase().includes("relative humidity") || - variable.toLowerCase().includes("dewpoint") || - variable.toLowerCase().includes("dpt") || - variable.toLowerCase().includes("td") - ) { - colorLowest = "rgb(140,81,00)"; - colorLow = "rgb(191,129,45)"; - colorModerate = "rgb(125,125,125)"; - colorHigh = "rgb(53,151,143)"; - colorHighest = "rgb(1,102,95)"; - } else if (variable.toLowerCase().includes("temp")) { - colorLowest = "rgb(24,28,247)"; - colorLow = "rgb(67,147,195)"; - colorModerate = "rgb(125,125,125)"; - colorHigh = "rgb(255,120,86)"; - colorHighest = "rgb(216,21,47)"; - } else { - colorLowest = "rgb(0,134,0)"; - colorLow = "rgb(80,255,80)"; - colorModerate = "rgb(125,125,125)"; - colorHigh = "rgb(255,80,255)"; - colorHighest = "rgb(134,0,134)"; - } - } else { - colorLowest = "rgb(125,125,125)"; - colorLow = "rgb(196,179,139)"; - colorModerate = "rgb(243,164,96)"; - colorHigh = "rgb(210,105,30)"; - colorHighest = "rgb(237,0,0)"; - } - dLowest.color = colorLowest; - dLow.color = colorLow; - dModerate.color = colorModerate; - dHigh.color = colorHigh; - dHighest.color = colorHighest; + // these arrays hold all the sub values and seconds (and levels) until they are sorted into bins + const curveSubStatsRaw = []; + const curveSubSecsRaw = []; + const curveSubLevsRaw = []; - let queryVal; - for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) { - const site = rows[rowIndex].sta_id; - const squareDiffSum = Number(rows[rowIndex].square_diff_sum); - const NSum = Number(rows[rowIndex].N_sum); - const obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); - const modelSum = Number(rows[rowIndex].model_sum); - const obsSum = Number(rows[rowIndex].obs_sum); - const absSum = Number(rows[rowIndex].abs_sum); - if (NSum > 0) { - queryVal = matsDataUtils.calculateStatScalar( - squareDiffSum, - NSum, - obsModelDiffSum, - modelSum, - obsSum, - absSum, - `${statistic}_${variable}` - ); - queryVal = isNaN(Number(queryVal)) ? null : queryVal; + // parse the data returned from the query + for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + let stat; + if (rows[rowIndex].stat === undefined && rows[rowIndex].hit !== undefined) { + // this is a contingency table plot + isCTC = true; + const hit = Number(rows[rowIndex].hit); + const fa = Number(rows[rowIndex].fa); + const miss = Number(rows[rowIndex].miss); + const cn = Number(rows[rowIndex].cn); + const n = rows[rowIndex].sub_data.toString().split(",").length; + if (hit + fa + miss + cn > 0) { + stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); + stat = Number.isNaN(Number(stat)) ? null : stat; + } else { + stat = null; + } + } else if ( + rows[rowIndex].stat === undefined && + rows[rowIndex].square_diff_sum !== undefined + ) { + // this is a scalar partial sums plot + isScalar = true; + const squareDiffSum = Number(rows[rowIndex].square_diff_sum); + const NSum = Number(rows[rowIndex].N_sum); + const obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); + const modelSum = Number(rows[rowIndex].model_sum); + const obsSum = Number(rows[rowIndex].obs_sum); + const absSum = Number(rows[rowIndex].abs_sum); + if (NSum > 0) { + stat = matsDataUtils.calculateStatScalar( + squareDiffSum, + NSum, + obsModelDiffSum, + modelSum, + obsSum, + absSum, + statisticStr + ); + stat = Number.isNaN(Number(stat)) ? null : stat; + } else { + stat = null; + } } else { - queryVal = null; + // not a contingency table plot or a scalar partial sums plot + stat = rows[rowIndex].stat === "NULL" ? null : rows[rowIndex].stat; } - // store sub values to test them for stdev. - const sub_square_diff_sum = []; - const sub_N_sum = []; - const sub_obs_model_diff_sum = []; - const sub_model_sum = []; - const sub_obs_sum = []; - const sub_abs_sum = []; - const sub_values = []; - const sub_secs = []; - const sub_levs = []; - let sub_stdev = 0; - let sub_mean = 0; - let sd_limit = 0; + const thisSubStats = []; + const thisSubSecs = []; + const thisSubLevs = []; if ( - queryVal !== null && + stat !== null && rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null ) { // parse the sub-data try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - var curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + if (isCTC) { + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubStats.push( + matsDataUtils.calculateStatCTC( + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + 1, + statisticStr + ) + ); } else { - sub_levs.push(curr_sub_data[1]); + thisSubStats.push( + matsDataUtils.calculateStatCTC( + Number(currSubData[1]), + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + 1, + statisticStr + ) + ); + } + } else if (isScalar) { + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubStats.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + Number(currSubData[7]), + statisticStr + ) + ); + } else { + thisSubStats.push( + matsDataUtils.calculateStatScalar( + Number(currSubData[1]), + Number(currSubData[2]), + Number(currSubData[3]), + Number(currSubData[4]), + Number(currSubData[5]), + Number(currSubData[6]), + statisticStr + ) + ); } - sub_square_diff_sum.push(Number(curr_sub_data[2])); - sub_N_sum.push(Number(curr_sub_data[3])); - sub_obs_model_diff_sum.push(Number(curr_sub_data[4])); - sub_model_sum.push(Number(curr_sub_data[5])); - sub_obs_sum.push(Number(curr_sub_data[6])); - sub_abs_sum.push(Number(curr_sub_data[7])); - sub_values.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - Number(curr_sub_data[7]), - `${statistic}_${variable}` - ) - ); } else { - sub_square_diff_sum.push(Number(curr_sub_data[1])); - sub_N_sum.push(Number(curr_sub_data[2])); - sub_obs_model_diff_sum.push(Number(curr_sub_data[3])); - sub_model_sum.push(Number(curr_sub_data[4])); - sub_obs_sum.push(Number(curr_sub_data[5])); - sub_abs_sum.push(Number(curr_sub_data[6])); - sub_values.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[1]), - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - `${statistic}_${variable}` - ) - ); - } - } - // Now that we have all the sub-values, we can get the standard deviation and remove the ones that exceed it - if (outlierQCParam !== "all") { - sub_stdev = matsDataUtils.stdev(sub_values); - sub_mean = matsDataUtils.average(sub_values); - sd_limit = outlierQCParam * sub_stdev; - for (let svIdx = sub_values.length - 1; svIdx >= 0; svIdx--) { - if (Math.abs(sub_values[svIdx] - sub_mean) > sd_limit) { - sub_square_diff_sum.splice(svIdx, 1); - sub_N_sum.splice(svIdx, 1); - sub_obs_model_diff_sum.splice(svIdx, 1); - sub_model_sum.splice(svIdx, 1); - sub_obs_sum.splice(svIdx, 1); - sub_abs_sum.splice(svIdx, 1); - sub_values.splice(svIdx, 1); - sub_secs.splice(svIdx, 1); - if (hasLevels) { - sub_levs.splice(svIdx, 1); + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); } + thisSubStats.push(Number(currSubData[2])); + } else { + thisSubStats.push(Number(currSubData[1])); } } } - const squareDiffSum = matsDataUtils.sum(sub_square_diff_sum); - const NSum = matsDataUtils.sum(sub_N_sum); - const obsModelDiffSum = matsDataUtils.sum(sub_obs_model_diff_sum); - const modelSum = matsDataUtils.sum(sub_model_sum); - const obsSum = matsDataUtils.sum(sub_obs_sum); - const absSum = matsDataUtils.sum(sub_abs_sum); - queryVal = matsDataUtils.calculateStatScalar( - squareDiffSum, - NSum, - obsModelDiffSum, - modelSum, - obsSum, - absSum, - `${statistic}_${variable}` - ); - queryVal = isNaN(Number(queryVal)) ? null : queryVal; + curveSubStatsRaw.push(thisSubStats); + curveSubSecsRaw.push(thisSubSecs); + curveSubLevsRaw.push(thisSubLevs); } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataMapScalar. The expected fields don't seem to be present in the results cache: ${e.message}`; + e.message = `Error in parseQueryDataHistogram. The expected fields don't seem to be present in the results cache: ${e.message}`; throw new Error(e.message); } - d.queryVal.push(queryVal); - d.stats.push({ - N_times: rows[rowIndex].N_times, - min_time: rows[rowIndex].min_secs, - max_time: rows[rowIndex].max_secs, - }); - - let thisSite; - if (isCouchbase) { - thisSite = siteMap.find((obj) => obj.name === site); - } else { - thisSite = siteMap.find((obj) => obj.options.id === site); - } - - const tooltips = - `${thisSite.origName}
${variable} ${statistic}
` + - `model: ${dataSource}
` + - `stat: ${queryVal} ${varUnits}
` + - `n: ${rows[rowIndex].N0}`; - d.text.push(tooltips); - d.siteName.push(thisSite.origName); - d.lat.push(thisSite.point[0]); - d.lon.push(thisSite.point[1]); - d.color.push("rgb(125,125,125)"); // dummy } } - // get range of values for colorscale, eliminating the highest and lowest as outliers - let filteredValues = d.queryVal.filter((x) => x || x === 0); - filteredValues = filteredValues.sort(function (a, b) { - return Number(a) - Number(b); + // we don't have bins yet, so we want all of the data in one array + const subVals = curveSubStatsRaw.reduce(function (a, b) { + return a.concat(b); }); - highLimit = filteredValues[Math.floor(filteredValues.length * 0.98)]; - lowLimit = filteredValues[Math.floor(filteredValues.length * 0.02)]; - - const maxValue = - Math.abs(highLimit) > Math.abs(lowLimit) ? Math.abs(highLimit) : Math.abs(lowLimit); - if (statistic === "Bias (Model - Obs)") { - // bias colorscale needs to be symmetrical around 0 - highLimit = maxValue; - lowLimit = -1 * maxValue; + const subSecs = curveSubSecsRaw.reduce(function (a, b) { + return a.concat(b); + }); + let subLevs; + if (hasLevels) { + subLevs = curveSubLevsRaw.reduce(function (a, b) { + return a.concat(b); + }); } - // get stdev threshold at which to exclude entire points - const all_mean = matsDataUtils.average(filteredValues); - const all_stdev = matsDataUtils.stdev(filteredValues); - let all_sd_limit; - if (outlierQCParam !== "all") { - all_sd_limit = outlierQCParam * all_stdev; - } + returnD.subVals = subVals; + returnD.subSecs = subSecs; + returnD.subLevs = subLevs; - for (let didx = d.queryVal.length - 1; didx >= 0; didx--) { - queryVal = d.queryVal[didx]; - if (outlierQCParam !== "all" && Math.abs(queryVal - all_mean) > all_sd_limit) { - // this point is too far from the mean. Exclude it. - d.queryVal.splice(didx, 1); - d.stats.splice(didx, 1); - d.text.splice(didx, 1); - d.siteName.splice(didx, 1); - d.lat.splice(didx, 1); - d.lon.splice(didx, 1); - d.color.splice(didx, 1); - continue; - } - var textMarker; - if (variable.includes("2m") || variable.includes("10m")) { - textMarker = queryVal === null ? "" : queryVal.toFixed(0); - } else { - textMarker = queryVal === null ? "" : queryVal.toFixed(1); - } - // sort the data by the color it will appear on the map - if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.2) { - d.color[didx] = colorLowest; - dLowest.siteName.push(d.siteName[didx]); - dLowest.queryVal.push(queryVal); - dLowest.text.push(textMarker); - dLowest.lat.push(d.lat[didx]); - dLowest.lon.push(d.lon[didx]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.4) { - d.color[didx] = colorLow; - dLow.siteName.push(d.siteName[didx]); - dLow.queryVal.push(queryVal); - dLow.text.push(textMarker); - dLow.lat.push(d.lat[didx]); - dLow.lon.push(d.lon[didx]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.6) { - d.color[didx] = colorModerate; - dModerate.siteName.push(d.siteName[didx]); - dModerate.queryVal.push(queryVal); - dModerate.text.push(textMarker); - dModerate.lat.push(d.lat[didx]); - dModerate.lon.push(d.lon[didx]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.8) { - d.color[didx] = colorHigh; - dHigh.siteName.push(d.siteName[didx]); - dHigh.queryVal.push(queryVal); - dHigh.text.push(textMarker); - dHigh.lat.push(d.lat[didx]); - dHigh.lon.push(d.lon[didx]); - } else { - d.color[didx] = colorHighest; - dHighest.siteName.push(d.siteName[didx]); - dHighest.queryVal.push(queryVal); - dHighest.text.push(textMarker); - dHighest.lat.push(d.lat[didx]); - dHighest.lon.push(d.lon[didx]); - } - } // end of loop row return { - d, - dLowest, - dLow, - dModerate, - dHigh, - dHighest, - valueLimits: { - maxValue, - highLimit, - lowLimit, - }, + d: returnD, + n0: subVals.length, + nTimes: subSecs.length, }; }; -// this method parses the returned query data for maps in CTC apps -const parseQueryDataMapCTC = function ( - rows, - d, - dPurple, - dPurpleBlue, - dBlue, - dBlueGreen, - dGreen, - dGreenYellow, - dYellow, - dOrange, - dOrangeRed, - dRed, - dataSource, - siteMap, - statistic, - appParams, - isCouchbase -) { +// this method parses the returned query data for contour plots +const parseQueryDataContour = function (rows, d, appParams, statisticStr) { + /* + var d = { // d will contain the curve data + x: [], + y: [], + z: [], + n: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subSquareDiffSum: [], + subNSum: [], + subObsModelDiffSum: [], + subModelSum: [], + subObsSum: [], + subAbsSum: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + text: [], + xTextOutput: [], + yTextOutput: [], + zTextOutput: [], + nTextOutput: [], + hitTextOutput: [], + faTextOutput: [], + missTextOutput: [], + cnTextOutput: [], + squareDiffSumTextOutput: [], + NSumTextOutput: [], + obsModelDiffSumTextOutput: [], + modelSumTextOutput: [], + obsSumTextOutput: [], + absSumTextOutput: [], + minDateTextOutput: [], + maxDateTextOutput: [], + stdev: [], + stats: [], + glob_stats: {}, + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + zmin: Number.MAX_VALUE, + zmax: Number.MIN_VALUE, + sum: 0 + }; + */ + const returnD = d; const { hasLevels } = appParams; - let highLimit = 100; - let lowLimit = -100; - const outlierQCParam = - appParams.outliers !== "all" ? Number(appParams.outliers) : appParams.outliers; + let isCTC = false; + let isScalar = false; - let queryVal; - for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) { - const site = rows[rowIndex].sta_id; - const hit = Number(rows[rowIndex].hit); - const fa = Number(rows[rowIndex].fa); - const miss = Number(rows[rowIndex].miss); - const cn = Number(rows[rowIndex].cn); - const n = rows[rowIndex].N_times; - if (hit + fa + miss + cn > 0) { - queryVal = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statistic); - queryVal = isNaN(Number(queryVal)) ? null : queryVal; - switch (statistic) { - case "TSS (True Skill Score)": - case "HSS (Heidke Skill Score)": - lowLimit = -100; - highLimit = 100; - break; - case "PODy (POD of value < threshold)": - case "PODy (POD of value > threshold)": - case "PODn (POD of value > threshold)": - case "PODn (POD of value < threshold)": - case "FAR (False Alarm Ratio)": - case "CSI (Critical Success Index)": - lowLimit = 0; - highLimit = 100; - break; - case "Bias (forecast/actual)": - lowLimit = 0; - highLimit = 2; - break; - case "ETS (Equitable Threat Score)": - lowLimit = -100 / 3; - highLimit = 100; - break; - case "Ratio Nlow / Ntot ((hit + miss)/(hit + miss + fa + cn))": - case "Ratio Nhigh / Ntot ((hit + miss)/(hit + miss + fa + cn))": - case "Ratio Nlow / Ntot ((fa + cn)/(hit + miss + fa + cn))": - case "Ratio Nhigh / Ntot ((fa + cn)/(hit + miss + fa + cn))": - lowLimit = 0; - highLimit = 1; - break; + // initialize local variables + const curveStatLookup = {}; + const curveStdevLookup = {}; + const curveNLookup = {}; + const curveSubHitLookup = {}; + const curveSubFaLookup = {}; + const curveSubMissLookup = {}; + const curveSubCnLookup = {}; + const curveSubSquareDiffSumLookup = {}; + const curveSubNSumLookup = {}; + const curveSubObsModelDiffSumLookup = {}; + const curveSubModelSumLookup = {}; + const curveSubObsSumLookup = {}; + const curveSubAbsSumLookup = {}; + const curveSubValLookup = {}; + const curveSubSecLookup = {}; + const curveSubLevLookup = {}; + + // get all the data out of the query array + for (let rowIndex = 0; rowIndex < rows.length; rowIndex += 1) { + const rowXVal = Number(rows[rowIndex].xVal); + const rowYVal = Number(rows[rowIndex].yVal); + const statKey = `${rowXVal.toString()}_${rowYVal.toString()}`; + let stat = null; + let hit = null; + let fa = null; + let miss = null; + let cn = null; + let n = + rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null + ? rows[rowIndex].sub_data.toString().split(",").length + : 0; + let squareDiffSum = null; + let NSum = null; + let obsModelDiffSum = null; + let modelSum = null; + let obsSum = null; + let absSum = null; + let stdev = null; + if (rows[rowIndex].stat === undefined && rows[rowIndex].hit !== undefined) { + // this is a contingency table plot + isCTC = true; + hit = Number(rows[rowIndex].hit); + fa = Number(rows[rowIndex].fa); + miss = Number(rows[rowIndex].miss); + cn = Number(rows[rowIndex].cn); + if (hit + fa + miss + cn > 0) { + stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); + stat = Number.isNaN(Number(stat)) ? null : stat; + } + } else if ( + rows[rowIndex].stat === undefined && + rows[rowIndex].square_diff_sum !== undefined + ) { + // this is a scalar partial sums plot + isScalar = true; + squareDiffSum = Number(rows[rowIndex].square_diff_sum); + NSum = Number(rows[rowIndex].N_sum); + obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); + modelSum = Number(rows[rowIndex].model_sum); + obsSum = Number(rows[rowIndex].obs_sum); + absSum = Number(rows[rowIndex].abs_sum); + if (NSum > 0) { + stat = matsDataUtils.calculateStatScalar( + squareDiffSum, + NSum, + obsModelDiffSum, + modelSum, + obsSum, + absSum, + statisticStr + ); + stat = Number.isNaN(Number(stat)) ? null : stat; + const variable = statisticStr.split("_")[1]; + stdev = matsDataUtils.calculateStatScalar( + squareDiffSum, + NSum, + obsModelDiffSum, + modelSum, + obsSum, + absSum, + `Std deviation_${variable}` + ); } } else { - queryVal = null; + // not a contingency table plot + stat = rows[rowIndex].stat === "NULL" ? null : rows[rowIndex].stat; + stdev = rows[rowIndex].stdev !== undefined ? rows[rowIndex].stdev : null; } - - // store sub values to test them for stdev. - const sub_hit = []; - const sub_fa = []; - const sub_miss = []; - const sub_cn = []; - const sub_values = []; - const sub_secs = []; - const sub_levs = []; - let sub_stdev = 0; - let sub_mean = 0; - let sd_limit = 0; + let minDate = rows[rowIndex].min_secs; + let maxDate = rows[rowIndex].max_secs; + if (stat === undefined || stat === null) { + stat = null; + stdev = 0; + n = 0; + minDate = null; + maxDate = null; + } + let thisSubHit = []; + let thisSubFa = []; + let thisSubMiss = []; + let thisSubCn = []; + let thisSubSquareDiffSum = []; + let thisSubNSum = []; + let thisSubObsModelDiffSum = []; + let thisSubModelSum = []; + let thisSubObsSum = []; + let thisSubAbsSum = []; + let thisSubValues = []; + let thisSubSecs = []; + let thisSubLevs = []; if ( - queryVal !== null && + stat !== null && rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null ) { // parse the sub-data try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - var curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); + const thisSubData = rows[rowIndex].sub_data.toString().split(","); + let currSubData; + for (let sdIdx = 0; sdIdx < thisSubData.length; sdIdx += 1) { + currSubData = thisSubData[sdIdx].split(";"); + if (isCTC) { + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubHit.push(Number(currSubData[2])); + thisSubFa.push(Number(currSubData[3])); + thisSubMiss.push(Number(currSubData[4])); + thisSubCn.push(Number(currSubData[5])); } else { - sub_levs.push(curr_sub_data[1]); + thisSubHit.push(Number(currSubData[1])); + thisSubFa.push(Number(currSubData[2])); + thisSubMiss.push(Number(currSubData[3])); + thisSubCn.push(Number(currSubData[4])); + } + } else if (isScalar) { + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubSquareDiffSum.push(Number(currSubData[2])); + thisSubNSum.push(Number(currSubData[3])); + thisSubObsModelDiffSum.push(Number(currSubData[4])); + thisSubModelSum.push(Number(currSubData[5])); + thisSubObsSum.push(Number(currSubData[6])); + thisSubAbsSum.push(Number(currSubData[7])); + } else { + thisSubSquareDiffSum.push(Number(currSubData[1])); + thisSubNSum.push(Number(currSubData[2])); + thisSubObsModelDiffSum.push(Number(currSubData[3])); + thisSubModelSum.push(Number(currSubData[4])); + thisSubObsSum.push(Number(currSubData[5])); + thisSubAbsSum.push(Number(currSubData[6])); } - sub_hit.push(Number(curr_sub_data[2])); - sub_fa.push(Number(curr_sub_data[3])); - sub_miss.push(Number(curr_sub_data[4])); - sub_cn.push(Number(curr_sub_data[5])); - sub_values.push( - matsDataUtils.calculateStatCTC( - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - curr_sub_data.length, - statistic - ) - ); } else { - sub_hit.push(Number(curr_sub_data[1])); - sub_fa.push(Number(curr_sub_data[2])); - sub_miss.push(Number(curr_sub_data[3])); - sub_cn.push(Number(curr_sub_data[4])); - sub_values.push( - matsDataUtils.calculateStatCTC( - Number(curr_sub_data[1]), - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - curr_sub_data.length, - statistic - ) - ); + thisSubSecs.push(Number(currSubData[0])); + if (hasLevels) { + if (!Number.isNaN(Number(currSubData[1]))) { + thisSubLevs.push(Number(currSubData[1])); + } else { + thisSubLevs.push(currSubData[1]); + } + thisSubValues.push(Number(currSubData[2])); + } else { + thisSubValues.push(Number(currSubData[1])); + } } } - // Now that we have all the sub-values, we can get the standard deviation and remove the ones that exceed it - if (outlierQCParam !== "all") { - sub_stdev = matsDataUtils.stdev(sub_values); - sub_mean = matsDataUtils.average(sub_values); - sd_limit = outlierQCParam * sub_stdev; - for (let svIdx = sub_values.length - 1; svIdx >= 0; svIdx--) { - if (Math.abs(sub_values[svIdx] - sub_mean) > sd_limit) { - sub_hit.splice(svIdx, 1); - sub_fa.splice(svIdx, 1); - sub_miss.splice(svIdx, 1); - sub_cn.splice(svIdx, 1); - sub_values.splice(svIdx, 1); - sub_secs.splice(svIdx, 1); - if (hasLevels) { - sub_levs.splice(svIdx, 1); - } - } - } + } catch (e) { + // this is an error produced by a bug in the query function, not an error returned by the mysql database + e.message = `Error in parseQueryDataContour. The expected fields don't seem to be present in the results cache: ${e.message}`; + throw new Error(e.message); + } + } else { + if (isCTC) { + thisSubHit = NaN; + thisSubFa = NaN; + thisSubMiss = NaN; + thisSubCn = NaN; + } else if (isScalar) { + thisSubSquareDiffSum = NaN; + thisSubNSum = NaN; + thisSubObsModelDiffSum = NaN; + thisSubModelSum = NaN; + thisSubObsSum = NaN; + thisSubAbsSum = NaN; + } else { + thisSubValues = NaN; + } + thisSubSecs = NaN; + if (hasLevels) { + thisSubLevs = NaN; + } + } + // store flat arrays of all the parsed data, used by the text output and for some calculations later + returnD.xTextOutput.push(rowXVal); + returnD.yTextOutput.push(rowYVal); + returnD.zTextOutput.push(stat); + returnD.nTextOutput.push(n); + returnD.hitTextOutput.push(hit); + returnD.faTextOutput.push(fa); + returnD.missTextOutput.push(miss); + returnD.cnTextOutput.push(cn); + returnD.squareDiffSumTextOutput.push(squareDiffSum); + returnD.NSumTextOutput.push(NSum); + returnD.obsModelDiffSumTextOutput.push(obsModelDiffSum); + returnD.modelSumTextOutput.push(modelSum); + returnD.obsSumTextOutput.push(obsSum); + returnD.absSumTextOutput.push(absSum); + returnD.minDateTextOutput.push(minDate); + returnD.maxDateTextOutput.push(maxDate); + curveStatLookup[statKey] = stat; + curveStdevLookup[statKey] = stdev; + curveNLookup[statKey] = n; + if (isCTC) { + curveSubHitLookup[statKey] = thisSubHit; + curveSubFaLookup[statKey] = thisSubFa; + curveSubMissLookup[statKey] = thisSubMiss; + curveSubCnLookup[statKey] = thisSubCn; + } else if (isScalar) { + curveSubSquareDiffSumLookup[statKey] = thisSubSquareDiffSum; + curveSubNSumLookup[statKey] = thisSubNSum; + curveSubObsModelDiffSumLookup[statKey] = thisSubObsModelDiffSum; + curveSubModelSumLookup[statKey] = thisSubModelSum; + curveSubObsSumLookup[statKey] = thisSubObsSum; + curveSubAbsSumLookup[statKey] = thisSubAbsSum; + } else { + curveSubValLookup[statKey] = thisSubValues; + } + curveSubSecLookup[statKey] = thisSubSecs; + if (hasLevels) { + curveSubLevLookup[statKey] = thisSubLevs; + } + } + + // get the unique x and y values and sort the stats into the 2D z array accordingly + returnD.x = matsDataUtils.arrayUnique(returnD.xTextOutput).sort(function (a, b) { + return a - b; + }); + returnD.y = matsDataUtils.arrayUnique(returnD.yTextOutput).sort(function (a, b) { + return a - b; + }); + let i; + let j; + let currX; + let currY; + let currStat; + let currStdev; + let currN; + let currSubHit; + let currSubFa; + let currSubMiss; + let currSubCn; + let currSubSquareDiffSum; + let currSubNSum; + let currSubObsModelDiffSum; + let currSubModelSum; + let currSubObsSum; + let currSubAbsSum; + let currSubVal; + let currSubSec; + let currSubLev; + let currStatKey; + let currYStatArray; + let currYStdevArray; + let currYNArray; + let currYSubHitArray; + let currYSubFaArray; + let currYSubMissArray; + let currYSubCnArray; + let currYSubSquareDiffSumArray; + let currYSubNSumArray; + let currYSubObsModelDiffSumArray; + let currYSubModelSumArray; + let currYSubObsSumArray; + let currYSubAbsSumArray; + let currYSubValArray; + let currYSubSecArray; + let currYSubLevArray; + let sum = 0; + let nPoints = 0; + let zmin = Number.MAX_VALUE; + let zmax = -1 * Number.MAX_VALUE; + + for (j = 0; j < returnD.y.length; j += 1) { + currY = returnD.y[j]; + currYStatArray = []; + currYStdevArray = []; + currYNArray = []; + if (isCTC) { + currYSubHitArray = []; + currYSubFaArray = []; + currYSubMissArray = []; + currYSubCnArray = []; + } else if (isScalar) { + currYSubSquareDiffSumArray = []; + currYSubNSumArray = []; + currYSubObsModelDiffSumArray = []; + currYSubModelSumArray = []; + currYSubObsSumArray = []; + currYSubAbsSumArray = []; + } else { + currYSubValArray = []; + } + currYSubSecArray = []; + if (hasLevels) { + currYSubLevArray = []; + } + for (i = 0; i < returnD.x.length; i += 1) { + currX = returnD.x[i]; + currStatKey = `${currX.toString()}_${currY.toString()}`; + currStat = curveStatLookup[currStatKey]; + currStdev = curveStdevLookup[currStatKey]; + currN = curveNLookup[currStatKey]; + if (isCTC) { + currSubHit = curveSubHitLookup[currStatKey]; + currSubFa = curveSubFaLookup[currStatKey]; + currSubMiss = curveSubMissLookup[currStatKey]; + currSubCn = curveSubCnLookup[currStatKey]; + } else if (isScalar) { + currSubSquareDiffSum = curveSubSquareDiffSumLookup[currStatKey]; + currSubNSum = curveSubNSumLookup[currStatKey]; + currSubObsModelDiffSum = curveSubObsModelDiffSumLookup[currStatKey]; + currSubModelSum = curveSubModelSumLookup[currStatKey]; + currSubObsSum = curveSubObsSumLookup[currStatKey]; + currSubAbsSum = curveSubAbsSumLookup[currStatKey]; + } else { + currSubVal = curveSubValLookup[currStatKey]; + } + currSubSec = curveSubSecLookup[currStatKey]; + if (hasLevels) { + currSubLev = curveSubLevLookup[currStatKey]; + } + if (currStat === undefined) { + currYStatArray.push(null); + currYStdevArray.push(null); + currYNArray.push(0); + if (isCTC) { + currYSubHitArray.push(null); + currYSubFaArray.push(null); + currYSubMissArray.push(null); + currYSubCnArray.push(null); + } else if (isScalar) { + currYSubSquareDiffSumArray.push(null); + currYSubNSumArray.push(null); + currYSubObsModelDiffSumArray.push(null); + currYSubModelSumArray.push(null); + currYSubObsSumArray.push(null); + currYSubAbsSumArray.push(null); + } else { + currYSubValArray.push(null); + } + currYSubSecArray.push(null); + if (hasLevels) { + currYSubLevArray.push(null); + } + } else { + sum += currStat; + nPoints += 1; + currYStatArray.push(currStat); + currYStdevArray.push(currStdev); + currYNArray.push(currN); + if (isCTC) { + currYSubHitArray.push(currSubHit); + currYSubFaArray.push(currSubFa); + currYSubMissArray.push(currSubMiss); + currYSubCnArray.push(currSubCn); + } else if (isScalar) { + currYSubSquareDiffSumArray.push(currSubSquareDiffSum); + currYSubNSumArray.push(currSubNSum); + currYSubObsModelDiffSumArray.push(currSubObsModelDiffSum); + currYSubModelSumArray.push(currSubModelSum); + currYSubObsSumArray.push(currSubObsSum); + currYSubAbsSumArray.push(currSubAbsSum); + } else { + currYSubValArray.push(currSubVal); + } + currYSubSecArray.push(currSubSec); + if (hasLevels) { + currYSubLevArray.push(currSubLev); } - const hit = matsDataUtils.sum(sub_hit); - const fa = matsDataUtils.sum(sub_fa); - const miss = matsDataUtils.sum(sub_miss); - const cn = matsDataUtils.sum(sub_cn); - queryVal = matsDataUtils.calculateStatCTC( - hit, - fa, - miss, - cn, - sub_hit.length, - statistic - ); - queryVal = isNaN(Number(queryVal)) ? null : queryVal; - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataMapCTC. The expected fields don't seem to be present in the results cache: ${e.message}`; - throw new Error(e.message); + zmin = currStat < zmin ? currStat : zmin; + zmax = currStat > zmax ? currStat : zmax; } - d.queryVal.push(queryVal); - d.stats.push({ - N_times: rows[rowIndex].N_times, - min_time: rows[rowIndex].min_secs, - max_time: rows[rowIndex].max_secs, - hit: rows[rowIndex].hit, - fa: rows[rowIndex].fa, - miss: rows[rowIndex].miss, - cn: rows[rowIndex].cn, - }); + } + returnD.z.push(currYStatArray); + returnD.stdev.push(currYStdevArray); + returnD.n.push(currYNArray); + if (isCTC) { + returnD.subHit.push(currYSubHitArray); + returnD.subFa.push(currYSubFaArray); + returnD.subMiss.push(currYSubMissArray); + returnD.subCn.push(currYSubCnArray); + } else if (isScalar) { + returnD.subSquareDiffSum.push(currYSubSquareDiffSumArray); + returnD.subNSum.push(currYSubNSumArray); + returnD.subObsModelDiffSum.push(currYSubObsModelDiffSumArray); + returnD.subModelSum.push(currYSubModelSumArray); + returnD.subObsSum.push(currYSubObsSumArray); + returnD.subAbsSum.push(currYSubAbsSumArray); + } else { + returnD.subVals.push(currYSubValArray); + } + returnD.subSecs.push(currYSubSecArray); + if (hasLevels) { + returnD.subLevs.push(currYSubLevArray); + } + } - let thisSite; - if (isCouchbase) { - thisSite = siteMap.find((obj) => obj.name === site); - } else { - thisSite = siteMap.find((obj) => obj.options.id === site); + // calculate statistics + [returnD.xmin] = returnD.x; + returnD.xmax = returnD.x[returnD.x.length - 1]; + [returnD.ymin] = returnD.y; + returnD.ymax = returnD.y[returnD.y.length - 1]; + returnD.zmin = zmin; + returnD.zmax = zmax; + returnD.sum = sum; + + const filteredMinDate = returnD.minDateTextOutput.filter((t) => t || t === 0); + const filteredMaxDate = returnD.maxDateTextOutput.filter((t) => t || t === 0); + returnD.glob_stats.mean = sum / nPoints; + returnD.glob_stats.minDate = Math.min(...filteredMinDate); + returnD.glob_stats.maxDate = Math.max(...filteredMaxDate); + returnD.glob_stats.n = nPoints; + + return { + d: returnD, + }; +}; + +// this method queries the database for timeseries plots +const queryDBTimeSeries = function ( + pool, + statementOrMwRows, + dataSource, + forecastOffset, + startDate, + endDate, + averageStr, + statisticStr, + validTimes, + appParams, + forceRegularCadence +) { + if (Meteor.isServer) { + // upper air is only verified at 00Z and 12Z, so you need to force irregular models to verify at that regular cadence + let cycles = getModelCadence(pool, dataSource, startDate, endDate); // if irregular model cadence, get cycle times. If regular, get empty array. + let theseValidTimes = validTimes; + if (theseValidTimes.length > 0 && theseValidTimes !== matsTypes.InputTypes.unused) { + if (typeof theseValidTimes === "string" || theseValidTimes instanceof String) { + theseValidTimes = theseValidTimes.split(","); } + let vtCycles = theseValidTimes.map(function (x) { + return (Number(x) - forecastOffset) * 3600 * 1000; + }); // selecting validTimes makes the cadence irregular + vtCycles = vtCycles.map(function (x) { + return x < 0 ? x + 24 * 3600 * 1000 : x; + }); // make sure no cycles are negative + vtCycles = vtCycles.sort(function (a, b) { + return Number(a) - Number(b); + }); // sort 'em + cycles = cycles.length > 0 ? _.intersection(cycles, vtCycles) : vtCycles; // if we already had cycles get the ones that correspond to valid times + } + const regular = + forceRegularCadence || + averageStr !== "None" || + !(cycles !== null && cycles.length > 0); // If curves have averaging, the cadence is always regular, i.e. it's the cadence of the average - const tooltips = - `${thisSite.origName}
` + - `model: ${dataSource}
${statistic}: ${queryVal}
` + - `n: ${rows[rowIndex].N_times}
` + - `hits: ${rows[rowIndex].hit}
` + - `false alarms: ${rows[rowIndex].fa}
` + - `misses: ${rows[rowIndex].miss}
` + - `correct nulls: ${rows[rowIndex].cn}`; - d.text.push(tooltips); - d.siteName.push(thisSite.origName); - d.lat.push(thisSite.point[0]); - d.lon.push(thisSite.point[1]); + let d = { + // d will contain the curve data + x: [], + y: [], + error_x: [], + error_y: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subSquareDiffSum: [], + subNSum: [], + subObsModelDiffSum: [], + subModelSum: [], + subObsSum: [], + subAbsSum: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], + glob_stats: {}, + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + sum: 0, + }; + let error = ""; + let n0 = []; + let nTimes = []; + let parsedData; - // sort the data by the color it will appear on the map - const textMarker = queryVal === null ? "" : queryVal.toFixed(0); - if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.1) { - d.color.push("rgb(128,0,255)"); - dPurple.siteName.push(thisSite.origName); - dPurple.queryVal.push(queryVal); - dPurple.text.push(textMarker); - dPurple.lat.push(thisSite.point[0]); - dPurple.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.2) { - d.color.push("rgb(64,0,255)"); - dPurpleBlue.siteName.push(thisSite.origName); - dPurpleBlue.queryVal.push(queryVal); - dPurpleBlue.text.push(textMarker); - dPurpleBlue.lat.push(thisSite.point[0]); - dPurpleBlue.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.3) { - d.color.push("rgb(0,0,255)"); - dBlue.siteName.push(thisSite.origName); - dBlue.queryVal.push(queryVal); - dBlue.text.push(textMarker); - dBlue.lat.push(thisSite.point[0]); - dBlue.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.4) { - d.color.push("rgb(64,128,128)"); - dBlueGreen.siteName.push(thisSite.origName); - dBlueGreen.queryVal.push(queryVal); - dBlueGreen.text.push(textMarker); - dBlueGreen.lat.push(thisSite.point[0]); - dBlueGreen.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.5) { - d.color.push("rgb(128,255,0)"); - dGreen.siteName.push(thisSite.origName); - dGreen.queryVal.push(queryVal); - dGreen.text.push(textMarker); - dGreen.lat.push(thisSite.point[0]); - dGreen.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.6) { - d.color.push("rgb(160,224,0)"); - dGreenYellow.siteName.push(thisSite.origName); - dGreenYellow.queryVal.push(queryVal); - dGreenYellow.text.push(textMarker); - dGreenYellow.lat.push(thisSite.point[0]); - dGreenYellow.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.7) { - d.color.push("rgb(192,192,0)"); - dYellow.siteName.push(thisSite.origName); - dYellow.queryVal.push(queryVal); - dYellow.text.push(textMarker); - dYellow.lat.push(thisSite.point[0]); - dYellow.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.8) { - d.color.push("rgb(255,128,0)"); - dOrange.siteName.push(thisSite.origName); - dOrange.queryVal.push(queryVal); - dOrange.text.push(textMarker); - dOrange.lat.push(thisSite.point[0]); - dOrange.lon.push(thisSite.point[1]); - } else if (queryVal <= lowLimit + (highLimit - lowLimit) * 0.9) { - d.color.push("rgb(255,64,0)"); - dOrangeRed.siteName.push(thisSite.origName); - dOrangeRed.queryVal.push(queryVal); - dOrangeRed.text.push(textMarker); - dOrangeRed.lat.push(thisSite.point[0]); - dOrangeRed.lon.push(thisSite.point[1]); + const Future = require("fibers/future"); + const dFuture = new Future(); + + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBTimeSeries' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + let rows = null; + if (Array.isArray(statementOrMwRows)) { + rows = statementOrMwRows; + dFuture.return(); + } else { + (async () => { + rows = await pool.queryCB(statementOrMwRows); + // done waiting - have results + dFuture.return(); + })(); + } + dFuture.wait(); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; } else { - d.color.push("rgb(255,0,0)"); - dRed.siteName.push(thisSite.origName); - dRed.queryVal.push(queryVal); - dRed.text.push(textMarker); - dRed.lat.push(thisSite.point[0]); - dRed.lon.push(thisSite.point[1]); + parsedData = parseQueryDataXYCurve( + rows, + d, + appParams, + statisticStr, + forecastOffset, + cycles, + regular + ); + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; } + } else { + // if this app isn't couchbase, use mysql + pool.query(statementOrMwRows, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else { + parsedData = parseQueryDataXYCurve( + rows, + d, + appParams, + statisticStr, + forecastOffset, + cycles, + regular + ); + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; + } + // done waiting - have results + dFuture.return(); + }); } - } // end of loop row + // wait for the future to finish - sort of like 'back to the future' ;) + dFuture.wait(); - return { - d, - dPurple, - dPurpleBlue, - dBlue, - dBlueGreen, - dGreen, - dGreenYellow, - dYellow, - dOrange, - dOrangeRed, - dRed, - valueLimits: { - maxValue: highLimit, - highLimit, - lowLimit, - }, - }; + return { + data: d, + error, + n0, + nTimes, + }; + } + return null; }; -// this method parses the returned query data for histograms -const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { - /* - var d = { // d will contain the curve data - x: [], - y: [], - error_x: [], - error_y: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subSquareDiffSum: [], - subNSum: [], - subObsModelDiffSum: [], - subModelSum: [], - subObsSum: [], - subAbsSum: [], - subData: [], - subHeaders: [], - subVals: [], - subSecs: [], - subLevs: [], - stats: [], - text: [], - glob_stats: {}, - bin_stats: [], - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - sum: 0 - }; - */ - const { hasLevels } = appParams; - let isCTC = false; - let isScalar = false; +// this method queries the database for specialty curves such as profiles, dieoffs, threshold plots, valid time plots, grid scale plots, and histograms +const queryDBSpecialtyCurve = function ( + pool, + statementOrMwRows, + appParams, + statisticStr +) { + if (Meteor.isServer) { + let d = { + // d will contain the curve data + x: [], + y: [], + error_x: [], + error_y: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subSquareDiffSum: [], + subNSum: [], + subObsModelDiffSum: [], + subModelSum: [], + subObsSum: [], + subAbsSum: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], + glob_stats: {}, + bin_stats: [], + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + sum: 0, + }; + let error = ""; + let n0 = []; + let nTimes = []; + let parsedData; - // these arrays hold all the sub values and seconds (and levels) until they are sorted into bins - const curveSubStatsRaw = []; - const curveSubSecsRaw = []; - const curveSubLevsRaw = []; + const Future = require("fibers/future"); + const dFuture = new Future(); - // parse the data returned from the query - for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) { - var stat; - if (rows[rowIndex].stat === undefined && rows[rowIndex].hit !== undefined) { - // this is a contingency table plot - isCTC = true; - const hit = Number(rows[rowIndex].hit); - const fa = Number(rows[rowIndex].fa); - const miss = Number(rows[rowIndex].miss); - const cn = Number(rows[rowIndex].cn); - const n = rows[rowIndex].sub_data.toString().split(",").length; - if (hit + fa + miss + cn > 0) { - stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = isNaN(Number(stat)) ? null : stat; + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBSpecialtyCurve' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + let rows = null; + if (Array.isArray(statementOrMwRows)) { + rows = statementOrMwRows; + dFuture.return(); } else { - stat = null; + (async () => { + rows = await pool.queryCB(statementOrMwRows); + dFuture.return(); + })(); } - } else if ( - rows[rowIndex].stat === undefined && - rows[rowIndex].square_diff_sum !== undefined - ) { - // this is a scalar partial sums plot - isScalar = true; - const squareDiffSum = Number(rows[rowIndex].square_diff_sum); - const NSum = Number(rows[rowIndex].N_sum); - const obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); - const modelSum = Number(rows[rowIndex].model_sum); - const obsSum = Number(rows[rowIndex].obs_sum); - const absSum = Number(rows[rowIndex].abs_sum); - if (NSum > 0) { - stat = matsDataUtils.calculateStatScalar( - squareDiffSum, - NSum, - obsModelDiffSum, - modelSum, - obsSum, - absSum, - statisticStr - ); - stat = isNaN(Number(stat)) ? null : stat; + dFuture.wait(); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; } else { - stat = null; + if (appParams.plotType !== matsTypes.PlotTypes.histogram) { + parsedData = parseQueryDataXYCurve( + rows, + d, + appParams, + statisticStr, + null, + null, + null + ); + } else { + parsedData = parseQueryDataHistogram(rows, d, appParams, statisticStr); + } + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; } } else { - // not a contingency table plot or a scalar partial sums plot - stat = rows[rowIndex].stat === "NULL" ? null : rows[rowIndex].stat; - } - const sub_stats = []; - const sub_secs = []; - const sub_levs = []; - if ( - stat !== null && - rows[rowIndex].sub_data !== undefined && - rows[rowIndex].sub_data !== null - ) { - // parse the sub-data - try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - var curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - if (isCTC) { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_stats.push( - matsDataUtils.calculateStatCTC( - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - 1, - statisticStr - ) - ); - } else { - sub_stats.push( - matsDataUtils.calculateStatCTC( - Number(curr_sub_data[1]), - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - 1, - statisticStr - ) - ); - } - } else if (isScalar) { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_stats.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - Number(curr_sub_data[7]), - statisticStr - ) - ); - } else { - sub_stats.push( - matsDataUtils.calculateStatScalar( - Number(curr_sub_data[1]), - Number(curr_sub_data[2]), - Number(curr_sub_data[3]), - Number(curr_sub_data[4]), - Number(curr_sub_data[5]), - Number(curr_sub_data[6]), - statisticStr - ) - ); - } + // if this app isn't couchbase, use mysql + pool.query(statementOrMwRows, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else { + if (appParams.plotType !== matsTypes.PlotTypes.histogram) { + parsedData = parseQueryDataXYCurve( + rows, + d, + appParams, + statisticStr, + null, + null, + null + ); } else { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_stats.push(Number(curr_sub_data[2])); - } else { - sub_stats.push(Number(curr_sub_data[1])); - } + parsedData = parseQueryDataHistogram(rows, d, appParams, statisticStr); } + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; } - curveSubStatsRaw.push(sub_stats); - curveSubSecsRaw.push(sub_secs); - curveSubLevsRaw.push(sub_levs); - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataHistogram. The expected fields don't seem to be present in the results cache: ${e.message}`; - throw new Error(e.message); - } + // done waiting - have results + dFuture.return(); + }); } - } + // wait for the future to finish - sort of like 'back to the future' ;) + dFuture.wait(); - // we don't have bins yet, so we want all of the data in one array - const subVals = [].concat.apply([], curveSubStatsRaw); - const subSecs = [].concat.apply([], curveSubSecsRaw); - let subLevs; - if (hasLevels) { - subLevs = [].concat.apply([], curveSubLevsRaw); + return { + data: d, + error, + n0, + nTimes, + }; } + return null; +}; + +// this method queries the database for performance diagrams +const queryDBReliability = function (pool, statement, appParams, kernel) { + if (Meteor.isServer) { + let d = { + // d will contain the curve data + x: [], + y: [], + binVals: [], + hitCount: [], + fcstCount: [], + fcstRawCount: [], + sample_climo: 0, + n: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subData: [], + subHeaders: [], + subRelHit: [], + subRelRawCount: [], + subRelCount: [], + subVals: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + }; + let error = ""; + let parsedData; - d.subVals = subVals; - d.subSecs = subSecs; - d.subLevs = subLevs; + const Future = require("fibers/future"); + const dFuture = new Future(); + + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBReliability' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + (async () => { + const rows = await pool.queryCB(statement); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; + } else { + parsedData = parseQueryDataReliability(rows, d, appParams, kernel); + d = parsedData.d; + } + dFuture.return(); + })(); + } else { + // if this app isn't couchbase, use mysql + pool.query(statement, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else { + parsedData = parseQueryDataReliability(rows, d, appParams, kernel); + d = parsedData.d; + } + // done waiting - have results + dFuture.return(); + }); + } + // wait for the future to finish - sort of like 'back to the future' ;) + dFuture.wait(); - return { - d, - N0: subVals.length, - N_times: subSecs.length, - }; + return { + data: d, + error, + }; + } + return null; }; -// this method parses the returned query data for contour plots -const parseQueryDataContour = function (rows, d, appParams, statisticStr) { - /* - var d = { // d will contain the curve data - x: [], - y: [], - z: [], - n: [], - subHit: [], - subFa: [], - subMiss: [], - subCn: [], - subSquareDiffSum: [], - subNSum: [], - subObsModelDiffSum: [], - subModelSum: [], - subObsSum: [], - subAbsSum: [], - subData: [], - subHeaders: [], - subVals: [], - subSecs: [], - subLevs: [], - text: [], - xTextOutput: [], - yTextOutput: [], - zTextOutput: [], - nTextOutput: [], - hitTextOutput: [], - faTextOutput: [], - missTextOutput: [], - cnTextOutput: [], - squareDiffSumTextOutput: [], - NSumTextOutput: [], - obsModelDiffSumTextOutput: [], - modelSumTextOutput: [], - obsSumTextOutput: [], - absSumTextOutput: [], - minDateTextOutput: [], - maxDateTextOutput: [], - stdev: [], - stats: [], - glob_stats: {}, - xmin: Number.MAX_VALUE, - xmax: Number.MIN_VALUE, - ymin: Number.MAX_VALUE, - ymax: Number.MIN_VALUE, - zmin: Number.MAX_VALUE, - zmax: Number.MIN_VALUE, - sum: 0 - }; - */ - const { hasLevels } = appParams; - let isCTC = false; - let isScalar = false; +// this method queries the database for performance diagrams +const queryDBPerformanceDiagram = function (pool, statement, appParams) { + if (Meteor.isServer) { + let d = { + // d will contain the curve data + x: [], + y: [], + binVals: [], + oy_all: [], + on_all: [], + n: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + stats: [], + text: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + }; + let error = ""; + let n0 = []; + let nTimes = []; + let parsedData; - // initialize local variables - const curveStatLookup = {}; - const curveStdevLookup = {}; - const curveNLookup = {}; - const curveSubHitLookup = {}; - const curveSubFaLookup = {}; - const curveSubMissLookup = {}; - const curveSubCnLookup = {}; - const curveSubSquareDiffSumLookup = {}; - const curveSubNSumLookup = {}; - const curveSubObsModelDiffSumLookup = {}; - const curveSubModelSumLookup = {}; - const curveSubObsSumLookup = {}; - const curveSubAbsSumLookup = {}; - const curveSubValLookup = {}; - const curveSubSecLookup = {}; - const curveSubLevLookup = {}; + const Future = require("fibers/future"); + const dFuture = new Future(); - // get all the data out of the query array - for (let rowIndex = 0; rowIndex < rows.length; rowIndex++) { - const rowXVal = Number(rows[rowIndex].xVal); - const rowYVal = Number(rows[rowIndex].yVal); - const statKey = `${rowXVal.toString()}_${rowYVal.toString()}`; - let stat = null; - let hit = null; - let fa = null; - let miss = null; - let cn = null; - let n = - rows[rowIndex].sub_data !== undefined && rows[rowIndex].sub_data !== null - ? rows[rowIndex].sub_data.toString().split(",").length - : 0; - let squareDiffSum = null; - let NSum = null; - let obsModelDiffSum = null; - let modelSum = null; - let obsSum = null; - let absSum = null; - let stdev = null; - if (rows[rowIndex].stat === undefined && rows[rowIndex].hit !== undefined) { - // this is a contingency table plot - isCTC = true; - hit = Number(rows[rowIndex].hit); - fa = Number(rows[rowIndex].fa); - miss = Number(rows[rowIndex].miss); - cn = Number(rows[rowIndex].cn); - if (hit + fa + miss + cn > 0) { - stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = isNaN(Number(stat)) ? null : stat; - } - } else if ( - rows[rowIndex].stat === undefined && - rows[rowIndex].square_diff_sum !== undefined - ) { - // this is a scalar partial sums plot - isScalar = true; - squareDiffSum = Number(rows[rowIndex].square_diff_sum); - NSum = Number(rows[rowIndex].N_sum); - obsModelDiffSum = Number(rows[rowIndex].obs_model_diff_sum); - modelSum = Number(rows[rowIndex].model_sum); - obsSum = Number(rows[rowIndex].obs_sum); - absSum = Number(rows[rowIndex].abs_sum); - if (NSum > 0) { - stat = matsDataUtils.calculateStatScalar( - squareDiffSum, - NSum, - obsModelDiffSum, - modelSum, - obsSum, - absSum, - statisticStr - ); - stat = isNaN(Number(stat)) ? null : stat; - const variable = statisticStr.split("_")[1]; - stdev = matsDataUtils.calculateStatScalar( - squareDiffSum, - NSum, - obsModelDiffSum, - modelSum, - obsSum, - absSum, - `Std deviation_${variable}` - ); - } + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBSPerformanceDiagram' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + (async () => { + const rows = await pool.queryCB(statement); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; + } else { + parsedData = parseQueryDataPerformanceDiagram(rows, d, appParams); + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; + } + dFuture.return(); + })(); } else { - // not a contingency table plot - stat = rows[rowIndex].stat === "NULL" ? null : rows[rowIndex].stat; - stdev = rows[rowIndex].stdev !== undefined ? rows[rowIndex].stdev : null; - } - let minDate = rows[rowIndex].min_secs; - let maxDate = rows[rowIndex].max_secs; - if (stat === undefined || stat === null) { - stat = null; - stdev = 0; - n = 0; - minDate = null; - maxDate = null; + // if this app isn't couchbase, use mysql + pool.query(statement, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else { + parsedData = parseQueryDataPerformanceDiagram(rows, d, appParams); + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; + } + // done waiting - have results + dFuture.return(); + }); } - let sub_hit = []; - let sub_fa = []; - let sub_miss = []; - let sub_cn = []; - let sub_square_diff_sum = []; - let sub_N_sum = []; - let sub_obs_model_diff_sum = []; - let sub_model_sum = []; - let sub_obs_sum = []; - let sub_abs_sum = []; - let sub_values = []; - let sub_secs = []; - let sub_levs = []; - if ( - stat !== null && - rows[rowIndex].sub_data !== undefined && - rows[rowIndex].sub_data !== null - ) { - // parse the sub-data - try { - const sub_data = rows[rowIndex].sub_data.toString().split(","); - var curr_sub_data; - for (let sd_idx = 0; sd_idx < sub_data.length; sd_idx++) { - curr_sub_data = sub_data[sd_idx].split(";"); - if (isCTC) { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_hit.push(Number(curr_sub_data[2])); - sub_fa.push(Number(curr_sub_data[3])); - sub_miss.push(Number(curr_sub_data[4])); - sub_cn.push(Number(curr_sub_data[5])); - } else { - sub_hit.push(Number(curr_sub_data[1])); - sub_fa.push(Number(curr_sub_data[2])); - sub_miss.push(Number(curr_sub_data[3])); - sub_cn.push(Number(curr_sub_data[4])); - } - } else if (isScalar) { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_square_diff_sum.push(Number(curr_sub_data[2])); - sub_N_sum.push(Number(curr_sub_data[3])); - sub_obs_model_diff_sum.push(Number(curr_sub_data[4])); - sub_model_sum.push(Number(curr_sub_data[5])); - sub_obs_sum.push(Number(curr_sub_data[6])); - sub_abs_sum.push(Number(curr_sub_data[7])); - } else { - sub_square_diff_sum.push(Number(curr_sub_data[1])); - sub_N_sum.push(Number(curr_sub_data[2])); - sub_obs_model_diff_sum.push(Number(curr_sub_data[3])); - sub_model_sum.push(Number(curr_sub_data[4])); - sub_obs_sum.push(Number(curr_sub_data[5])); - sub_abs_sum.push(Number(curr_sub_data[6])); - } - } else { - sub_secs.push(Number(curr_sub_data[0])); - if (hasLevels) { - if (!isNaN(Number(curr_sub_data[1]))) { - sub_levs.push(Number(curr_sub_data[1])); - } else { - sub_levs.push(curr_sub_data[1]); - } - sub_values.push(Number(curr_sub_data[2])); - } else { - sub_values.push(Number(curr_sub_data[1])); - } - } + // wait for the future to finish - sort of like 'back to the future' ;) + dFuture.wait(); + + return { + data: d, + error, + n0, + nTimes, + }; + } + return null; +}; + +// this method queries the database for performance diagrams +const queryDBSimpleScatter = function ( + pool, + statement, + appParams, + statisticXStr, + statisticYStr +) { + if (Meteor.isServer) { + let d = { + // d will contain the curve data + x: [], + y: [], + binVals: [], + subSquareDiffSumX: [], + subNSumX: [], + subObsModelDiffSumX: [], + subModelSumX: [], + subObsSumX: [], + subAbsSumX: [], + subSquareDiffSumY: [], + subNSumY: [], + subObsModelDiffSumY: [], + subModelSumY: [], + subObsSumY: [], + subAbsSumY: [], + subValsX: [], + subValsY: [], + subSecsX: [], + subSecsY: [], + subSecs: [], + subLevsX: [], + subLevsY: [], + subLevs: [], + stats: [], + text: [], + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + sum: 0, + }; + let error = ""; + let n0 = []; + let nTimes = []; + let parsedData; + + const Future = require("fibers/future"); + const dFuture = new Future(); + + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBSPerformanceDiagram' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + (async () => { + const rows = await pool.queryCB(statement); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; + } else { + parsedData = parseQueryDataSimpleScatter( + rows, + d, + appParams, + statisticXStr, + statisticYStr + ); + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; } - } catch (e) { - // this is an error produced by a bug in the query function, not an error returned by the mysql database - e.message = `Error in parseQueryDataContour. The expected fields don't seem to be present in the results cache: ${e.message}`; - throw new Error(e.message); - } + dFuture.return(); + })(); } else { - if (isCTC) { - sub_hit = NaN; - sub_fa = NaN; - sub_miss = NaN; - sub_cn = NaN; - } else if (isScalar) { - sub_square_diff_sum = NaN; - sub_N_sum = NaN; - sub_obs_model_diff_sum = NaN; - sub_model_sum = NaN; - sub_obs_sum = NaN; - sub_abs_sum = NaN; + // if this app isn't couchbase, use mysql + pool.query(statement, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else { + parsedData = parseQueryDataSimpleScatter( + rows, + d, + appParams, + statisticXStr, + statisticYStr + ); + d = parsedData.d; + n0 = parsedData.n0; + nTimes = parsedData.nTimes; + } + // done waiting - have results + dFuture.return(); + }); + } + // wait for the future to finish - sort of like 'back to the future' ;) + dFuture.wait(); + + return { + data: d, + error, + n0, + nTimes, + }; + } + return null; +}; + +// this method queries the database for map plots +const queryDBMapScalar = function ( + pool, + statementOrMwRows, + dataSource, + statistic, + variable, + varUnits, + siteMap, + appParams +) { + if (Meteor.isServer) { + // d will contain the curve data + let d = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + color: [], + stats: [], + text: [], + }; + let dLowest = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "", + }; + let dLow = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "", + }; + let dModerate = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "", + }; + let dHigh = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "", + }; + let dHighest = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "", + }; + let valueLimits = {}; + + let error = ""; + let parsedData; + + const Future = require("fibers/future"); + const dFuture = new Future(); + + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBSpecialtyCurve' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + let rows = null; + if (Array.isArray(statementOrMwRows)) { + rows = statementOrMwRows; + dFuture.return(); } else { - sub_values = NaN; + (async () => { + rows = await pool.queryCB(statementOrMwRows); + dFuture.return(); + })(); } - sub_secs = NaN; - if (hasLevels) { - sub_levs = NaN; + dFuture.wait(); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; + } else { + parsedData = parseQueryDataMapScalar( + rows, + d, + dLowest, + dLow, + dModerate, + dHigh, + dHighest, + dataSource, + siteMap, + statistic, + variable, + varUnits, + appParams, + true + ); + d = parsedData.d; + dLowest = parsedData.dLowest; + dLow = parsedData.dLow; + dModerate = parsedData.dModerate; + dHigh = parsedData.dHigh; + dHighest = parsedData.dHighest; + valueLimits = parsedData.valueLimits; } - } - // store flat arrays of all the parsed data, used by the text output and for some calculations later - d.xTextOutput.push(rowXVal); - d.yTextOutput.push(rowYVal); - d.zTextOutput.push(stat); - d.nTextOutput.push(n); - d.hitTextOutput.push(hit); - d.faTextOutput.push(fa); - d.missTextOutput.push(miss); - d.cnTextOutput.push(cn); - d.squareDiffSumTextOutput.push(squareDiffSum); - d.NSumTextOutput.push(NSum); - d.obsModelDiffSumTextOutput.push(obsModelDiffSum); - d.modelSumTextOutput.push(modelSum); - d.obsSumTextOutput.push(obsSum); - d.absSumTextOutput.push(absSum); - d.minDateTextOutput.push(minDate); - d.maxDateTextOutput.push(maxDate); - curveStatLookup[statKey] = stat; - curveStdevLookup[statKey] = stdev; - curveNLookup[statKey] = n; - if (isCTC) { - curveSubHitLookup[statKey] = sub_hit; - curveSubFaLookup[statKey] = sub_fa; - curveSubMissLookup[statKey] = sub_miss; - curveSubCnLookup[statKey] = sub_cn; - } else if (isScalar) { - curveSubSquareDiffSumLookup[statKey] = sub_square_diff_sum; - curveSubNSumLookup[statKey] = sub_N_sum; - curveSubObsModelDiffSumLookup[statKey] = sub_obs_model_diff_sum; - curveSubModelSumLookup[statKey] = sub_model_sum; - curveSubObsSumLookup[statKey] = sub_obs_sum; - curveSubAbsSumLookup[statKey] = sub_abs_sum; } else { - curveSubValLookup[statKey] = sub_values; - } - curveSubSecLookup[statKey] = sub_secs; - if (hasLevels) { - curveSubLevLookup[statKey] = sub_levs; + // if this app isn't couchbase, use mysql + pool.query(statementOrMwRows, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else { + parsedData = parseQueryDataMapScalar( + rows, + d, + dLowest, + dLow, + dModerate, + dHigh, + dHighest, + dataSource, + siteMap, + statistic, + variable, + varUnits, + appParams, + false + ); + d = parsedData.d; + dLowest = parsedData.dLowest; + dLow = parsedData.dLow; + dModerate = parsedData.dModerate; + dHigh = parsedData.dHigh; + dHighest = parsedData.dHighest; + valueLimits = parsedData.valueLimits; + } + // done waiting - have results + dFuture.return(); + }); + // wait for future to finish + dFuture.wait(); } + + return { + data: d, + dataLowest: dLowest, + dataLow: dLow, + dataModerate: dModerate, + dataHigh: dHigh, + dataHighest: dHighest, + valueLimits, + error, + }; } + return null; +}; - // get the unique x and y values and sort the stats into the 2D z array accordingly - d.x = matsDataUtils.arrayUnique(d.xTextOutput).sort(function (a, b) { - return a - b; - }); - d.y = matsDataUtils.arrayUnique(d.yTextOutput).sort(function (a, b) { - return a - b; - }); - let i; - let j; - let currX; - let currY; - let currStat; - let currStdev; - let currN; - let currSubHit; - let currSubFa; - let currSubMiss; - let currSubCn; - let currSubSquareDiffSum; - let currSubNSum; - let currSubObsModelDiffSum; - let currSubModelSum; - let currSubObsSum; - let currSubAbsSum; - let currSubVal; - let currSubSec; - let currSubLev; - let currStatKey; - let currYStatArray; - let currYStdevArray; - let currYNArray; - let currYSubHitArray; - let currYSubFaArray; - let currYSubMissArray; - let currYSubCnArray; - let currYSubSquareDiffSumArray; - let currYSubNSumArray; - let currYSubObsModelDiffSumArray; - let currYSubModelSumArray; - let currYSubObsSumArray; - let currYSubAbsSumArray; - let currYSubValArray; - let currYSubSecArray; - let currYSubLevArray; - let sum = 0; - let nPoints = 0; - let zmin = Number.MAX_VALUE; - let zmax = -1 * Number.MAX_VALUE; +// this method queries the database for map plots in CTC apps +const queryDBMapCTC = function ( + pool, + statementOrMwRows, + dataSource, + statistic, + siteMap, + appParams +) { + if (Meteor.isServer) { + // d will contain the curve data + let d = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + color: [], + stats: [], + text: [], + }; + // for skill scores <= 10% + let dPurple = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(128,0,255)", + }; + // for skill scores <= 20% + let dPurpleBlue = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(64,0,255)", + }; + // for skill scores <= 30% + let dBlue = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(0,0,255)", + }; + // for skill scores <= 40% + let dBlueGreen = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(64,128,128)", + }; + // for skill scores <= 50% + let dGreen = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(128,255,0)", + }; + // for skill scores <= 60% + let dGreenYellow = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(160,224,0)", + }; + // for skill scores <= 70% + let dYellow = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(192,192,0)", + }; + // for skill scores <= 80% + let dOrange = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(255,128,0)", + }; + // for skill scores <= 90% + let dOrangeRed = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(255,64,0)", + }; + // for skill scores <= 100% + let dRed = { + siteName: [], + queryVal: [], + lat: [], + lon: [], + stats: [], + text: [], + color: "rgb(255,0,0)", + }; + let valueLimits = {}; - for (j = 0; j < d.y.length; j++) { - currY = d.y[j]; - currYStatArray = []; - currYStdevArray = []; - currYNArray = []; - if (isCTC) { - currYSubHitArray = []; - currYSubFaArray = []; - currYSubMissArray = []; - currYSubCnArray = []; - } else if (isScalar) { - currYSubSquareDiffSumArray = []; - currYSubNSumArray = []; - currYSubObsModelDiffSumArray = []; - currYSubModelSumArray = []; - currYSubObsSumArray = []; - currYSubAbsSumArray = []; - } else { - currYSubValArray = []; - } - currYSubSecArray = []; - if (hasLevels) { - currYSubLevArray = []; - } - for (i = 0; i < d.x.length; i++) { - currX = d.x[i]; - currStatKey = `${currX.toString()}_${currY.toString()}`; - currStat = curveStatLookup[currStatKey]; - currStdev = curveStdevLookup[currStatKey]; - currN = curveNLookup[currStatKey]; - if (isCTC) { - currSubHit = curveSubHitLookup[currStatKey]; - currSubFa = curveSubFaLookup[currStatKey]; - currSubMiss = curveSubMissLookup[currStatKey]; - currSubCn = curveSubCnLookup[currStatKey]; - } else if (isScalar) { - currSubSquareDiffSum = curveSubSquareDiffSumLookup[currStatKey]; - currSubNSum = curveSubNSumLookup[currStatKey]; - currSubObsModelDiffSum = curveSubObsModelDiffSumLookup[currStatKey]; - currSubModelSum = curveSubModelSumLookup[currStatKey]; - currSubObsSum = curveSubObsSumLookup[currStatKey]; - currSubAbsSum = curveSubAbsSumLookup[currStatKey]; + let error = ""; + let parsedData; + + const Future = require("fibers/future"); + const dFuture = new Future(); + + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBSpecialtyCurve' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + let rows = null; + if (Array.isArray(statementOrMwRows)) { + rows = statementOrMwRows; + dFuture.return(); } else { - currSubVal = curveSubValLookup[currStatKey]; + (async () => { + rows = await pool.queryCB(statementOrMwRows); + dFuture.return(); + })(); } - currSubSec = curveSubSecLookup[currStatKey]; - if (hasLevels) { - currSubLev = curveSubLevLookup[currStatKey]; + dFuture.wait(); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; + } else { + parsedData = parseQueryDataMapCTC( + rows, + d, + dPurple, + dPurpleBlue, + dBlue, + dBlueGreen, + dGreen, + dGreenYellow, + dYellow, + dOrange, + dOrangeRed, + dRed, + dataSource, + siteMap, + statistic, + appParams, + true + ); + d = parsedData.d; + dPurple = parsedData.dPurple; + dPurpleBlue = parsedData.dPurpleBlue; + dBlue = parsedData.dBlue; + dBlueGreen = parsedData.dBlueGreen; + dGreen = parsedData.dGreen; + dGreenYellow = parsedData.dGreenYellow; + dYellow = parsedData.dYellow; + dOrange = parsedData.dOrange; + dOrangeRed = parsedData.dOrangeRed; + dRed = parsedData.dRed; + valueLimits = parsedData.valueLimits; } - if (currStat === undefined) { - currYStatArray.push(null); - currYStdevArray.push(null); - currYNArray.push(0); - if (isCTC) { - currYSubHitArray.push(null); - currYSubFaArray.push(null); - currYSubMissArray.push(null); - currYSubCnArray.push(null); - } else if (isScalar) { - currYSubSquareDiffSumArray.push(null); - currYSubNSumArray.push(null); - currYSubObsModelDiffSumArray.push(null); - currYSubModelSumArray.push(null); - currYSubObsSumArray.push(null); - currYSubAbsSumArray.push(null); + } else { + // if this app isn't couchbase, use mysql + pool.query(statementOrMwRows, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; } else { - currYSubValArray.push(null); - } - currYSubSecArray.push(null); - if (hasLevels) { - currYSubLevArray.push(null); + parsedData = parseQueryDataMapCTC( + rows, + d, + dPurple, + dPurpleBlue, + dBlue, + dBlueGreen, + dGreen, + dGreenYellow, + dYellow, + dOrange, + dOrangeRed, + dRed, + dataSource, + siteMap, + statistic, + appParams, + false + ); + d = parsedData.d; + dPurple = parsedData.dPurple; + dPurpleBlue = parsedData.dPurpleBlue; + dBlue = parsedData.dBlue; + dBlueGreen = parsedData.dBlueGreen; + dGreen = parsedData.dGreen; + dGreenYellow = parsedData.dGreenYellow; + dYellow = parsedData.dYellow; + dOrange = parsedData.dOrange; + dOrangeRed = parsedData.dOrangeRed; + dRed = parsedData.dRed; + valueLimits = parsedData.valueLimits; } - } else { - sum += currStat; - nPoints += 1; - currYStatArray.push(currStat); - currYStdevArray.push(currStdev); - currYNArray.push(currN); - if (isCTC) { - currYSubHitArray.push(currSubHit); - currYSubFaArray.push(currSubFa); - currYSubMissArray.push(currSubMiss); - currYSubCnArray.push(currSubCn); - } else if (isScalar) { - currYSubSquareDiffSumArray.push(currSubSquareDiffSum); - currYSubNSumArray.push(currSubNSum); - currYSubObsModelDiffSumArray.push(currSubObsModelDiffSum); - currYSubModelSumArray.push(currSubModelSum); - currYSubObsSumArray.push(currSubObsSum); - currYSubAbsSumArray.push(currSubAbsSum); + // done waiting - have results + dFuture.return(); + }); + // wait for future to finish + dFuture.wait(); + } + + return { + data: d, + dataPurple: dPurple, + dataPurpleBlue: dPurpleBlue, + dataBlue: dBlue, + dataBlueGreen: dBlueGreen, + dataGreen: dGreen, + dataGreenYellow: dGreenYellow, + dataYellow: dYellow, + dataOrange: dOrange, + dataOrangeRed: dOrangeRed, + dataRed: dRed, + valueLimits, + error, + }; + } + return null; +}; + +// this method queries the database for contour plots +const queryDBContour = function (pool, statement, appParams, statisticStr) { + if (Meteor.isServer) { + let d = { + // d will contain the curve data + x: [], + y: [], + z: [], + n: [], + subHit: [], + subFa: [], + subMiss: [], + subCn: [], + subSquareDiffSum: [], + subNSum: [], + subObsModelDiffSum: [], + subModelSum: [], + subObsSum: [], + subAbsSum: [], + subData: [], + subHeaders: [], + subVals: [], + subSecs: [], + subLevs: [], + text: [], + xTextOutput: [], + yTextOutput: [], + zTextOutput: [], + nTextOutput: [], + hitTextOutput: [], + faTextOutput: [], + missTextOutput: [], + cnTextOutput: [], + squareDiffSumTextOutput: [], + NSumTextOutput: [], + obsModelDiffSumTextOutput: [], + modelSumTextOutput: [], + obsSumTextOutput: [], + absSumTextOutput: [], + minDateTextOutput: [], + maxDateTextOutput: [], + stdev: [], + stats: [], + nForecast: [], + nMatched: [], + nSimple: [], + nTotal: [], + glob_stats: {}, + xmin: Number.MAX_VALUE, + xmax: Number.MIN_VALUE, + ymin: Number.MAX_VALUE, + ymax: Number.MIN_VALUE, + zmin: Number.MAX_VALUE, + zmax: Number.MIN_VALUE, + sum: 0, + }; + let error = ""; + let parsedData; + + const Future = require("fibers/future"); + const dFuture = new Future(); + + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + /* + we have to call the couchbase utilities as async functions but this + routine 'queryDBContour' cannot itself be async because the graph page needs to wait + for its result, so we use an anonymous async() function here to wrap the queryCB call + */ + (async () => { + const rows = await pool.queryCB(statement); + if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else if (rows.includes("queryCB ERROR: ")) { + error = rows; } else { - currYSubValArray.push(currSubVal); - } - currYSubSecArray.push(currSubSec); - if (hasLevels) { - currYSubLevArray.push(currSubLev); + parsedData = parseQueryDataContour(rows, d, appParams, statisticStr); + d = parsedData.d; } - zmin = currStat < zmin ? currStat : zmin; - zmax = currStat > zmax ? currStat : zmax; - } - } - d.z.push(currYStatArray); - d.stdev.push(currYStdevArray); - d.n.push(currYNArray); - if (isCTC) { - d.subHit.push(currYSubHitArray); - d.subFa.push(currYSubFaArray); - d.subMiss.push(currYSubMissArray); - d.subCn.push(currYSubCnArray); - } else if (isScalar) { - d.subSquareDiffSum.push(currYSubSquareDiffSumArray); - d.subNSum.push(currYSubNSumArray); - d.subObsModelDiffSum.push(currYSubObsModelDiffSumArray); - d.subModelSum.push(currYSubModelSumArray); - d.subObsSum.push(currYSubObsSumArray); - d.subAbsSum.push(currYSubAbsSumArray); + dFuture.return(); + })(); } else { - d.subVals.push(currYSubValArray); - } - d.subSecs.push(currYSubSecArray); - if (hasLevels) { - d.subLevs.push(currYSubLevArray); + // if this app isn't couchbase, use mysql + pool.query(statement, function (err, rows) { + // query callback - build the curve data from the results - or set an error + if (err !== undefined && err !== null) { + error = err.message; + } else if (rows === undefined || rows === null || rows.length === 0) { + error = matsTypes.Messages.NO_DATA_FOUND; + } else { + parsedData = parseQueryDataContour(rows, d, appParams, statisticStr); + d = parsedData.d; + } + // done waiting - have results + dFuture.return(); + }); } - } - - // calculate statistics - d.xmin = d.x[0]; - d.xmax = d.x[d.x.length - 1]; - d.ymin = d.y[0]; - d.ymax = d.y[d.y.length - 1]; - d.zmin = zmin; - d.zmax = zmax; - d.sum = sum; - - const filteredMinDate = d.minDateTextOutput.filter((t) => t || t === 0); - const filteredMaxDate = d.maxDateTextOutput.filter((t) => t || t === 0); - d.glob_stats.mean = sum / nPoints; - d.glob_stats.minDate = Math.min(...filteredMinDate); - d.glob_stats.maxDate = Math.max(...filteredMaxDate); - d.glob_stats.n = nPoints; + // wait for the future to finish - sort of like 'back to the future' ;) + dFuture.wait(); - return { - d, - }; + return { + data: d, + error, + }; + } + return null; }; +// eslint-disable-next-line no-undef export default matsDataQueryUtils = { simplePoolQueryWrapSynchronous, getStationsInCouchbaseRegion, diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_common.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_common.js index 80a64f242..7140937ed 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_common.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_common.js @@ -169,7 +169,7 @@ class MatsMiddleCommon { const varValM = modelSingleFve.stations[station]; if (varValO && varValM) { - ctc.N0 += 1; + ctc.n0 += 1; let sub = `${fve};`; if (varValO < threshold && varValM < threshold) { ctc.hit += 1; @@ -217,7 +217,7 @@ class MatsMiddleCommon { const obsSum = varValO; const absSum = Math.abs(varValO - varValM); - sums.N0 += 1; + sums.n0 += 1; sums.square_diff_sum += squareDiffSum; sums.N_sum += nSum; sums.obs_model_diff_sum += obsModelDiffSum; diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js index 742b867fc..19d1db948 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dailyModelCycle.js @@ -306,8 +306,8 @@ class MatsMiddleDailyModelCycle { ctcFve.miss = 0; ctcFve.fa = 0; ctcFve.cn = 0; - ctcFve.N0 = 0; - ctcFve.N_times = 1; + ctcFve.n0 = 0; + ctcFve.nTimes = 1; ctcFve.sub_data = []; this.mmCommon.computeCtcForStations( @@ -318,7 +318,7 @@ class MatsMiddleDailyModelCycle { obsSingleFve, modelSingleFve ); - if (ctcFve.N0 > 0) { + if (ctcFve.n0 > 0) { this.stats.push(ctcFve); } } @@ -343,8 +343,8 @@ class MatsMiddleDailyModelCycle { sumsFve.model_sum = 0; sumsFve.obs_sum = 0; sumsFve.abs_sum = 0; - sumsFve.N0 = 0; - sumsFve.N_times = 1; + sumsFve.n0 = 0; + sumsFve.nTimes = 1; sumsFve.sub_data = []; this.mmCommon.computeSumsForStations( @@ -354,7 +354,7 @@ class MatsMiddleDailyModelCycle { obsSingleFve, modelSingleFve ); - if (sumsFve.N0 > 0) { + if (sumsFve.n0 > 0) { this.stats.push(sumsFve); } } diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js index f7e6451c2..060a7b6b4 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_dieoff.js @@ -367,8 +367,8 @@ class MatsMiddleDieoff { ctcFcstLead.miss = 0; ctcFcstLead.fa = 0; ctcFcstLead.cn = 0; - ctcFcstLead.N0 = 0; - ctcFcstLead.N_times = new Set(fcstLeadArray).size; + ctcFcstLead.n0 = 0; + ctcFcstLead.nTimes = new Set(fcstLeadArray).size; ctcFcstLead.sub_data = []; // get all the fve for this fcst_lead @@ -443,8 +443,8 @@ class MatsMiddleDieoff { sumsFcstLead.model_sum = 0; sumsFcstLead.obs_sum = 0; sumsFcstLead.abs_sum = 0; - sumsFcstLead.N0 = 0; - sumsFcstLead.N_times = new Set(fcstLeadArray).size; + sumsFcstLead.n0 = 0; + sumsFcstLead.nTimes = new Set(fcstLeadArray).size; sumsFcstLead.sub_data = []; // get all the fve for this fcst_lead diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_map.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_map.js index 0507ae108..527be6589 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_map.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_map.js @@ -319,8 +319,8 @@ class MatsMiddleMap { ctcFve.miss = 0; ctcFve.fa = 0; ctcFve.cn = 0; - ctcFve.N0 = 0; - ctcFve.N_times = 0; + ctcFve.n0 = 0; + ctcFve.nTimes = 0; ctcFve.sub_data = []; [ctcFve.min_secs] = this.fcstValidEpoch_Array; ctcFve.max_secs = @@ -341,8 +341,8 @@ class MatsMiddleMap { this.validTimes.length > 0 && this.validTimes.includes((fve % (24 * 3600)) / 3600))) ) { - ctcFve.N0 += 1; - ctcFve.N_times += 1; + ctcFve.n0 += 1; + ctcFve.nTimes += 1; let sub = `${fve};`; if (varValO < threshold && varValM < threshold) { @@ -375,7 +375,7 @@ class MatsMiddleMap { // stats_fve.sub_data.push(sub); } } - if (ctcFve.N0 > 0) { + if (ctcFve.n0 > 0) { const sub = `${this.fcstValidEpoch_Array[0]};${ctcFve.hit};${ctcFve.fa};${ctcFve.miss};${ctcFve.cn}`; ctcFve.sub_data.push(sub); this.stats.push(ctcFve); @@ -399,8 +399,8 @@ class MatsMiddleMap { sumsFve.model_sum = 0; sumsFve.obs_sum = 0; sumsFve.abs_sum = 0; - sumsFve.N0 = 0; - sumsFve.N_times = 0; + sumsFve.n0 = 0; + sumsFve.nTimes = 0; sumsFve.sub_data = []; [sumsFve.min_secs] = this.fcstValidEpoch_Array; sumsFve.max_secs = @@ -421,8 +421,8 @@ class MatsMiddleMap { this.validTimes.length > 0 && this.validTimes.includes((fve % (24 * 3600)) / 3600))) ) { - sumsFve.N0 += 1; - sumsFve.N_times += 1; + sumsFve.n0 += 1; + sumsFve.nTimes += 1; if (varValO && varValM) { sumsFve.square_diff_sum += (varValO - varValM) ** 2; @@ -434,7 +434,7 @@ class MatsMiddleMap { } } } - if (sumsFve.N0 > 0) { + if (sumsFve.n0 > 0) { const sub = `${this.fcstValidEpoch_Array[0]};${sumsFve.square_diff_sum};${sumsFve.N_sum};${sumsFve.obs_model_diff_sum};${sumsFve.model_sum};${sumsFve.obs_sum};${sumsFve.abs_sum}`; sumsFve.sub_data.push(sub); this.stats.push(sumsFve); diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js index 681b4cece..fa00f7e00 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_timeSeries.js @@ -343,7 +343,7 @@ class MatsMiddleTimeSeries { ctcAvtime.miss = 0; ctcAvtime.fa = 0; ctcAvtime.cn = 0; - ctcAvtime.N0 = 0; + ctcAvtime.n0 = 0; ctcAvtime.sub_data = []; // get all the fve for this avtime @@ -353,7 +353,7 @@ class MatsMiddleTimeSeries { [ctcAvtime.min_secs] = fveArray; ctcAvtime.max_secs = fveArray[fveArray.length - 1]; - ctcAvtime.N_times = fveArray.length; + ctcAvtime.nTimes = fveArray.length; for (let imfve = 0; imfve < fveArray.length; imfve += 1) { const fve = fveArray[imfve]; const obsSingleFve = this.fveObs[avtimeKey][fve]; @@ -406,7 +406,7 @@ class MatsMiddleTimeSeries { sumsAvtime.model_sum = 0; sumsAvtime.obs_sum = 0; sumsAvtime.abs_sum = 0; - sumsAvtime.N0 = 0; + sumsAvtime.n0 = 0; sumsAvtime.sub_data = []; // get all the fve for this avtime @@ -416,7 +416,7 @@ class MatsMiddleTimeSeries { [sumsAvtime.min_secs] = fveArray; sumsAvtime.max_secs = fveArray[fveArray.length - 1]; - sumsAvtime.N_times = fveArray.length; + sumsAvtime.nTimes = fveArray.length; for (let imfve = 0; imfve < fveArray.length; imfve += 1) { const fve = fveArray[imfve]; const obsSingleFve = this.fveObs[avtimeKey][fve]; diff --git a/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js b/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js index 0f5c9224a..991975ce9 100644 --- a/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js +++ b/meteor_packages/mats-common/imports/startup/server/matsMiddle_validTime.js @@ -322,7 +322,7 @@ class MatsMiddleValidTime { ctcHod.miss = 0; ctcHod.fa = 0; ctcHod.cn = 0; - ctcHod.N0 = 0; + ctcHod.n0 = 0; ctcHod.sub_data = []; // get all the fve for this hod @@ -332,7 +332,7 @@ class MatsMiddleValidTime { [ctcHod.min_secs] = fveArray; ctcHod.max_secs = fveArray[fveArray.length - 1]; - ctcHod.N_times = fveArray.length; + ctcHod.nTimes = fveArray.length; for (let imfve = 0; imfve < fveArray.length; imfve += 1) { const fve = fveArray[imfve]; const obsSingleFve = this.fveObs[hodKey][fve]; @@ -377,7 +377,7 @@ class MatsMiddleValidTime { sumsHod.model_sum = 0; sumsHod.obs_sum = 0; sumsHod.abs_sum = 0; - sumsHod.N0 = 0; + sumsHod.n0 = 0; sumsHod.sub_data = []; // get all the fve for this hod @@ -387,7 +387,7 @@ class MatsMiddleValidTime { [sumsHod.min_secs] = fveArray; sumsHod.max_secs = fveArray[fveArray.length - 1]; - sumsHod.N_times = fveArray.length; + sumsHod.nTimes = fveArray.length; for (let imfve = 0; imfve < fveArray.length; imfve += 1) { const fve = fveArray[imfve]; const obsSingleFve = this.fveObs[hodKey][fve]; diff --git a/meteor_packages/mats-common/public/python/python_query_util.py b/meteor_packages/mats-common/public/python/python_query_util.py index f706ebafa..c0c4187af 100644 --- a/meteor_packages/mats-common/public/python/python_query_util.py +++ b/meteor_packages/mats-common/public/python/python_query_util.py @@ -135,7 +135,7 @@ class QueryUtil: returned data. In the future, we plan to split this into two classes, one for querying and one for statistics.""" error = [] # one of the four fields to return at the end -- records any error message n0 = [] # one of the four fields to return at the end -- number of sub_values for each independent variable - n_times = [] # one of the four fields to return at the end -- number of sub_secs for each independent variable + nTimes = [] # one of the four fields to return at the end -- number of sub_secs for each independent variable data = [] # one of the four fields to return at the end -- the parsed data structure output_JSON = {} # JSON structure to pass the five output fields back to the MATS JS @@ -185,10 +185,10 @@ def set_up_output_fields(self, number_of_curves): "threshold_all": [], "oy_all": [], "on_all": [], - "n_forecast": [], - "n_matched": [], - "n_simple": [], - "n_total": [], + "nForecast": [], + "nMatched": [], + "nSimple": [], + "nTotal": [], "sample_climo": 0, "auc": 0, "glob_stats": { @@ -207,7 +207,7 @@ def set_up_output_fields(self, number_of_curves): "sum": 0 }) self.n0.append([]) - self.n_times.append([]) + self.nTimes.append([]) self.error.append("") def construct_output_json(self, plot_type, queries): @@ -230,24 +230,24 @@ def construct_output_json(self, plot_type, queries): elif stat_line_type == 'mode_single': for j in range(len(self.data[i]["subData"])): if self.data[i]["subHeaders"][j] == 'NaN' or len(self.data[i]["subHeaders"][j]) == 0: - self.data[i]["n_forecast"].append(0) - self.data[i]["n_matched"].append(0) - self.data[i]["n_simple"].append(0) - self.data[i]["n_total"].append(0) + self.data[i]["nForecast"].append(0) + self.data[i]["nMatched"].append(0) + self.data[i]["nSimple"].append(0) + self.data[i]["nTotal"].append(0) else: try: forecast_idx = self.data[i]["subHeaders"][j].index('fcst_flag') matched_idx = self.data[i]["subHeaders"][j].index('matched_flag') simple_idx = self.data[i]["subHeaders"][j].index('simple_flag') - self.data[i]["n_forecast"].append(sum([int(a[forecast_idx]) for a in self.data[i]["subData"][j]])) - self.data[i]["n_matched"].append(sum([int(a[matched_idx]) for a in self.data[i]["subData"][j]])) - self.data[i]["n_simple"].append(sum([int(a[simple_idx]) for a in self.data[i]["subData"][j]])) - self.data[i]["n_total"].append(len([int(a[forecast_idx]) for a in self.data[i]["subData"][j]])) + self.data[i]["nForecast"].append(sum([int(a[forecast_idx]) for a in self.data[i]["subData"][j]])) + self.data[i]["nMatched"].append(sum([int(a[matched_idx]) for a in self.data[i]["subData"][j]])) + self.data[i]["nSimple"].append(sum([int(a[simple_idx]) for a in self.data[i]["subData"][j]])) + self.data[i]["nTotal"].append(len([int(a[forecast_idx]) for a in self.data[i]["subData"][j]])) except Exception as e: - self.data[i]["n_forecast"].append(0) - self.data[i]["n_matched"].append(0) - self.data[i]["n_simple"].append(0) - self.data[i]["n_total"].append(0) + self.data[i]["nForecast"].append(0) + self.data[i]["nMatched"].append(0) + self.data[i]["nSimple"].append(0) + self.data[i]["nTotal"].append(0) elif stat_line_type == 'ctc': for j in range(len(self.data[i]["subData"])): if self.data[i]["subHeaders"][j] == 'NaN' or len(self.data[i]["subHeaders"][j]) == 0: @@ -276,8 +276,8 @@ def construct_output_json(self, plot_type, queries): self.output_JSON = { "data": self.data, - "N0": self.n0, - "N_times": self.n_times, + "n0": self.n0, + "nTimes": self.nTimes, "error": self.error } self.output_JSON = json.dumps(self.output_JSON, cls=NpEncoder) @@ -358,11 +358,11 @@ def parse_query_data_xy_curve(self, idx, cursor, stat_line_type, statistic, app_ data_exists = row['area'] != "null" and row['area'] != "NULL" else: data_exists = row['stat'] != "null" and row['stat'] != "NULL" - if hasattr(row, 'N0'): - self.n0[idx].append(int(row['N0'])) + if hasattr(row, 'n0'): + self.n0[idx].append(int(row['n0'])) else: - self.n0[idx].append(int(row['N_times'])) - self.n_times[idx].append(int(row['N_times'])) + self.n0[idx].append(int(row['nTimes'])) + self.nTimes[idx].append(int(row['nTimes'])) if plot_type == 'TimeSeries' and row_idx < len(query_data) - 1: # make sure we have the smallest time interval for the while loop later time_diff = int(query_data[row_idx + 1]['avtime']) - int(row['avtime']) @@ -428,7 +428,7 @@ def parse_query_data_xy_curve(self, idx, cursor, stat_line_type, statistic, app_ *sorted(zip(curve_ind_vars, curve_stats, sub_data_all, sub_headers_all, sub_vals_all, sub_secs_all))) n0_max = max(self.n0[idx]) - n_times_max = max(self.n_times[idx]) + n_times_max = max(self.nTimes[idx]) time_interval = time_interval * 1000 loop_sum = 0 dep_var_min = sys.float_info.max @@ -449,7 +449,7 @@ def parse_query_data_xy_curve(self, idx, cursor, stat_line_type, statistic, app_ # for any bad data points along the curve. d_idx = curve_ind_vars.index(ind_var) this_n0 = self.n0[idx][d_idx] - this_n_times = self.n_times[idx][d_idx] + this_n_times = self.nTimes[idx][d_idx] # add a null if there were too many missing sub-values if curve_stats[d_idx] == 'null' or this_n_times < completeness_qc_param * n_times_max: if not hide_gaps: @@ -585,11 +585,11 @@ def parse_query_data_histogram(self, idx, cursor, stat_line_type, statistic, app data_exists = row['fss'] != "null" and row['fss'] != "NULL" else: data_exists = row['stat'] != "null" and row['stat'] != "NULL" - if hasattr(row, 'N0'): - self.n0[idx].append(int(row['N0'])) + if hasattr(row, 'n0'): + self.n0[idx].append(int(row['n0'])) else: - self.n0[idx].append(int(row['N_times'])) - self.n_times[idx].append(int(row['N_times'])) + self.n0[idx].append(int(row['nTimes'])) + self.nTimes[idx].append(int(row['nTimes'])) if data_exists: stat, sub_levs, sub_secs, sub_values, sub_data, sub_headers, self.error[idx] \ @@ -655,11 +655,11 @@ def parse_query_data_ensemble_histogram(self, idx, cursor, stat_line_type, stati if data_exists: bin_number = int(row['bin']) bin_count = int(row['bin_count']) - if hasattr(row, 'N0'): - self.n0[idx].append(int(row['N0'])) + if hasattr(row, 'n0'): + self.n0[idx].append(int(row['n0'])) else: - self.n0[idx].append(int(row['N_times'])) - self.n_times[idx].append(int(row['N_times'])) + self.n0[idx].append(int(row['nTimes'])) + self.nTimes[idx].append(int(row['nTimes'])) # this function deals with rhist/phist/relp and rhist_rank/phist_bin/relp_ens tables stat, sub_levs, sub_secs, sub_values, sub_data, sub_headers, self.error[idx] \ @@ -746,11 +746,11 @@ def parse_query_data_ensemble(self, idx, cursor, app_params): threshold = row['threshold'] oy = int(row['oy_i']) on = int(row['on_i']) - number_times = int(row['N_times']) - if hasattr(row, 'N0'): - number_values = int(row['N0']) + number_times = int(row['nTimes']) + if hasattr(row, 'n0'): + number_values = int(row['n0']) else: - number_values = int(row['N_times']) + number_values = int(row['nTimes']) # we must add up all of the observed and not-observed values for each probability bin observed_total = observed_total + oy @@ -783,7 +783,7 @@ def parse_query_data_ensemble(self, idx, cursor, app_params): # Since everything is combined already, put it into the data structure self.n0[idx] = total_values - self.n_times[idx] = total_times + self.nTimes[idx] = total_times self.data[idx]['x'] = ens_stats[ens_stats["x_var"]] self.data[idx]['y'] = ens_stats[ens_stats["y_var"]] self.data[idx]['sample_climo'] = ens_stats["sample_climo"] diff --git a/meteor_packages/mats-common/templates/textOutput/textOutput.js b/meteor_packages/mats-common/templates/textOutput/textOutput.js index 0b23d89e5..cccf039d7 100644 --- a/meteor_packages/mats-common/templates/textOutput/textOutput.js +++ b/meteor_packages/mats-common/templates/textOutput/textOutput.js @@ -159,7 +159,7 @@ Template.textOutput.helpers({ const isModeSingle = curveData !== undefined && curveData[0] !== undefined && - Object.keys(curveData[0]).indexOf("n_forecast") !== -1; + Object.keys(curveData[0]).indexOf("nForecast") !== -1; const isModePairs = curveData !== undefined && curveData[0] !== undefined && @@ -398,7 +398,7 @@ Template.textOutput.helpers({ let line = ""; const isCTC = element.hit !== undefined && element.hit !== null; const isModeSingle = - element.n_forecast !== undefined && element.n_forecast !== null; + element.nForecast !== undefined && element.nForecast !== null; const isModePairs = element.avgInterest !== undefined && element.avgInterest !== null; const plotType = Session.get("plotType"); @@ -472,23 +472,23 @@ Template.textOutput.helpers({ : fillStr }` + `${ - element.n_forecast !== undefined && element.n_forecast !== null - ? element.n_forecast.toString() + element.nForecast !== undefined && element.nForecast !== null + ? element.nForecast.toString() : fillStr }` + `${ - element.n_matched !== undefined && element.n_matched !== null - ? element.n_matched.toString() + element.nMatched !== undefined && element.nMatched !== null + ? element.nMatched.toString() : fillStr }` + `${ - element.n_simple !== undefined && element.n_simple !== null - ? element.n_simple.toString() + element.nSimple !== undefined && element.nSimple !== null + ? element.nSimple.toString() : fillStr }` + `${ - element.n_total !== undefined && element.n_total !== null - ? element.n_total.toString() + element.nTotal !== undefined && element.nTotal !== null + ? element.nTotal.toString() : fillStr }`; } else if (isModePairs) { @@ -588,23 +588,23 @@ Template.textOutput.helpers({ : fillStr }` + `${ - element.n_forecast !== undefined && element.n_forecast !== null - ? element.n_forecast.toString() + element.nForecast !== undefined && element.nForecast !== null + ? element.nForecast.toString() : fillStr }` + `${ - element.n_matched !== undefined && element.n_matched !== null - ? element.n_matched.toString() + element.nMatched !== undefined && element.nMatched !== null + ? element.nMatched.toString() : fillStr }` + `${ - element.n_simple !== undefined && element.n_simple !== null - ? element.n_simple.toString() + element.nSimple !== undefined && element.nSimple !== null + ? element.nSimple.toString() : fillStr }` + `${ - element.n_total !== undefined && element.n_total !== null - ? element.n_total.toString() + element.nTotal !== undefined && element.nTotal !== null + ? element.nTotal.toString() : fillStr }`; } else if (isModePairs) { From a7f944f2322a94e0b5a923cc4caef0f180541a0e Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Wed, 14 Aug 2024 12:00:36 -0600 Subject: [PATCH 34/55] Linted the rest of the server routines except for middleware --- .../imports/startup/api/matsMethods.js | 14 +- .../imports/startup/server/cache.js | 17 +- .../imports/startup/server/cb_utilities.js | 35 +- .../startup/server/data_curve_ops_util.js | 146 +- .../imports/startup/server/data_diff_util.js | 10 +- .../imports/startup/server/data_match_util.js | 574 ++++---- .../startup/server/data_plot_ops_util.js | 46 +- .../startup/server/data_process_util.js | 1180 ++++++++++------- .../imports/startup/server/data_query_util.js | 6 +- .../imports/startup/server/data_util.js | 1055 +++++++-------- .../imports/startup/server/publications.js | 4 +- .../templates/textOutput/textOutput.js | 3 +- 12 files changed, 1704 insertions(+), 1386 deletions(-) diff --git a/meteor_packages/mats-common/imports/startup/api/matsMethods.js b/meteor_packages/mats-common/imports/startup/api/matsMethods.js index f1b806a49..bb8e5d4e0 100644 --- a/meteor_packages/mats-common/imports/startup/api/matsMethods.js +++ b/meteor_packages/mats-common/imports/startup/api/matsMethods.js @@ -1493,14 +1493,14 @@ const _getFlattenedResultData = function (rk, p, np) { } var stats = {}; stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.d_mean; + stats.mean = data[ci].glob_stats.dMean; stats["standard deviation"] = data[ci].glob_stats.sd; - stats.n = data[ci].glob_stats.n_good; + stats.n = data[ci].glob_stats.nGood; if ( plotType === matsTypes.PlotTypes.timeSeries || plotType === matsTypes.PlotTypes.profile ) { - stats["standard error"] = data[ci].glob_stats.stde_betsy; + stats["standard error"] = data[ci].glob_stats.stdeBetsy; stats.lag1 = data[ci].glob_stats.lag1; } stats.minimum = data[ci].glob_stats.minVal; @@ -1541,10 +1541,10 @@ const _getFlattenedResultData = function (rk, p, np) { plotType === matsTypes.PlotTypes.timeSeries || plotType === matsTypes.PlotTypes.profile ) { - curveDataElement["std error"] = data[ci].stats[cdi].stde_betsy; + curveDataElement["std error"] = data[ci].stats[cdi].stdeBetsy; curveDataElement.lag1 = data[ci].stats[cdi].lag1; } - curveDataElement.n = data[ci].stats[cdi].n_good; + curveDataElement.n = data[ci].stats[cdi].nGood; } curveData.push(curveDataElement); } @@ -1774,9 +1774,9 @@ const _getFlattenedResultData = function (rk, p, np) { } var stats = {}; stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.d_mean; + stats.mean = data[ci].glob_stats.dMean; stats["standard deviation"] = data[ci].glob_stats.sd; - stats.n = data[ci].glob_stats.n_good; + stats.n = data[ci].glob_stats.nGood; stats.minimum = data[ci].glob_stats.minVal; stats.maximum = data[ci].glob_stats.maxVal; returnData.stats[data[ci].label] = stats; diff --git a/meteor_packages/mats-common/imports/startup/server/cache.js b/meteor_packages/mats-common/imports/startup/server/cache.js index ebcb21e7c..b5cbaa806 100644 --- a/meteor_packages/mats-common/imports/startup/server/cache.js +++ b/meteor_packages/mats-common/imports/startup/server/cache.js @@ -3,33 +3,40 @@ */ import { Meteor } from "meteor/meteor"; -import { matsCollections } from "meteor/randyp:mats-common"; + +/* eslint-disable global-require */ + +let getResult; +let storeResult; +let clear; +let expireKey; if (Meteor.isServer) { const Results = require("node-file-cache").create({ file: "fileCache", life: 8 * 3600, }); - var getResult = function (key) { + getResult = function (key) { // console.log('asked to get result from cache for key:', key); const result = Results.get(key); return result === null ? undefined : result; }; - var storeResult = function (key, result) { + storeResult = function (key, result) { // console.log('asked to set result in cache for app: ',process.env.PWD, ' key:', key); Results.set(key, result); // console.log('set result in cache for app: ', process.env.PWD, 'key:', key); }; - var clear = function () { + clear = function () { // console.log('asked to clear result cache'); Results.clear(); }; - var expireKey = function (key) { + expireKey = function (key) { // console.log('asked to clear result cache for key ', key); Results.expire(key); }; } +// eslint-disable-next-line no-undef export default matsCache = { getResult, storeResult, diff --git a/meteor_packages/mats-common/imports/startup/server/cb_utilities.js b/meteor_packages/mats-common/imports/startup/server/cb_utilities.js index e8d9a3284..19cefc109 100644 --- a/meteor_packages/mats-common/imports/startup/server/cb_utilities.js +++ b/meteor_packages/mats-common/imports/startup/server/cb_utilities.js @@ -9,6 +9,10 @@ class CBUtilities { this.conn = undefined; } + /* eslint-disable global-require */ + /* eslint-disable no-console */ + /* eslint-disable class-methods-use-this */ + // The app doesn't directly require couchbase, but it does need to know about the DurabilityLevel enum // Follow the pattern below to define other enums that are needed by the app. // see https://docs.couchbase.com/sdk-api/couchbase-node-client/enums/DurabilityLevel.html @@ -63,12 +67,9 @@ class CBUtilities { }; upsertCB = async (key, doc, options = {}) => { - const couchbase = require("couchbase"); try { const conn = await this.getConnection(); - let result; - result = await conn.collection.upsert(key, doc, options); - return result; + return await conn.collection.upsert(key, doc, options); } catch (err) { console.log("upsertCB ERROR: ", err); throw new Error(`upsertCB ERROR: ${err}`); @@ -76,7 +77,6 @@ class CBUtilities { }; removeCB = async (key) => { - const couchbase = require("couchbase"); try { const conn = await this.getConnection(); const result = await conn.collection.remove(key); @@ -88,7 +88,6 @@ class CBUtilities { }; getCB = async (key) => { - const couchbase = require("couchbase"); try { const conn = await this.getConnection(); const result = await conn.collection.get(key); @@ -100,7 +99,6 @@ class CBUtilities { }; queryCB = async (statement) => { - const couchbase = require("couchbase"); const Future = require("fibers/future"); try { let result; @@ -132,20 +130,20 @@ class CBUtilities { }; searchStationsByBoundingBox = async ( - topleft_lon, - topleft_lat, - bottomright_lon, - bottomright_lat + topLeftLon, + topLeftLat, + bottomRightLon, + bottomRightLat ) => { const couchbase = require("couchbase"); const index = "station_geo"; try { const conn = await this.getConnection(); const geoBoundingBoxQuery = couchbase.SearchQuery.geoBoundingBox( - topleft_lon, - topleft_lat, - bottomright_lon, - bottomright_lat + topLeftLon, + topLeftLat, + bottomRightLon, + bottomRightLat ); const results = await conn.cluster.searchQuery(index, geoBoundingBoxQuery, { fields: ["*"], @@ -188,8 +186,8 @@ class CBUtilities { trfmSQLRemoveClause = (sqlstr, clauseFragment) => { const lines = sqlstr.split("\n"); let rv = ""; - for (let i = 0; i < lines.length; i++) { - if (lines[i].includes(clauseFragment) == false) { + for (let i = 0; i < lines.length; i += 1) { + if (!lines[i].includes(clauseFragment)) { rv = `${rv + lines[i]}\n`; } } @@ -197,8 +195,9 @@ class CBUtilities { }; } -test = async () => {}; +const test = async () => {}; +// eslint-disable-next-line no-undef export default matsCouchbaseUtils = { CBUtilities, test, diff --git a/meteor_packages/mats-common/imports/startup/server/data_curve_ops_util.js b/meteor_packages/mats-common/imports/startup/server/data_curve_ops_util.js index b74d5c201..2d18f425f 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_curve_ops_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_curve_ops_util.js @@ -33,8 +33,8 @@ const getHorizontalValueLine = function ( subSecs: [], subLevs: [], stats: [ - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, ], xmin, xmax, @@ -82,8 +82,8 @@ const getVerticalValueLine = function ( subSecs: [], subLevs: [], stats: [ - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, ], xmin: xValue, xmax: xValue, @@ -136,9 +136,9 @@ const getLinearValueLine = function ( subSecs: [], subLevs: [], stats: [ - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, ], xmin, xmax, @@ -187,8 +187,8 @@ const getDashedLinearValueLine = function ( subSecs: [], subLevs: [], stats: [ - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, ], xmin, xmax, @@ -235,8 +235,8 @@ const getCurveLine = function ( subSecs: [], subLevs: [], stats: [ - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, - { d_mean: 0, sd: 0, n_good: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, + { dMean: 0, sd: 0, nGood: 0, lag1: 0, stde: 0 }, ], xmin: Math.min(xvals), xmax: Math.max(xvals), @@ -264,6 +264,7 @@ const generateSeriesCurveOptions = function ( dataSeries, appParams ) { + const thisAxisMap = axisMap; const { label } = curve; const longLabel = matsPlotUtils.getCurveText(appParams.plotType, curve); const { annotation } = curve; @@ -274,15 +275,19 @@ const generateSeriesCurveOptions = function ( const { xmin } = curve; const { xmax } = curve; const { axisKey } = curve; - if (axisKey in axisMap) { - axisMap[axisKey].axisLabel = axisKey; - axisMap[axisKey].ymin = ymin < axisMap[axisKey].ymin ? ymin : axisMap[axisKey].ymin; - axisMap[axisKey].ymax = ymax > axisMap[axisKey].ymax ? ymax : axisMap[axisKey].ymax; - axisMap[axisKey].xmin = xmin < axisMap[axisKey].xmin ? xmin : axisMap[axisKey].xmin; - axisMap[axisKey].xmax = xmax > axisMap[axisKey].xmax ? xmax : axisMap[axisKey].xmax; + if (axisKey in thisAxisMap) { + thisAxisMap[axisKey].axisLabel = axisKey; + thisAxisMap[axisKey].ymin = + ymin < thisAxisMap[axisKey].ymin ? ymin : thisAxisMap[axisKey].ymin; + thisAxisMap[axisKey].ymax = + ymax > thisAxisMap[axisKey].ymax ? ymax : thisAxisMap[axisKey].ymax; + thisAxisMap[axisKey].xmin = + xmin < thisAxisMap[axisKey].xmin ? xmin : thisAxisMap[axisKey].xmin; + thisAxisMap[axisKey].xmax = + xmax > thisAxisMap[axisKey].xmax ? xmax : thisAxisMap[axisKey].xmax; } else { - axisMap[axisKey] = { - index: Object.keys(axisMap).length + 1, + thisAxisMap[axisKey] = { + index: Object.keys(thisAxisMap).length + 1, xmin, xmax, ymin, @@ -291,9 +296,9 @@ const generateSeriesCurveOptions = function ( }; } - const axisNumber = Object.keys(axisMap).indexOf(axisKey); + const axisNumber = Object.keys(thisAxisMap).indexOf(axisKey); - const error_y_temp = { + const errorYTemp = { error_y: { array: dataSeries.error_y, thickness: 1, // set the thickness of the error bars @@ -331,7 +336,7 @@ const generateSeriesCurveOptions = function ( delete curveOptions.error_y; - curveOptions.error_y = error_y_temp.error_y; + curveOptions.error_y = errorYTemp.error_y; // for performance diagram point filtering, need to know what we're plotting if ( @@ -366,6 +371,7 @@ const generateProfileCurveOptions = function ( dataProfile, appParams ) { + const thisAxisMap = axisMap; const { label } = curve; const longLabel = matsPlotUtils.getCurveText(appParams.plotType, curve); const { annotation } = curve; @@ -376,15 +382,19 @@ const generateProfileCurveOptions = function ( const { xmin } = curve; const { xmax } = curve; const { axisKey } = curve; - if (axisKey in axisMap) { - axisMap[axisKey].axisLabel = axisKey; - axisMap[axisKey].ymin = ymin < axisMap[axisKey].ymin ? ymin : axisMap[axisKey].ymin; - axisMap[axisKey].ymax = ymax > axisMap[axisKey].ymax ? ymax : axisMap[axisKey].ymax; - axisMap[axisKey].xmin = xmin < axisMap[axisKey].xmin ? xmin : axisMap[axisKey].xmin; - axisMap[axisKey].xmax = xmax > axisMap[axisKey].xmax ? xmax : axisMap[axisKey].xmax; + if (axisKey in thisAxisMap) { + thisAxisMap[axisKey].axisLabel = axisKey; + thisAxisMap[axisKey].ymin = + ymin < thisAxisMap[axisKey].ymin ? ymin : thisAxisMap[axisKey].ymin; + thisAxisMap[axisKey].ymax = + ymax > thisAxisMap[axisKey].ymax ? ymax : thisAxisMap[axisKey].ymax; + thisAxisMap[axisKey].xmin = + xmin < thisAxisMap[axisKey].xmin ? xmin : thisAxisMap[axisKey].xmin; + thisAxisMap[axisKey].xmax = + xmax > thisAxisMap[axisKey].xmax ? xmax : thisAxisMap[axisKey].xmax; } else { - axisMap[axisKey] = { - index: Object.keys(axisMap).length + 1, + thisAxisMap[axisKey] = { + index: Object.keys(thisAxisMap).length + 1, xmin, xmax, ymin, @@ -393,9 +403,9 @@ const generateProfileCurveOptions = function ( }; } - const axisNumber = Object.keys(axisMap).indexOf(axisKey); + const axisNumber = Object.keys(thisAxisMap).indexOf(axisKey); - const error_x_temp = { + const errorXTemp = { error_x: { array: dataProfile.error_x, thickness: 1, // set the thickness of the error bars @@ -432,7 +442,7 @@ const generateProfileCurveOptions = function ( delete curveOptions.error_x; - curveOptions.error_x = error_x_temp.error_x; + curveOptions.error_x = errorXTemp.error_x; return curveOptions; }; @@ -445,6 +455,7 @@ const generateBarChartCurveOptions = function ( dataBars, appParams ) { + const thisAxisMap = axisMap; const { label } = curve; const longLabel = matsPlotUtils.getCurveText(appParams.plotType, curve); const { annotation } = curve; @@ -455,15 +466,19 @@ const generateBarChartCurveOptions = function ( const { xmin } = dataBars; const { xmax } = dataBars; const { axisKey } = curve; - if (axisKey in axisMap) { - axisMap[axisKey].axisLabel = axisKey; - axisMap[axisKey].ymin = ymin < axisMap[axisKey].ymin ? ymin : axisMap[axisKey].ymin; - axisMap[axisKey].ymax = ymax > axisMap[axisKey].ymax ? ymax : axisMap[axisKey].ymax; - axisMap[axisKey].xmin = xmin < axisMap[axisKey].xmin ? xmin : axisMap[axisKey].xmin; - axisMap[axisKey].xmax = xmax > axisMap[axisKey].xmax ? xmax : axisMap[axisKey].xmax; + if (axisKey in thisAxisMap) { + thisAxisMap[axisKey].axisLabel = axisKey; + thisAxisMap[axisKey].ymin = + ymin < thisAxisMap[axisKey].ymin ? ymin : thisAxisMap[axisKey].ymin; + thisAxisMap[axisKey].ymax = + ymax > thisAxisMap[axisKey].ymax ? ymax : thisAxisMap[axisKey].ymax; + thisAxisMap[axisKey].xmin = + xmin < thisAxisMap[axisKey].xmin ? xmin : thisAxisMap[axisKey].xmin; + thisAxisMap[axisKey].xmax = + xmax > thisAxisMap[axisKey].xmax ? xmax : thisAxisMap[axisKey].xmax; } else { - axisMap[axisKey] = { - index: Object.keys(axisMap).length + 1, + thisAxisMap[axisKey] = { + index: Object.keys(thisAxisMap).length + 1, xmin, xmax, ymin, @@ -530,7 +545,7 @@ const generateMapCurveOptions = function (curve, dataSeries, appParams, maxValue }; const generateCTCMapCurveOptions = function (curve, dataSeries, appParams) { - const markerSizes = dataSeries.queryVal.map(function (val) { + const markerSizes = dataSeries.queryVal.map(function () { return 10; }); @@ -605,7 +620,7 @@ const generateContourCurveOptions = function (curve, axisMap, dataset, appParams [0.7, "rgb(230,145,90)"], [1, "rgb(178,10,28)"], ]; - const MPL_BrBG = [ + const mplBrBG = [ [0, "rgb(86,49,5)"], [0.008, "rgb(91,52,6)"], [0.016, "rgb(95,54,6)"], @@ -735,7 +750,7 @@ const generateContourCurveOptions = function (curve, axisMap, dataset, appParams [0.984, "rgb(0,63,52)"], [1, "rgb(0,60,48)"], ]; - const MPL_BrBWG = [ + const mplBrBWG = [ [0, "rgb(86,49,5)"], [0.008, "rgb(91,52,6)"], [0.016, "rgb(95,54,6)"], @@ -875,9 +890,9 @@ const generateContourCurveOptions = function (curve, axisMap, dataset, appParams statistic === "Bias (Model - Obs)" || appParams.plotType === matsTypes.PlotTypes.contourDiff ) { - colorscale = MPL_BrBWG; + colorscale = mplBrBWG; } else { - colorscale = MPL_BrBG; + colorscale = mplBrBG; } } else if ( statistic === "Bias (Model - Obs)" || @@ -961,9 +976,9 @@ const getContourSignificanceLayer = function (dataset) { let yidx; let currX; let currY; - for (xidx = 0; xidx < xs.length; xidx++) { + for (xidx = 0; xidx < xs.length; xidx += 1) { currX = xs[xidx]; - for (yidx = 0; yidx < ys.length; yidx++) { + for (yidx = 0; yidx < ys.length; yidx += 1) { currY = ys[yidx]; if (sigMask[yidx][xidx] === 1) { curveOptions.x.push(currX); @@ -984,6 +999,8 @@ const generateScatterCurveOptions = function ( dataSeries, appParams ) { + const thisAxisXMap = axisXMap; + const thisAxisYMap = axisYMap; const { label } = curve; const longLabel = matsPlotUtils.getCurveText(appParams.plotType, curve); const { annotation } = curve; @@ -995,35 +1012,35 @@ const generateScatterCurveOptions = function ( const { xmax } = curve; const { axisXKey } = curve; const { axisYKey } = curve; - if (axisXKey in axisXMap) { - axisXMap[axisXKey].xmin = - xmin < axisXMap[axisXKey].xmin ? xmin : axisXMap[axisXKey].xmin; - axisXMap[axisXKey].xmax = - xmax > axisXMap[axisXKey].xmax ? xmax : axisXMap[axisXKey].xmax; + if (axisXKey in thisAxisXMap) { + thisAxisXMap[axisXKey].xmin = + xmin < thisAxisXMap[axisXKey].xmin ? xmin : thisAxisXMap[axisXKey].xmin; + thisAxisXMap[axisXKey].xmax = + xmax > thisAxisXMap[axisXKey].xmax ? xmax : thisAxisXMap[axisXKey].xmax; } else { - axisXMap[axisXKey] = { - index: Object.keys(axisXMap).length + 1, + thisAxisXMap[axisXKey] = { + index: Object.keys(thisAxisXMap).length + 1, xmin, xmax, axisLabel: axisXKey, }; } - if (axisYKey in axisYMap) { - axisYMap[axisYKey].ymin = - ymin < axisYMap[axisYKey].ymin ? ymin : axisYMap[axisYKey].ymin; - axisYMap[axisYKey].ymax = - ymax > axisYMap[axisYKey].ymax ? ymax : axisYMap[axisYKey].ymax; + if (axisYKey in thisAxisYMap) { + thisAxisYMap[axisYKey].ymin = + ymin < thisAxisYMap[axisYKey].ymin ? ymin : thisAxisYMap[axisYKey].ymin; + thisAxisYMap[axisYKey].ymax = + ymax > thisAxisYMap[axisYKey].ymax ? ymax : thisAxisYMap[axisYKey].ymax; } else { - axisYMap[axisYKey] = { - index: Object.keys(axisYMap).length + 1, + thisAxisYMap[axisYKey] = { + index: Object.keys(thisAxisYMap).length + 1, ymin, ymax, axisLabel: axisYKey, }; } - const axisXNumber = Object.keys(axisXMap).indexOf(axisXKey); - const axisYNumber = Object.keys(axisYMap).indexOf(axisYKey); + const axisXNumber = Object.keys(thisAxisXMap).indexOf(axisXKey); + const axisYNumber = Object.keys(thisAxisYMap).indexOf(axisYKey); const curveOptions = { ...{ @@ -1050,6 +1067,7 @@ const generateScatterCurveOptions = function ( return curveOptions; }; +// eslint-disable-next-line no-undef export default matsDataCurveOpsUtils = { getHorizontalValueLine, getVerticalValueLine, diff --git a/meteor_packages/mats-common/imports/startup/server/data_diff_util.js b/meteor_packages/mats-common/imports/startup/server/data_diff_util.js index 361785637..5012bdbd2 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_diff_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_diff_util.js @@ -311,10 +311,7 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes minuendDataSubModelSum = minuendData.subModelSum[minuendIndex]; minuendDataSubObsSum = minuendData.subObsSum[minuendIndex]; minuendDataSubAbsSum = minuendData.subAbsSum[minuendIndex]; - } else if ( - minuendData.nTotal.length > 0 && - subtrahendData.nTotal.length - ) { + } else if (minuendData.nTotal.length > 0 && subtrahendData.nTotal.length) { minuendDataNForecast = minuendData.nForecast[minuendIndex]; minuendDataNMatched = minuendData.nMatched[minuendIndex]; minuendDataNSimple = minuendData.nSimple[minuendIndex]; @@ -339,10 +336,7 @@ const getDataForDiffCurve = function (dataset, diffFrom, appParams, allStatTypes subtrahendDataSubModelSum = subtrahendData.subModelSum[subtrahendIndex]; subtrahendDataSubObsSum = subtrahendData.subObsSum[subtrahendIndex]; subtrahendDataSubAbsSum = subtrahendData.subAbsSum[subtrahendIndex]; - } else if ( - minuendData.nTotal.length > 0 && - subtrahendData.nTotal.length - ) { + } else if (minuendData.nTotal.length > 0 && subtrahendData.nTotal.length) { subtrahendDataNForecast = subtrahendData.nForecast[subtrahendIndex]; subtrahendDataNMatched = subtrahendData.nMatched[subtrahendIndex]; subtrahendDataNSimple = subtrahendData.nSimple[subtrahendIndex]; diff --git a/meteor_packages/mats-common/imports/startup/server/data_match_util.js b/meteor_packages/mats-common/imports/startup/server/data_match_util.js index 0b86b3773..124032307 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_match_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_match_util.js @@ -3,6 +3,7 @@ */ import { matsDataUtils, matsTypes } from "meteor/randyp:mats-common"; +import { _ } from "meteor/underscore"; // function for removing unmatched data from a dataset containing multiple curves const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStats) { @@ -82,6 +83,7 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat let fi; let si; + const returnDataset = dataset; const { plotType } = appParams; const { hasLevels } = appParams; const { curvesLength } = curveInfoParams; @@ -163,7 +165,7 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat if (hasLevels) { subLevs[curveIndex] = {}; // map of the individual record levels (subLevs) going into each independentVar for each curve } - data = dataset[curveIndex]; + data = returnDataset[curveIndex]; // loop over every independentVar value in this curve for (di = 0; di < data[independentVarName].length; di += 1) { currIndependentVar = data[independentVarName][di]; @@ -182,8 +184,13 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat } } - const matchingIndependentVars = _.intersection.apply(_, independentVarGroups); // all of the non-null independentVar values common across all the curves - const matchingIndependentHasPoint = _.intersection.apply(_, independentVarHasPoint); // all of the independentVar values common across all the curves, regardless of whether or not they're null + const matchingIndependentVars = independentVarGroups.reduce((a, b) => + a.filter((c) => b.includes(c)) + ); // all of the non-null independentVar values common across all the curves + const matchingIndependentHasPoint = independentVarHasPoint.reduce((a, b) => + a.filter((c) => b.includes(c)) + ); // all of the independentVar values common across all the curves, regardless of whether or not they're null + if (removeNonMatchingIndVars) { if (hasLevels) { // loop over each common non-null independentVar value @@ -240,7 +247,7 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat } else { // pull all subSecs and subLevs out of their bins, and back into one main array for (curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { - data = dataset[curveIndex]; + data = returnDataset[curveIndex]; subSecsRaw[curveIndex] = []; subSecs[curveIndex] = []; if (hasLevels) { @@ -253,9 +260,13 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat subLevsRaw[curveIndex].push(data.subLevs[di]); } } - subSecs[curveIndex] = [].concat.apply([], subSecsRaw[curveIndex]); + subSecs[curveIndex] = subSecsRaw[curveIndex].reduce(function (a, b) { + return a.concat(b); + }); if (hasLevels) { - subLevs[curveIndex] = [].concat.apply([], subLevsRaw[curveIndex]); + subLevs[curveIndex] = subLevsRaw[curveIndex].reduce(function (a, b) { + return a.concat(b); + }); } } @@ -283,7 +294,7 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat } else { // determine which seconds are present in all curves // fill current subSecs intersection array with subSecs from the first curve - subSecIntersection = subSecs[0]; + [subSecIntersection] = subSecs; // loop over every curve after the first for (curveIndex = 1; curveIndex < curvesLength; curveIndex += 1) { // keep taking the intersection of the current subSecs intersection array with each curve's subSecs array @@ -295,11 +306,12 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat // remove non-matching independentVars and subSecs for (curveIndex = 0; curveIndex < curvesLength; curveIndex += 1) { // loop over every curve - data = dataset[curveIndex]; + data = returnDataset[curveIndex]; // need to loop backwards through the data array so that we can splice non-matching indices // while still having the remaining indices in the correct order let dataLength = data[independentVarName].length; for (di = dataLength - 1; di >= 0; di -= 1) { + let processSubData = true; if (removeNonMatchingIndVars) { if (matchingIndependentVars.indexOf(data[independentVarName][di]) === -1) { // if this is not a common non-null independentVar value, we'll have to remove some data @@ -307,7 +319,7 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat matchingIndependentHasPoint.indexOf(data[independentVarName][di]) === -1 ) { // if at least one curve doesn't even have a null here, much less a matching value (because of the cadence), just drop this independentVar - matsDataUtils.removePoint( + data = matsDataUtils.removePoint( data, di, plotType, @@ -318,271 +330,328 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat ); } else { // if all of the curves have either data or nulls at this independentVar, and there is at least one null, ensure all of the curves are null - matsDataUtils.nullPoint(data, di, statVarName, isCTC, isScalar, hasLevels); + data = matsDataUtils.nullPoint( + data, + di, + statVarName, + isCTC, + isScalar, + hasLevels + ); } // then move on to the next independentVar. There's no need to mess with the subSecs or subLevs - continue; + processSubData = false; } } - subSecs = data.subSecs[di]; - if (isCTC) { - subHit = data.subHit[di]; - subFa = data.subFa[di]; - subMiss = data.subMiss[di]; - subCn = data.subCn[di]; - } else if (isScalar) { + if (processSubData) { + subSecs = data.subSecs[di]; + if (isCTC) { + subHit = data.subHit[di]; + subFa = data.subFa[di]; + subMiss = data.subMiss[di]; + subCn = data.subCn[di]; + } else if (isScalar) { + if (isSimpleScatter) { + subSquareDiffSumX = data.subSquareDiffSumX[di]; + subNSumX = data.subNSumX[di]; + subObsModelDiffSumX = data.subObsModelDiffSumX[di]; + subModelSumX = data.subModelSumX[di]; + subObsSumX = data.subObsSumX[di]; + subAbsSumX = data.subAbsSumX[di]; + subSquareDiffSumY = data.subSquareDiffSumY[di]; + subNSumY = data.subNSumY[di]; + subObsModelDiffSumY = data.subObsModelDiffSumY[di]; + subModelSumY = data.subModelSumY[di]; + subObsSumY = data.subObsSumY[di]; + subAbsSumY = data.subAbsSumY[di]; + } else { + subSquareDiffSum = data.subSquareDiffSum[di]; + subNSum = data.subNSum[di]; + subObsModelDiffSum = data.subObsModelDiffSum[di]; + subModelSum = data.subModelSum[di]; + subObsSum = data.subObsSum[di]; + subAbsSum = data.subAbsSum[di]; + } + } else if (isReliability) { + subRelHit = data.subRelHit[di]; + subRelRawCount = data.subRelRawCount[di]; + subRelCount = data.subRelCount[di]; + } if (isSimpleScatter) { - subSquareDiffSumX = data.subSquareDiffSumX[di]; - subNSumX = data.subNSumX[di]; - subObsModelDiffSumX = data.subObsModelDiffSumX[di]; - subModelSumX = data.subModelSumX[di]; - subObsSumX = data.subObsSumX[di]; - subAbsSumX = data.subAbsSumX[di]; - subSquareDiffSumY = data.subSquareDiffSumY[di]; - subNSumY = data.subNSumY[di]; - subObsModelDiffSumY = data.subObsModelDiffSumY[di]; - subModelSumY = data.subModelSumY[di]; - subObsSumY = data.subObsSumY[di]; - subAbsSumY = data.subAbsSumY[di]; + subValuesX = data.subValsX[di]; + subValuesY = data.subValsY[di]; } else { - subSquareDiffSum = data.subSquareDiffSum[di]; - subNSum = data.subNSum[di]; - subObsModelDiffSum = data.subObsModelDiffSum[di]; - subModelSum = data.subModelSum[di]; - subObsSum = data.subObsSum[di]; - subAbsSum = data.subAbsSum[di]; + subValues = data.subVals[di]; } - } else if (isReliability) { - subRelHit = data.subRelHit[di]; - subRelRawCount = data.subRelRawCount[di]; - subRelCount = data.subRelCount[di]; - } - if (isSimpleScatter) { - subValuesX = data.subValsX[di]; - subValuesY = data.subValsY[di]; - } else { - subValues = data.subVals[di]; - } - if (hasLevels) { - subLevs = data.subLevs[di]; - } - - if ( - (!hasLevels && subSecs.length > 0) || - (hasLevels && subSecs.length > 0 && subLevs.length > 0) - ) { - currIndependentVar = data[independentVarName][di]; - newSubHit = []; - newSubFa = []; - newSubMiss = []; - newSubCn = []; - newSubSquareDiffSum = []; - newSubNSum = []; - newSubObsModelDiffSum = []; - newSubModelSum = []; - newSubObsSum = []; - newSubAbsSum = []; - newSubValues = []; - newSubSquareDiffSumX = []; - newSubNSumX = []; - newSubObsModelDiffSumX = []; - newSubModelSumX = []; - newSubObsSumX = []; - newSubAbsSumX = []; - newSubValuesX = []; - newSubSquareDiffSumY = []; - newSubNSumY = []; - newSubObsModelDiffSumY = []; - newSubModelSumY = []; - newSubObsSumY = []; - newSubAbsSumY = []; - newSubValuesY = []; - newSubRelCount = []; - newSubRelRawCount = []; - newSubRelHit = []; - newSubSecs = []; if (hasLevels) { - newSubLevs = []; + subLevs = data.subLevs[di]; } - // loop over all subSecs for this independentVar - for (si = 0; si < subSecs.length; si += 1) { + + if ( + (!hasLevels && subSecs.length > 0) || + (hasLevels && subSecs.length > 0 && subLevs.length > 0) + ) { + currIndependentVar = data[independentVarName][di]; + newSubHit = []; + newSubFa = []; + newSubMiss = []; + newSubCn = []; + newSubSquareDiffSum = []; + newSubNSum = []; + newSubObsModelDiffSum = []; + newSubModelSum = []; + newSubObsSum = []; + newSubAbsSum = []; + newSubValues = []; + newSubSquareDiffSumX = []; + newSubNSumX = []; + newSubObsModelDiffSumX = []; + newSubModelSumX = []; + newSubObsSumX = []; + newSubAbsSumX = []; + newSubValuesX = []; + newSubSquareDiffSumY = []; + newSubNSumY = []; + newSubObsModelDiffSumY = []; + newSubModelSumY = []; + newSubObsSumY = []; + newSubAbsSumY = []; + newSubValuesY = []; + newSubRelCount = []; + newSubRelRawCount = []; + newSubRelHit = []; + newSubSecs = []; if (hasLevels) { - // create sec-lev pair for each sub value - tempPair = [subSecs[si], subLevs[si]]; + newSubLevs = []; } - // keep the subValue only if its associated subSec/subLev is common to all curves for this independentVar - if ( - (!removeNonMatchingIndVars && - ((!hasLevels && subSecIntersection.indexOf(subSecs[si]) !== -1) || - (hasLevels && - matsDataUtils.arrayContainsSubArray(subIntersections, tempPair)))) || - (removeNonMatchingIndVars && - ((!hasLevels && - subSecIntersection[currIndependentVar].indexOf(subSecs[si]) !== -1) || - (hasLevels && - matsDataUtils.arrayContainsSubArray( - subIntersections[currIndependentVar], - tempPair - )))) - ) { - if (isCTC) { - var newHit = subHit[si]; - var newFa = subFa[si]; - var newMiss = subMiss[si]; - var newCn = subCn[si]; - } else if (isScalar) { - if (isSimpleScatter) { - var newSquareDiffSumX = subSquareDiffSumX[si]; - var newNSumX = subNSumX[si]; - var newObsModelDiffSumX = subObsModelDiffSumX[si]; - var newModelSumX = subModelSumX[si]; - var newObsSumX = subObsSumX[si]; - var newAbsSumX = subAbsSumX[si]; - var newSquareDiffSumY = subSquareDiffSumY[si]; - var newNSumY = subNSumY[si]; - var newObsModelDiffSumY = subObsModelDiffSumY[si]; - var newModelSumY = subModelSumY[si]; - var newObsSumY = subObsSumY[si]; - var newAbsSumY = subAbsSumY[si]; - } else { - var newSquareDiffSum = subSquareDiffSum[si]; - var newNSum = subNSum[si]; - var newObsModelDiffSum = subObsModelDiffSum[si]; - var newModelSum = subModelSum[si]; - var newObsSum = subObsSum[si]; - var newAbsSum = subAbsSum[si]; - } - } else if (isReliability) { - var newRelCount = subRelCount[si]; - var newRelRawCount = subRelRawCount[si]; - var newRelHit = subRelHit[si]; - } - if (isSimpleScatter) { - var newValX = subValuesX[si]; - var newValY = subValuesY[si]; - } else { - var newVal = subValues[si]; - } - const newSec = subSecs[si]; + // loop over all subSecs for this independentVar + for (si = 0; si < subSecs.length; si += 1) { + let newHit; + let newFa; + let newMiss; + let newCn; + let newSquareDiffSumX; + let newNSumX; + let newObsModelDiffSumX; + let newModelSumX; + let newObsSumX; + let newAbsSumX; + let newSquareDiffSumY; + let newNSumY; + let newObsModelDiffSumY; + let newModelSumY; + let newObsSumY; + let newAbsSumY; + let newSquareDiffSum; + let newNSum; + let newObsModelDiffSum; + let newModelSum; + let newObsSum; + let newAbsSum; + let newRelCount; + let newRelRawCount; + let newRelHit; + let newValX; + let newValY; + let newVal; + let newSec; + let newLev; + if (hasLevels) { - var newLev = subLevs[si]; + // create sec-lev pair for each sub value + tempPair = [subSecs[si], subLevs[si]]; } - if (isCTC) { - if (newHit !== undefined) { - newSubHit.push(newHit); - newSubFa.push(newFa); - newSubMiss.push(newMiss); - newSubCn.push(newCn); - newSubValues.push(newVal); - newSubSecs.push(newSec); - if (hasLevels) { - newSubLevs.push(newLev); + // keep the subValue only if its associated subSec/subLev is common to all curves for this independentVar + if ( + (!removeNonMatchingIndVars && + ((!hasLevels && subSecIntersection.indexOf(subSecs[si]) !== -1) || + (hasLevels && + matsDataUtils.arrayContainsSubArray( + subIntersections, + tempPair + )))) || + (removeNonMatchingIndVars && + ((!hasLevels && + subSecIntersection[currIndependentVar].indexOf(subSecs[si]) !== -1) || + (hasLevels && + matsDataUtils.arrayContainsSubArray( + subIntersections[currIndependentVar], + tempPair + )))) + ) { + if (isCTC) { + newHit = subHit[si]; + newFa = subFa[si]; + newMiss = subMiss[si]; + newCn = subCn[si]; + } else if (isScalar) { + if (isSimpleScatter) { + newSquareDiffSumX = subSquareDiffSumX[si]; + newNSumX = subNSumX[si]; + newObsModelDiffSumX = subObsModelDiffSumX[si]; + newModelSumX = subModelSumX[si]; + newObsSumX = subObsSumX[si]; + newAbsSumX = subAbsSumX[si]; + newSquareDiffSumY = subSquareDiffSumY[si]; + newNSumY = subNSumY[si]; + newObsModelDiffSumY = subObsModelDiffSumY[si]; + newModelSumY = subModelSumY[si]; + newObsSumY = subObsSumY[si]; + newAbsSumY = subAbsSumY[si]; + } else { + newSquareDiffSum = subSquareDiffSum[si]; + newNSum = subNSum[si]; + newObsModelDiffSum = subObsModelDiffSum[si]; + newModelSum = subModelSum[si]; + newObsSum = subObsSum[si]; + newAbsSum = subAbsSum[si]; } + } else if (isReliability) { + newRelCount = subRelCount[si]; + newRelRawCount = subRelRawCount[si]; + newRelHit = subRelHit[si]; } - } else if (isScalar) { if (isSimpleScatter) { - if (newSquareDiffSumX !== undefined) { - newSubSquareDiffSumX.push(newSquareDiffSumX); - newSubNSumX.push(newNSumX); - newSubObsModelDiffSumX.push(newObsModelDiffSumX); - newSubModelSumX.push(newModelSumX); - newSubObsSumX.push(newObsSumX); - newSubAbsSumX.push(newAbsSumX); - newSubValuesX.push(newValX); - } - if (newSquareDiffSumY !== undefined) { - newSubSquareDiffSumY.push(newSquareDiffSumY); - newSubNSumY.push(newNSumY); - newSubObsModelDiffSumY.push(newObsModelDiffSumY); - newSubModelSumY.push(newModelSumY); - newSubObsSumY.push(newObsSumY); - newSubAbsSumY.push(newAbsSumY); - newSubValuesY.push(newValY); - } - } else if (newSquareDiffSum !== undefined) { - newSubSquareDiffSum.push(newSquareDiffSum); - newSubNSum.push(newNSum); - newSubObsModelDiffSum.push(newObsModelDiffSum); - newSubModelSum.push(newModelSum); - newSubObsSum.push(newObsSum); - newSubAbsSum.push(newAbsSum); - newSubValues.push(newVal); + newValX = subValuesX[si]; + newValY = subValuesY[si]; + } else { + newVal = subValues[si]; } - newSubSecs.push(newSec); + newSec = subSecs[si]; if (hasLevels) { - newSubLevs.push(newLev); + newLev = subLevs[si]; } - } else if (isReliability) { - if (newRelHit !== undefined) { - newSubRelCount.push(newRelCount); - newSubRelRawCount.push(newRelRawCount); - newSubRelHit.push(newRelHit); + if (isCTC) { + if (newHit !== undefined) { + newSubHit.push(newHit); + newSubFa.push(newFa); + newSubMiss.push(newMiss); + newSubCn.push(newCn); + newSubValues.push(newVal); + newSubSecs.push(newSec); + if (hasLevels) { + newSubLevs.push(newLev); + } + } + } else if (isScalar) { + if (isSimpleScatter) { + if (newSquareDiffSumX !== undefined) { + newSubSquareDiffSumX.push(newSquareDiffSumX); + newSubNSumX.push(newNSumX); + newSubObsModelDiffSumX.push(newObsModelDiffSumX); + newSubModelSumX.push(newModelSumX); + newSubObsSumX.push(newObsSumX); + newSubAbsSumX.push(newAbsSumX); + newSubValuesX.push(newValX); + } + if (newSquareDiffSumY !== undefined) { + newSubSquareDiffSumY.push(newSquareDiffSumY); + newSubNSumY.push(newNSumY); + newSubObsModelDiffSumY.push(newObsModelDiffSumY); + newSubModelSumY.push(newModelSumY); + newSubObsSumY.push(newObsSumY); + newSubAbsSumY.push(newAbsSumY); + newSubValuesY.push(newValY); + } + } else if (newSquareDiffSum !== undefined) { + newSubSquareDiffSum.push(newSquareDiffSum); + newSubNSum.push(newNSum); + newSubObsModelDiffSum.push(newObsModelDiffSum); + newSubModelSum.push(newModelSum); + newSubObsSum.push(newObsSum); + newSubAbsSum.push(newAbsSum); + newSubValues.push(newVal); + } + newSubSecs.push(newSec); + if (hasLevels) { + newSubLevs.push(newLev); + } + } else if (isReliability) { + if (newRelHit !== undefined) { + newSubRelCount.push(newRelCount); + newSubRelRawCount.push(newRelRawCount); + newSubRelHit.push(newRelHit); + newSubValues.push(newVal); + newSubSecs.push(newSec); + if (hasLevels) { + newSubLevs.push(newLev); + } + } + } else if (newVal !== undefined) { newSubValues.push(newVal); newSubSecs.push(newSec); if (hasLevels) { newSubLevs.push(newLev); } } - } else if (newVal !== undefined) { - newSubValues.push(newVal); - newSubSecs.push(newSec); - if (hasLevels) { - newSubLevs.push(newLev); - } } } - } - if (newSubSecs.length === 0) { - // no matching sub-values, so null the point - matsDataUtils.nullPoint(data, di, statVarName, isCTC, isScalar, hasLevels); - } else { - // store the filtered data - if (isCTC) { - data.subHit[di] = newSubHit; - data.subFa[di] = newSubFa; - data.subMiss[di] = newSubMiss; - data.subCn[di] = newSubCn; - } else if (isScalar) { + if (newSubSecs.length === 0) { + // no matching sub-values, so null the point + data = matsDataUtils.nullPoint( + data, + di, + statVarName, + isCTC, + isScalar, + hasLevels + ); + } else { + // store the filtered data + if (isCTC) { + data.subHit[di] = newSubHit; + data.subFa[di] = newSubFa; + data.subMiss[di] = newSubMiss; + data.subCn[di] = newSubCn; + } else if (isScalar) { + if (isSimpleScatter) { + data.subSquareDiffSumX[di] = newSubSquareDiffSumX; + data.subNSumX[di] = newSubNSumX; + data.subObsModelDiffSumX[di] = newSubObsModelDiffSumX; + data.subModelSumX[di] = newSubModelSumX; + data.subObsSumX[di] = newSubObsSumX; + data.subAbsSumX[di] = newSubAbsSumX; + data.subSquareDiffSumY[di] = newSubSquareDiffSumY; + data.subNSumY[di] = newSubNSumY; + data.subObsModelDiffSumY[di] = newSubObsModelDiffSumY; + data.subModelSumY[di] = newSubModelSumY; + data.subObsSumY[di] = newSubObsSumY; + data.subAbsSumY[di] = newSubAbsSumY; + } else { + data.subSquareDiffSum[di] = newSubSquareDiffSum; + data.subNSum[di] = newSubNSum; + data.subObsModelDiffSum[di] = newSubObsModelDiffSum; + data.subModelSum[di] = newSubModelSum; + data.subObsSum[di] = newSubObsSum; + data.subAbsSum[di] = newSubAbsSum; + } + } else if (isReliability) { + data.subRelCount[di] = newSubRelCount; + data.subRelRawCount[di] = newSubRelRawCount; + data.subRelHit[di] = newSubRelHit; + } if (isSimpleScatter) { - data.subSquareDiffSumX[di] = newSubSquareDiffSumX; - data.subNSumX[di] = newSubNSumX; - data.subObsModelDiffSumX[di] = newSubObsModelDiffSumX; - data.subModelSumX[di] = newSubModelSumX; - data.subObsSumX[di] = newSubObsSumX; - data.subAbsSumX[di] = newSubAbsSumX; - data.subSquareDiffSumY[di] = newSubSquareDiffSumY; - data.subNSumY[di] = newSubNSumY; - data.subObsModelDiffSumY[di] = newSubObsModelDiffSumY; - data.subModelSumY[di] = newSubModelSumY; - data.subObsSumY[di] = newSubObsSumY; - data.subAbsSumY[di] = newSubAbsSumY; + data.subValsX[di] = newSubValuesX; + data.subValsY[di] = newSubValuesY; } else { - data.subSquareDiffSum[di] = newSubSquareDiffSum; - data.subNSum[di] = newSubNSum; - data.subObsModelDiffSum[di] = newSubObsModelDiffSum; - data.subModelSum[di] = newSubModelSum; - data.subObsSum[di] = newSubObsSum; - data.subAbsSum[di] = newSubAbsSum; + data.subVals[di] = newSubValues; + } + data.subSecs[di] = newSubSecs; + if (hasLevels) { + data.subLevs[di] = newSubLevs; } - } else if (isReliability) { - data.subRelCount[di] = newSubRelCount; - data.subRelRawCount[di] = newSubRelRawCount; - data.subRelHit[di] = newSubRelHit; - } - if (isSimpleScatter) { - data.subValsX[di] = newSubValuesX; - data.subValsY[di] = newSubValuesY; - } else { - data.subVals[di] = newSubValues; - } - data.subSecs[di] = newSubSecs; - if (hasLevels) { - data.subLevs[di] = newSubLevs; } + } else { + // no sub-values to begin with, so null the point + data = matsDataUtils.nullPoint( + data, + di, + statVarName, + isCTC, + isScalar, + hasLevels + ); } - } else { - // no sub-values to begin with, so null the point - matsDataUtils.nullPoint(data, di, statVarName, isCTC, isScalar, hasLevels); } } @@ -787,11 +856,17 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat }; if (data.x.length > 0) { // need to recalculate bins and stats - const curveSubVals = [].concat.apply([], data.subVals); - const curveSubSecs = [].concat.apply([], data.subSecs); - var curveSubLevs; + const curveSubVals = data.subVals.reduce(function (a, b) { + return a.concat(b); + }); + const curveSubSecs = data.subSecs.reduce(function (a, b) { + return a.concat(b); + }); + let curveSubLevs; if (hasLevels) { - curveSubLevs = [].concat.apply([], data.subLevs); + curveSubLevs = data.subLevs.reduce(function (a, b) { + return a.concat(b); + }); } else { curveSubLevs = []; } @@ -811,7 +886,7 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat } const newCurveDataKeys = Object.keys(newCurveData); for (let didx = 0; didx < newCurveDataKeys.length; didx += 1) { - dataset[curveIndex][newCurveDataKeys[didx]] = + returnDataset[curveIndex][newCurveDataKeys[didx]] = newCurveData[newCurveDataKeys[didx]]; } } @@ -828,12 +903,13 @@ const getMatchedDataSet = function (dataset, curveInfoParams, appParams, binStat data.xmax = Math.max(...filteredx); data.ymin = Math.min(...filteredy); data.ymax = Math.max(...filteredy); - dataset[curveIndex] = data; + returnDataset[curveIndex] = data; } - return dataset; + return returnDataset; }; +// eslint-disable-next-line no-undef export default matsDataMatchUtils = { getMatchedDataSet, }; diff --git a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js index bbd1ffe56..fbbf1917d 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js @@ -3,7 +3,9 @@ */ import { matsCollections, matsTypes } from "meteor/randyp:mats-common"; +import { Meteor } from "meteor/meteor"; import { moment } from "meteor/momentjs:moment"; +import { _ } from "meteor/underscore"; // sets plot options for timeseries plots const generateSeriesPlotOptions = function (axisMap, errorMax) { @@ -86,7 +88,7 @@ const generateSeriesPlotOptions = function (axisMap, errorMax) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this y-axis axisKey = Object.keys(axisMap)[axisIdx]; let { ymin } = axisMap[axisKey]; @@ -97,7 +99,7 @@ const generateSeriesPlotOptions = function (axisMap, errorMax) { xmin = axisMap[axisKey].xmin < xmin ? axisMap[axisKey].xmin : xmin; xmax = axisMap[axisKey].xmax > xmax ? axisMap[axisKey].xmax : xmax; axisLabel = axisMap[axisKey].axisLabel; - var axisObjectKey; + let axisObjectKey; const axisObjectBegin = { title: axisLabel, titlefont: { @@ -252,7 +254,7 @@ const generateProfilePlotOptions = function (axisMap, errorMax) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < xAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < xAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this x-axis axisKey = Object.keys(axisMap)[axisIdx]; let { xmin } = axisMap[axisKey]; @@ -261,7 +263,7 @@ const generateProfilePlotOptions = function (axisMap, errorMax) { xmin -= errorMax; const xPad = (xmax - xmin) * 0.025 !== 0 ? (xmax - xmin) * 0.025 : 0.025; axisLabel = axisMap[axisKey].axisLabel; - var axisObjectKey; + let axisObjectKey; const axisObjectBegin = { title: axisLabel, titlefont: { @@ -405,7 +407,7 @@ const generateDieoffPlotOptions = function (axisMap, errorMax) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this y-axis axisKey = Object.keys(axisMap)[axisIdx]; let { ymin } = axisMap[axisKey]; @@ -416,7 +418,7 @@ const generateDieoffPlotOptions = function (axisMap, errorMax) { xmin = axisMap[axisKey].xmin < xmin ? axisMap[axisKey].xmin : xmin; xmax = axisMap[axisKey].xmax > xmax ? axisMap[axisKey].xmax : xmax; axisLabel = axisMap[axisKey].axisLabel; - var axisObjectKey; + let axisObjectKey; const axisObjectBegin = { title: axisLabel, titlefont: { @@ -477,7 +479,7 @@ const generateThresholdPlotOptions = function (dataset, axisMap, errorMax) { let xLabel = ""; let xUnits; let tickvals = []; - for (let didx = 0; didx < dataset.length; didx++) { + for (let didx = 0; didx < dataset.length; didx += 1) { xUnits = dataset[didx].thresholdAxisLabel; if (!xLabel.includes(xUnits) && xUnits !== undefined) { xLabel = xLabel.length === 0 ? xUnits : `${xLabel},${xUnits}`; @@ -566,7 +568,7 @@ const generateThresholdPlotOptions = function (dataset, axisMap, errorMax) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this y-axis axisKey = Object.keys(axisMap)[axisIdx]; let { ymin } = axisMap[axisKey]; @@ -577,7 +579,7 @@ const generateThresholdPlotOptions = function (dataset, axisMap, errorMax) { xmin = axisMap[axisKey].xmin < xmin ? axisMap[axisKey].xmin : xmin; xmax = axisMap[axisKey].xmax > xmax ? axisMap[axisKey].xmax : xmax; axisLabel = axisMap[axisKey].axisLabel; - var axisObjectKey; + let axisObjectKey; const axisObjectBegin = { title: axisLabel, titlefont: { @@ -738,7 +740,7 @@ const generateValidTimePlotOptions = function (axisMap, errorMax) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this y-axis axisKey = Object.keys(axisMap)[axisIdx]; let { ymin } = axisMap[axisKey]; @@ -747,7 +749,7 @@ const generateValidTimePlotOptions = function (axisMap, errorMax) { ymin -= errorMax; const yPad = (ymax - ymin) * 0.025 !== 0 ? (ymax - ymin) * 0.025 : 0.025; axisLabel = axisMap[axisKey].axisLabel; - var axisObjectKey; + let axisObjectKey; const axisObjectBegin = { title: axisLabel, titlefont: { @@ -886,7 +888,7 @@ const generateGridScalePlotOptions = function (axisMap, errorMax) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this y-axis axisKey = Object.keys(axisMap)[axisIdx]; let { ymin } = axisMap[axisKey]; @@ -897,7 +899,7 @@ const generateGridScalePlotOptions = function (axisMap, errorMax) { xmin = axisMap[axisKey].xmin < xmin ? axisMap[axisKey].xmin : xmin; xmax = axisMap[axisKey].xmax > xmax ? axisMap[axisKey].xmax : xmax; axisLabel = axisMap[axisKey].axisLabel; - var axisObjectKey; + let axisObjectKey; const axisObjectBegin = { title: axisLabel, titlefont: { @@ -1032,7 +1034,7 @@ const generateYearToYearPlotOptions = function (axisMap, errorMax) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this y-axis axisKey = Object.keys(axisMap)[axisIdx]; let { ymin } = axisMap[axisKey]; @@ -1043,7 +1045,7 @@ const generateYearToYearPlotOptions = function (axisMap, errorMax) { xmin = axisMap[axisKey].xmin < xmin ? axisMap[axisKey].xmin : xmin; xmax = axisMap[axisKey].xmax > xmax ? axisMap[axisKey].xmax : xmax; axisLabel = axisMap[axisKey].axisLabel; - var axisObjectKey; + let axisObjectKey; const axisObjectBegin = { title: axisLabel, titlefont: { @@ -1649,7 +1651,7 @@ const generateEnsembleHistogramPlotOptions = function (dataset, curves, axisMap) // get actual bins from the query to place on the x-axis let tickvals = []; - for (let didx = 0; didx < dataset.length; didx++) { + for (let didx = 0; didx < dataset.length; didx += 1) { tickvals = _.union(tickvals, dataset[didx].x); } tickvals = tickvals.sort(function (a, b) { @@ -1932,15 +1934,15 @@ const generateScatterPlotOptions = function (axisXMap, axisYMap) { let axisKey; let axisIdx; let axisLabel; - for (axisIdx = 0; axisIdx < xAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < xAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this x-axis axisKey = Object.keys(axisXMap)[axisIdx]; const { xmin } = axisXMap[axisKey]; const { xmax } = axisXMap[axisKey]; const xPad = (xmax - xmin) * 0.025 !== 0 ? (xmax - xmin) * 0.025 : 0.025; axisLabel = axisXMap[axisKey].axisLabel; - var axisObjectKey; - var axisObjectBegin = { + let axisObjectKey; + const axisObjectBegin = { title: axisLabel, titlefont: { size: 24, @@ -1984,14 +1986,15 @@ const generateScatterPlotOptions = function (axisXMap, axisYMap) { } // loop over all y-axes - for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx++) { + for (axisIdx = 0; axisIdx < yAxisNumber; axisIdx += 1) { // get max and min values and label for curves on this y-axis axisKey = Object.keys(axisYMap)[axisIdx]; const { ymin } = axisYMap[axisKey]; const { ymax } = axisYMap[axisKey]; const yPad = (ymax - ymin) * 0.025 !== 0 ? (ymax - ymin) * 0.025 : 0.025; axisLabel = axisYMap[axisKey].axisLabel; - axisObjectBegin = { + let axisObjectKey; + const axisObjectBegin = { title: axisLabel, titlefont: { size: 24, @@ -2036,6 +2039,7 @@ const generateScatterPlotOptions = function (axisXMap, axisYMap) { return layout; }; +// eslint-disable-next-line no-undef export default matsDataPlotOpsUtils = { generateSeriesPlotOptions, generateProfilePlotOptions, diff --git a/meteor_packages/mats-common/imports/startup/server/data_process_util.js b/meteor_packages/mats-common/imports/startup/server/data_process_util.js index 132b38e40..13906a9f0 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_process_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_process_util.js @@ -12,6 +12,7 @@ import { matsDataPlotOpsUtils, } from "meteor/randyp:mats-common"; import { moment } from "meteor/momentjs:moment"; +import { _ } from "meteor/underscore"; const processDataXYCurve = function ( dataset, @@ -21,6 +22,9 @@ const processDataXYCurve = function ( bookkeepingParams ) { // variable to store maximum error bar length + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; let errorMax = Number.MIN_VALUE; const error = ""; @@ -29,10 +33,10 @@ const processDataXYCurve = function ( matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); @@ -42,24 +46,29 @@ const processDataXYCurve = function ( const axisLimitReprocessed = {}; // calculate data statistics (including error bars) for each curve - for (var curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] = - axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] !== undefined; - const { diffFrom } = curveInfoParams.curves[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] = + axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] !== + undefined; + const { diffFrom } = returnCurveInfoParams.curves[curveIndex]; const statisticSelect = appName.indexOf("anomalycor") !== -1 ? "ACC" - : curveInfoParams.curves[curveIndex].statistic; - var data = dataset[curveIndex]; - var statType; - if (curveInfoParams.statType === undefined) { + : returnCurveInfoParams.curves[curveIndex].statistic; + const data = returnDataset[curveIndex]; + let statType; + if (returnCurveInfoParams.statType === undefined) { statType = "default"; // dummy stat type - } else if (Array.isArray(curveInfoParams.statType)) { - statType = curveInfoParams.statType[curveIndex]; + } else if (Array.isArray(returnCurveInfoParams.statType)) { + statType = returnCurveInfoParams.statType[curveIndex]; } else { - statType = curveInfoParams.statType; + statType = returnCurveInfoParams.statType; } - const { label } = dataset[curveIndex]; + const { label } = returnDataset[curveIndex]; let di = 0; const values = []; @@ -68,17 +77,17 @@ const processDataXYCurve = function ( while (di < data.x.length) { // errorResult holds all the calculated curve stats like mean, sd, etc. // These don't make sense for aggregated MODE stats, so skip for them. - var errorResult; + let errorResult; if (!statType.includes("met-mode")) { if (appParams.hasLevels) { - errorResult = matsDataUtils.get_err( + errorResult = matsDataUtils.getErr( data.subVals[di], data.subSecs[di], data.subLevs[di], appParams ); } else { - errorResult = matsDataUtils.get_err( + errorResult = matsDataUtils.getErr( data.subVals[di], data.subSecs[di], [], @@ -89,12 +98,13 @@ const processDataXYCurve = function ( if (diffFrom !== null && diffFrom !== undefined) { if ( - dataset[diffFrom[0]].y[di] !== null && - dataset[diffFrom[1]].y[di] !== null + returnDataset[diffFrom[0]].y[di] !== null && + returnDataset[diffFrom[1]].y[di] !== null ) { // make sure that the diff curve actually shows the difference when matching. // otherwise outlier filtering etc. can make it slightly off. - data.y[di] = dataset[diffFrom[0]].y[di] - dataset[diffFrom[1]].y[di]; + data.y[di] = + returnDataset[diffFrom[0]].y[di] - returnDataset[diffFrom[1]].y[di]; } else { // keep the null for no data at this point data.y[di] = null; @@ -114,27 +124,27 @@ const processDataXYCurve = function ( diffFrom === undefined || diffFrom === null || !( - Array.isArray(dataset[diffFrom[0]].subHit[di]) || - !isNaN(dataset[diffFrom[0]].subHit[di]) + Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || + !Number.isNaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( - Array.isArray(dataset[diffFrom[1]].subHit[di]) || - !isNaN(dataset[diffFrom[1]].subHit[di]) + Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || + !Number.isNaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_y.array[di] = null; } else { const minuendData = { - hit: dataset[diffFrom[0]].subHit[di], - fa: dataset[diffFrom[0]].subFa[di], - miss: dataset[diffFrom[0]].subMiss[di], - cn: dataset[diffFrom[0]].subCn[di], + hit: returnDataset[diffFrom[0]].subHit[di], + fa: returnDataset[diffFrom[0]].subFa[di], + miss: returnDataset[diffFrom[0]].subMiss[di], + cn: returnDataset[diffFrom[0]].subCn[di], }; const subtrahendData = { - hit: dataset[diffFrom[1]].subHit[di], - fa: dataset[diffFrom[1]].subFa[di], - miss: dataset[diffFrom[1]].subMiss[di], - cn: dataset[diffFrom[1]].subCn[di], + hit: returnDataset[diffFrom[1]].subHit[di], + fa: returnDataset[diffFrom[1]].subFa[di], + miss: returnDataset[diffFrom[1]].subMiss[di], + cn: returnDataset[diffFrom[1]].subCn[di], }; errorLength = matsDataUtils.ctcErrorPython( statisticSelect, @@ -145,7 +155,7 @@ const processDataXYCurve = function ( data.error_y.array[di] = errorLength; } } else { - const errorBar = errorResult.stde_betsy * 1.96; + const errorBar = errorResult.stdeBetsy * 1.96; errorMax = errorMax > errorBar ? errorMax : errorBar; data.error_y.array[di] = errorBar; } @@ -163,7 +173,7 @@ const processDataXYCurve = function ( { let fhr = ((data.x[di] / 1000) % (24 * 3600)) / 3600 - - curveInfoParams.utcCycleStarts[curveIndex]; + returnCurveInfoParams.utcCycleStarts[curveIndex]; fhr = fhr < 0 ? fhr + 24 : fhr; data.text[di] = `${data.text[di]}
time: ${moment .utc(data.x[di]) @@ -196,46 +206,46 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.y[di] - errorLength).toPrecision(4)} to ${Number( @@ -245,27 +255,27 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.y[di], - n_good: - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + nGood: + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; @@ -291,31 +301,29 @@ const processDataXYCurve = function ( } else { data.stats[di] = { stat: data.y[di], - n: errorResult.n_good, + n: errorResult.nGood, mean: statisticSelect === "N" || statisticSelect === "N times*levels(*stations if station plot) per graph point" ? errorResult.sum - : errorResult.d_mean, + : errorResult.dMean, sd: errorResult.sd, - n_good: errorResult.n_good, + nGood: errorResult.nGood, lag1: errorResult.lag1, - stde_betsy: errorResult.stde_betsy, + stdeBetsy: errorResult.stdeBetsy, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
sd: ${ errorResult.sd === null ? null : errorResult.sd.toPrecision(4) }
mean: ${ - errorResult.d_mean === null ? null : errorResult.d_mean.toPrecision(4) - }
n: ${errorResult.n_good}
stde: ${ - errorResult.stde_betsy - }
errorbars: ${Number( - data.y[di] - errorResult.stde_betsy * 1.96 - ).toPrecision(4)} to ${Number( - data.y[di] + errorResult.stde_betsy * 1.96 - ).toPrecision(4)}`; + errorResult.dMean === null ? null : errorResult.dMean.toPrecision(4) + }
n: ${errorResult.nGood}
stde: ${ + errorResult.stdeBetsy + }
errorbars: ${Number(data.y[di] - errorResult.stdeBetsy * 1.96).toPrecision( + 4 + )} to ${Number(data.y[di] + errorResult.stdeBetsy * 1.96).toPrecision(4)}`; } di += 1; @@ -332,11 +340,11 @@ const processDataXYCurve = function ( } // get the overall stats for the text output. - const stats = matsDataUtils.get_err(values, indVars, [], appParams); + const stats = matsDataUtils.getErr(values, indVars, [], appParams); const filteredValues = values.filter((x) => x || x === 0); stats.miny = Math.min(...filteredValues); stats.maxy = Math.max(...filteredValues); - dataset[curveIndex].glob_stats = stats; + returnDataset[curveIndex].glob_stats = stats; // recalculate axis options after QC and matching const filteredIndVars = []; @@ -345,41 +353,63 @@ const processDataXYCurve = function ( } const minx = Math.min(...filteredIndVars); const maxx = Math.max(...filteredIndVars); - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax < - stats.maxy || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymax = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .ymax < stats.maxy || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? stats.maxy - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin > - stats.miny || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymax; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymin = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .ymin > stats.miny || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? stats.miny - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax < maxx || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymin; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmax = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .xmax < maxx || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? maxx - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin > minx || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmax; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmin = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .xmin > minx || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? minx - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin; + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmin; // recalculate curve annotation after QC and matching const newMean = matsDataUtils.average(filteredValues); const newMedian = matsDataUtils.median(filteredValues); const newStdev = matsDataUtils.stdev(filteredValues); if (newMean !== undefined && newMean !== null) { - dataset[curveIndex].annotation = `${label} mean = ${newMean.toPrecision(4)}`; - dataset[curveIndex].annotation = `${ - dataset[curveIndex].annotation + returnDataset[curveIndex].annotation = `${label} mean = ${newMean.toPrecision( + 4 + )}`; + returnDataset[curveIndex].annotation = `${ + returnDataset[curveIndex].annotation }, median = ${newMedian.toPrecision(4)}`; - dataset[curveIndex].annotation = `${ - dataset[curveIndex].annotation + returnDataset[curveIndex].annotation = `${ + returnDataset[curveIndex].annotation }, stdev = ${newStdev.toPrecision(4)}`; } else { - dataset[ + returnDataset[ curveIndex ].annotation = `${label} mean = NoData, median = NoData, stdev = NoData`; } @@ -395,9 +425,13 @@ const processDataXYCurve = function ( } } - for (curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { // remove sub values and times to save space - data = dataset[curveIndex]; + const data = returnDataset[curveIndex]; data.subHit = []; data.subFa = []; data.subMiss = []; @@ -426,38 +460,38 @@ const processDataXYCurve = function ( case matsTypes.PlotTypes.timeSeries: case matsTypes.PlotTypes.dailyModelCycle: resultOptions = matsDataPlotOpsUtils.generateSeriesPlotOptions( - curveInfoParams.axisMap, + returnCurveInfoParams.axisMap, errorMax ); break; case matsTypes.PlotTypes.dieoff: resultOptions = matsDataPlotOpsUtils.generateDieoffPlotOptions( - curveInfoParams.axisMap, + returnCurveInfoParams.axisMap, errorMax ); break; case matsTypes.PlotTypes.threshold: resultOptions = matsDataPlotOpsUtils.generateThresholdPlotOptions( - dataset, - curveInfoParams.axisMap, + returnDataset, + returnCurveInfoParams.axisMap, errorMax ); break; case matsTypes.PlotTypes.validtime: resultOptions = matsDataPlotOpsUtils.generateValidTimePlotOptions( - curveInfoParams.axisMap, + returnCurveInfoParams.axisMap, errorMax ); break; case matsTypes.PlotTypes.gridscale: resultOptions = matsDataPlotOpsUtils.generateGridScalePlotOptions( - curveInfoParams.axisMap, + returnCurveInfoParams.axisMap, errorMax ); break; case matsTypes.PlotTypes.yearToYear: resultOptions = matsDataPlotOpsUtils.generateYearToYearPlotOptions( - curveInfoParams.axisMap, + returnCurveInfoParams.axisMap, errorMax ); break; @@ -476,43 +510,46 @@ const processDataXYCurve = function ( "rgb(0,0,0)", 1 ); - dataset.push(zeroLine); + returnDataset.push(zeroLine); // add ideal value lines, if any let idealValueLine; let idealLabel; - for (let ivIdx = 0; ivIdx < curveInfoParams.idealValues.length; ivIdx += 1) { + for (let ivIdx = 0; ivIdx < returnCurveInfoParams.idealValues.length; ivIdx += 1) { idealLabel = `ideal${ivIdx.toString()}`; idealValueLine = matsDataCurveOpsUtils.getHorizontalValueLine( resultOptions.xaxis.range[1], resultOptions.xaxis.range[0], - curveInfoParams.idealValues[ivIdx], + returnCurveInfoParams.idealValues[ivIdx], "bottom left", matsTypes.ReservedWords[idealLabel], "rgb(0,0,0)", 1 ); - dataset.push(idealValueLine); + returnDataset.push(idealValueLine); } const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -525,6 +562,9 @@ const processDataProfile = function ( bookkeepingParams ) { // variable to store maximum error bar length + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; let errorMax = Number.MIN_VALUE; const error = ""; @@ -533,10 +573,10 @@ const processDataProfile = function ( matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); @@ -546,24 +586,29 @@ const processDataProfile = function ( const axisLimitReprocessed = {}; // calculate data statistics (including error bars) for each curve - for (var curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] = - axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] !== undefined; - const { diffFrom } = curveInfoParams.curves[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] = + axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] !== + undefined; + const { diffFrom } = returnCurveInfoParams.curves[curveIndex]; const statisticSelect = appName.indexOf("anomalycor") !== -1 ? "ACC" - : curveInfoParams.curves[curveIndex].statistic; - var data = dataset[curveIndex]; - var statType; - if (curveInfoParams.statType === undefined) { + : returnCurveInfoParams.curves[curveIndex].statistic; + const data = returnDataset[curveIndex]; + let statType; + if (returnCurveInfoParams.statType === undefined) { statType = "default"; // dummy stat type - } else if (Array.isArray(curveInfoParams.statType)) { - statType = curveInfoParams.statType[curveIndex]; + } else if (Array.isArray(returnCurveInfoParams.statType)) { + statType = returnCurveInfoParams.statType[curveIndex]; } else { - statType = curveInfoParams.statType; + statType = returnCurveInfoParams.statType; } - const { label } = dataset[curveIndex]; + const { label } = returnDataset[curveIndex]; let di = 0; const values = []; @@ -572,8 +617,9 @@ const processDataProfile = function ( while (di < data.y.length) { // errorResult holds all the calculated curve stats like mean, sd, etc. // These don't make sense for aggregated MODE stats, so skip for them. + let errorResult; if (!statType.includes("met-mode")) { - var errorResult = matsDataUtils.get_err( + errorResult = matsDataUtils.getErr( data.subVals[di], data.subSecs[di], data.subLevs[di], @@ -583,12 +629,13 @@ const processDataProfile = function ( if (diffFrom !== null && diffFrom !== undefined) { if ( - dataset[diffFrom[0]].x[di] !== null && - dataset[diffFrom[1]].x[di] !== null + returnDataset[diffFrom[0]].x[di] !== null && + returnDataset[diffFrom[1]].x[di] !== null ) { // make sure that the diff curve actually shows the difference when matching. // otherwise outlier filtering etc. can make it slightly off. - data.x[di] = dataset[diffFrom[0]].x[di] - dataset[diffFrom[1]].x[di]; + data.x[di] = + returnDataset[diffFrom[0]].x[di] - returnDataset[diffFrom[1]].x[di]; } else { // keep the null for no data at this point data.x[di] = null; @@ -608,27 +655,27 @@ const processDataProfile = function ( diffFrom === undefined || diffFrom === null || !( - Array.isArray(dataset[diffFrom[0]].subHit[di]) || - !isNaN(dataset[diffFrom[0]].subHit[di]) + Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || + !Number.isNaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( - Array.isArray(dataset[diffFrom[1]].subHit[di]) || - !isNaN(dataset[diffFrom[1]].subHit[di]) + Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || + !Number.isNaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_x.array[di] = null; } else { const minuendData = { - hit: dataset[diffFrom[0]].subHit[di], - fa: dataset[diffFrom[0]].subFa[di], - miss: dataset[diffFrom[0]].subMiss[di], - cn: dataset[diffFrom[0]].subCn[di], + hit: returnDataset[diffFrom[0]].subHit[di], + fa: returnDataset[diffFrom[0]].subFa[di], + miss: returnDataset[diffFrom[0]].subMiss[di], + cn: returnDataset[diffFrom[0]].subCn[di], }; const subtrahendData = { - hit: dataset[diffFrom[1]].subHit[di], - fa: dataset[diffFrom[1]].subFa[di], - miss: dataset[diffFrom[1]].subMiss[di], - cn: dataset[diffFrom[1]].subCn[di], + hit: returnDataset[diffFrom[1]].subHit[di], + fa: returnDataset[diffFrom[1]].subFa[di], + miss: returnDataset[diffFrom[1]].subMiss[di], + cn: returnDataset[diffFrom[1]].subCn[di], }; errorLength = matsDataUtils.ctcErrorPython( statisticSelect, @@ -639,7 +686,7 @@ const processDataProfile = function ( data.error_x.array[di] = errorLength; } } else { - const errorBar = errorResult.stde_betsy * 1.96; + const errorBar = errorResult.stdeBetsy * 1.96; errorMax = errorMax > errorBar ? errorMax : errorBar; data.error_x.array[di] = errorBar; } @@ -649,23 +696,23 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; @@ -674,23 +721,23 @@ const processDataProfile = function ( `
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.x[di] - errorLength).toPrecision( @@ -700,27 +747,27 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.x[di], - n_good: - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + nGood: + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; @@ -746,17 +793,17 @@ const processDataProfile = function ( } else { data.stats[di] = { stat: data.x[di], - n: errorResult.n_good, + n: errorResult.nGood, mean: statisticSelect === "N" || statisticSelect === "N times*levels(*stations if station plot) per graph point" ? errorResult.sum - : errorResult.d_mean, + : errorResult.dMean, sd: errorResult.sd, - n_good: errorResult.n_good, + nGood: errorResult.nGood, lag1: errorResult.lag1, - stde_betsy: errorResult.stde_betsy, + stdeBetsy: errorResult.stdeBetsy, }; data.text[di] = `${label}
${data.y[di]}mb` + @@ -765,13 +812,13 @@ const processDataProfile = function ( }
sd: ${ errorResult.sd === null ? null : errorResult.sd.toPrecision(4) }
mean: ${ - errorResult.d_mean === null ? null : errorResult.d_mean.toPrecision(4) - }
n: ${errorResult.n_good}
stde: ${ - errorResult.stde_betsy + errorResult.dMean === null ? null : errorResult.dMean.toPrecision(4) + }
n: ${errorResult.nGood}
stde: ${ + errorResult.stdeBetsy }
errorbars: ${Number( - data.x[di] - errorResult.stde_betsy * 1.96 + data.x[di] - errorResult.stdeBetsy * 1.96 ).toPrecision(4)} to ${Number( - data.x[di] + errorResult.stde_betsy * 1.96 + data.x[di] + errorResult.stdeBetsy * 1.96 ).toPrecision(4)}`; } @@ -789,7 +836,7 @@ const processDataProfile = function ( } // get the overall stats for the text output. - const stats = matsDataUtils.get_err( + const stats = matsDataUtils.getErr( values.reverse(), levels.reverse(), [], @@ -798,7 +845,7 @@ const processDataProfile = function ( const filteredValues = values.filter((x) => x || x === 0); stats.minx = Math.min(...filteredValues); stats.maxx = Math.max(...filteredValues); - dataset[curveIndex].glob_stats = stats; + returnDataset[curveIndex].glob_stats = stats; // recalculate axis options after QC and matching const filteredLevels = []; @@ -807,49 +854,75 @@ const processDataProfile = function ( } const miny = Math.min(...filteredLevels); const maxy = Math.max(...filteredLevels); - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax < maxy || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymax = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .ymax < maxy || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? maxy - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin > miny || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymax; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymin = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .ymin > miny || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? miny - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax < - stats.maxx || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymin; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmax = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .xmax < stats.maxx || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? stats.maxx - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin > - stats.minx || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmax; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmin = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .xmin > stats.minx || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? stats.minx - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin; + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmin; // recalculate curve annotation after QC and matching const newMean = matsDataUtils.average(filteredValues); const newMedian = matsDataUtils.median(filteredValues); const newStdev = matsDataUtils.stdev(filteredValues); if (newMean !== undefined && newMean !== null) { - dataset[curveIndex].annotation = `${label} mean = ${newMean.toPrecision(4)}`; - dataset[curveIndex].annotation = `${ - dataset[curveIndex].annotation + returnDataset[curveIndex].annotation = `${label} mean = ${newMean.toPrecision( + 4 + )}`; + returnDataset[curveIndex].annotation = `${ + returnDataset[curveIndex].annotation }, median = ${newMedian.toPrecision(4)}`; - dataset[curveIndex].annotation = `${ - dataset[curveIndex].annotation + returnDataset[curveIndex].annotation = `${ + returnDataset[curveIndex].annotation }, stdev = ${newStdev.toPrecision(4)}`; } else { - dataset[ + returnDataset[ curveIndex ].annotation = `${label} mean = NoData, median = NoData, stdev = NoData`; } } - for (curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { // remove sub values and times to save space - data = dataset[curveIndex]; + const data = returnDataset[curveIndex]; data.subHit = []; data.subFa = []; data.subMiss = []; @@ -874,7 +947,7 @@ const processDataProfile = function ( // generate plot options const resultOptions = matsDataPlotOpsUtils.generateProfilePlotOptions( - curveInfoParams.axisMap, + returnCurveInfoParams.axisMap, errorMax ); @@ -889,43 +962,46 @@ const processDataProfile = function ( "rgb(0,0,0)", 1 ); - dataset.push(zeroLine); + returnDataset.push(zeroLine); // add ideal value lines, if any let idealValueLine; let idealLabel; - for (let ivIdx = 0; ivIdx < curveInfoParams.idealValues.length; ivIdx += 1) { + for (let ivIdx = 0; ivIdx < returnCurveInfoParams.idealValues.length; ivIdx += 1) { idealLabel = `ideal${ivIdx.toString()}`; idealValueLine = matsDataCurveOpsUtils.getVerticalValueLine( resultOptions.yaxis.range[1], resultOptions.yaxis.range[0], - curveInfoParams.idealValues[ivIdx], + returnCurveInfoParams.idealValues[ivIdx], "bottom left", matsTypes.ReservedWords[idealLabel], "rgb(0,0,0)", 1 ); - dataset.push(idealValueLine); + returnDataset.push(idealValueLine); } const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -937,24 +1013,31 @@ const processDataReliability = function ( plotParams, bookkeepingParams ) { + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; const isMetexpress = matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); } // sort data statistics for each curve - for (let curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - const data = dataset[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + const data = returnDataset[curveIndex]; const { label } = data; const sampleClimo = data.sample_climo; @@ -992,7 +1075,7 @@ const processDataReliability = function ( di += 1; } - dataset[curveIndex].glob_stats = { + returnDataset[curveIndex].glob_stats = { sample_climo: sampleClimo, }; } @@ -1012,11 +1095,15 @@ const processDataReliability = function ( "rgb(0,0,0)", 1 ); - dataset.push(perfectLine); + returnDataset.push(perfectLine); // assign no skill lines for each curve - for (let curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - const data = dataset[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + const data = returnDataset[curveIndex]; const { label } = data; const color = data.annotateColor; const sampleClimo = data.sample_climo; @@ -1052,12 +1139,16 @@ const processDataReliability = function ( color, 1 ); - dataset.push(noSkillLine); + returnDataset.push(noSkillLine); } - for (let curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { // remove sub values and times to save space - const data = dataset[curveIndex]; + const data = returnDataset[curveIndex]; data.subRelHit = []; data.subRelRawCount = []; data.subRelCount = []; @@ -1067,23 +1158,26 @@ const processDataReliability = function ( } const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -1095,33 +1189,40 @@ const processDataROC = function ( plotParams, bookkeepingParams ) { + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; const isMetexpress = matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); } // sort data statistics for each curve - for (let curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - var data = dataset[curveIndex]; - var statType; - if (curveInfoParams.statType === undefined) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + const data = returnDataset[curveIndex]; + let statType; + if (returnCurveInfoParams.statType === undefined) { statType = "scalar"; - } else if (Array.isArray(curveInfoParams.statType)) { - statType = curveInfoParams.statType[curveIndex]; + } else if (Array.isArray(returnCurveInfoParams.statType)) { + statType = returnCurveInfoParams.statType[curveIndex]; } else { - statType = curveInfoParams.statType; + statType = returnCurveInfoParams.statType; } - const { label } = dataset[curveIndex]; + const { label } = returnDataset[curveIndex]; const { auc } = data; let di = 0; @@ -1153,7 +1254,7 @@ const processDataROC = function ( di += 1; } - dataset[curveIndex].glob_stats = { + returnDataset[curveIndex].glob_stats = { auc, }; } @@ -1166,14 +1267,14 @@ const processDataROC = function ( resultOptions.xaxis.range[1], resultOptions.xaxis.range[0], resultOptions.yaxis.range[1], - data.ymin, + returnDataset.ymin, matsTypes.ReservedWords.noSkill, "top left", matsTypes.ReservedWords.noSkill, "rgb(0,0,0)", 1 ); - dataset.push(noSkillLine); + returnDataset.push(noSkillLine); // add perfect forecast lines const xPerfectLine = matsDataCurveOpsUtils.getHorizontalValueLine( @@ -1185,7 +1286,7 @@ const processDataROC = function ( "rgb(0,0,0)", 1 ); - dataset.push(xPerfectLine); + returnDataset.push(xPerfectLine); const yPerfectLine = matsDataCurveOpsUtils.getVerticalValueLine( resultOptions.yaxis.range[0], @@ -1196,26 +1297,29 @@ const processDataROC = function ( "rgb(0,0,0)", 1 ); - dataset.push(yPerfectLine); + returnDataset.push(yPerfectLine); const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -1227,33 +1331,40 @@ const processDataPerformanceDiagram = function ( plotParams, bookkeepingParams ) { + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; const isMetexpress = matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); } // sort data statistics for each curve - for (var curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - var data = dataset[curveIndex]; - var statType; - if (curveInfoParams.statType === undefined) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + const data = returnDataset[curveIndex]; + let statType; + if (returnCurveInfoParams.statType === undefined) { statType = "scalar"; - } else if (Array.isArray(curveInfoParams.statType)) { - statType = curveInfoParams.statType[curveIndex]; + } else if (Array.isArray(returnCurveInfoParams.statType)) { + statType = returnCurveInfoParams.statType[curveIndex]; } else { - statType = curveInfoParams.statType; + statType = returnCurveInfoParams.statType; } - const { label } = dataset[curveIndex]; + const { label } = returnDataset[curveIndex]; let di = 0; while (di < data.x.length) { @@ -1282,7 +1393,7 @@ const processDataPerformanceDiagram = function ( di += 1; } - dataset[curveIndex].glob_stats = {}; + returnDataset[curveIndex].glob_stats = {}; } // generate plot options @@ -1300,7 +1411,7 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(biasLine); + returnDataset.push(biasLine); biasLine = matsDataCurveOpsUtils.getDashedLinearValueLine( 0.25, 0, @@ -1312,7 +1423,7 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(biasLine); + returnDataset.push(biasLine); biasLine = matsDataCurveOpsUtils.getDashedLinearValueLine( 0.5, 0, @@ -1324,7 +1435,7 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(biasLine); + returnDataset.push(biasLine); biasLine = matsDataCurveOpsUtils.getDashedLinearValueLine( 1, 0, @@ -1336,7 +1447,7 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(biasLine); + returnDataset.push(biasLine); biasLine = matsDataCurveOpsUtils.getDashedLinearValueLine( 1, 0, @@ -1348,7 +1459,7 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(biasLine); + returnDataset.push(biasLine); biasLine = matsDataCurveOpsUtils.getDashedLinearValueLine( 1, 0, @@ -1360,7 +1471,7 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(biasLine); + returnDataset.push(biasLine); biasLine = matsDataCurveOpsUtils.getDashedLinearValueLine( 1, 0, @@ -1372,7 +1483,7 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(biasLine); + returnDataset.push(biasLine); let xvals; let yvals; @@ -1384,8 +1495,8 @@ const processDataPerformanceDiagram = function ( xvals = _.range(cval, 1.01, 0.01); yvals = []; textVals = []; - var xval; - var yval; + let xval; + let yval; for (let xidx = 0; xidx < xvals.length; xidx += 1) { xval = xvals[xidx]; yval = (xval * cval) / (xval + xval * cval - cval); @@ -1403,12 +1514,16 @@ const processDataPerformanceDiagram = function ( "rgb(0,0,0)", 1 ); - dataset.push(csiLine); + returnDataset.push(csiLine); } - for (curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { // remove sub values and times to save space - data = dataset[curveIndex]; + const data = returnDataset[curveIndex]; data.subHit = []; data.subFa = []; data.subMiss = []; @@ -1432,23 +1547,26 @@ const processDataPerformanceDiagram = function ( } const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -1460,25 +1578,32 @@ const processDataGridScaleProb = function ( plotParams, bookkeepingParams ) { + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; const isMetexpress = matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); } // sort data statistics for each curve - for (let curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - const data = dataset[curveIndex]; - const { label } = dataset[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + const data = returnDataset[curveIndex]; + const { label } = returnDataset[curveIndex]; let di = 0; while (di < data.x.length) { @@ -1496,17 +1621,21 @@ const processDataGridScaleProb = function ( di += 1; } - dataset[curveIndex].glob_stats = {}; + returnDataset[curveIndex].glob_stats = {}; } // generate plot options const resultOptions = matsDataPlotOpsUtils.generateGridScaleProbPlotOptions( - curveInfoParams.axisMap + returnCurveInfoParams.axisMap ); - for (let curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { // remove sub values and times to save space - const data = dataset[curveIndex]; + const data = returnDataset[curveIndex]; data.subHit = []; data.subFa = []; data.subMiss = []; @@ -1530,23 +1659,26 @@ const processDataGridScaleProb = function ( } const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -1558,15 +1690,19 @@ const processDataEnsembleHistogram = function ( plotParams, bookkeepingParams ) { + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; + const isMetexpress = matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); @@ -1576,30 +1712,34 @@ const processDataEnsembleHistogram = function ( const axisLimitReprocessed = {}; // calculate data statistics (including error bars) for each curve - for (let curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] = - axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] !== undefined; - const { diffFrom } = curveInfoParams.curves[curveIndex]; - const data = dataset[curveIndex]; - const { label } = dataset[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] = + axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] !== + undefined; + const { diffFrom } = returnCurveInfoParams.curves[curveIndex]; + const data = returnDataset[curveIndex]; + const { label } = returnDataset[curveIndex]; let di = 0; const values = []; const indVars = []; - var rawStat; while (di < data.x.length) { // errorResult holds all the calculated curve stats like mean, sd, etc. - var errorResult; + let errorResult; if (appParams.hasLevels) { - errorResult = matsDataUtils.get_err( + errorResult = matsDataUtils.getErr( data.subVals[di], data.subSecs[di], data.subLevs[di], appParams ); } else { - errorResult = matsDataUtils.get_err( + errorResult = matsDataUtils.getErr( data.subVals[di], data.subSecs[di], [], @@ -1608,18 +1748,17 @@ const processDataEnsembleHistogram = function ( } // store raw statistic from query before recalculating that statistic to account for data removed due to matching, QC, etc. - rawStat = data.y[di]; if (diffFrom === null || diffFrom === undefined || !appParams.matching) { // assign recalculated statistic to data[di][1], which is the value to be plotted data.y[di] = errorResult.sum; } else if ( - dataset[diffFrom[0]].y[di] !== null && - dataset[diffFrom[1]].y[di] !== null + returnDataset[diffFrom[0]].y[di] !== null && + returnDataset[diffFrom[1]].y[di] !== null ) { // make sure that the diff curve actually shows the difference. Otherwise outlier filtering etc. can make it slightly off. data.y[di] = - dataset[diffFrom[0]].bin_stats[di].bin_n - - dataset[diffFrom[1]].bin_stats[di].bin_n; + returnDataset[diffFrom[0]].bin_stats[di].bin_n - + returnDataset[diffFrom[1]].bin_stats[di].bin_n; } else { // keep the null for no data at this point data.y[di] = null; @@ -1655,79 +1794,102 @@ const processDataEnsembleHistogram = function ( // calculate the relative frequency for all the bins. // for diff curves, there's no good way to produce a diff of only matching data, so just diff the two parent curves. let diffIndexVal = 0; - for (let d_idx = 0; d_idx < data.y.length; d_idx += 1) { - if (data.y[d_idx] !== null) { + for (let didx = 0; didx < data.y.length; didx += 1) { + if (data.y[didx] !== null) { if (diffFrom === null || diffFrom === undefined) { - data.bin_stats[d_idx].bin_rf = data.bin_stats[d_idx].bin_rf / valueTotal; + data.bin_stats[didx].bin_rf /= valueTotal; } else { for ( let diffIndex = diffIndexVal; diffIndex < data.x.length; diffIndex += 1 ) { - if (dataset[diffFrom[0]].x[d_idx] === dataset[diffFrom[1]].x[diffIndex]) { - data.bin_stats[d_idx].bin_rf = - dataset[diffFrom[0]].bin_stats[d_idx].bin_rf - - dataset[diffFrom[1]].bin_stats[diffIndex].bin_rf; + if ( + returnDataset[diffFrom[0]].x[didx] === + returnDataset[diffFrom[1]].x[diffIndex] + ) { + data.bin_stats[didx].bin_rf = + returnDataset[diffFrom[0]].bin_stats[didx].bin_rf - + returnDataset[diffFrom[1]].bin_stats[diffIndex].bin_rf; diffIndexVal = diffIndex; break; } - data.bin_stats[d_idx].bin_rf = null; + data.bin_stats[didx].bin_rf = null; } } } else { - data.bin_stats[d_idx].bin_rf = null; + data.bin_stats[didx].bin_rf = null; } - if (curveInfoParams.yAxisFormat === "Relative frequency") { + if (returnCurveInfoParams.yAxisFormat === "Relative frequency") { // replace the bin number with the bin relative frequency for the plotted statistic - data.y[d_idx] = data.bin_stats[d_idx].bin_rf; - values[d_idx] = data.y[d_idx]; + data.y[didx] = data.bin_stats[didx].bin_rf; + values[didx] = data.y[didx]; } - data.text[d_idx] = - `${data.text[d_idx]}
` + + data.text[didx] = + `${data.text[didx]}
` + `bin rel freq for this curve: ${ - data.bin_stats[d_idx].bin_rf === null + data.bin_stats[didx].bin_rf === null ? null - : data.bin_stats[d_idx].bin_rf.toPrecision(4) + : data.bin_stats[didx].bin_rf.toPrecision(4) }`; } // get the overall stats for the text output - this uses the means not the stats. - const stats = matsDataUtils.get_err(values, indVars, [], appParams); + const stats = matsDataUtils.getErr(values, indVars, [], appParams); const filteredValues = values.filter((x) => x || x === 0); stats.miny = Math.min(...filteredValues); stats.maxy = Math.max(...filteredValues); - dataset[curveIndex].glob_stats = stats; + returnDataset[curveIndex].glob_stats = stats; // recalculate axis options after QC and matching const minx = Math.min(...indVars); const maxx = Math.max(...indVars); - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax < - stats.maxy || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymax = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .ymax < stats.maxy || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? stats.maxy - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymax; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin > - stats.miny || !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymax; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymin = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .ymin > stats.miny || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? stats.miny - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].ymin; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax < maxx || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].ymin; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmax = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .xmax < maxx || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? maxx - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmax; - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin = - curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin > minx || - !axisLimitReprocessed[curveInfoParams.curves[curveIndex].axisKey] + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmax; + returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmin = + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[curveIndex].axisKey] + .xmin > minx || + !axisLimitReprocessed[returnCurveInfoParams.curves[curveIndex].axisKey] ? minx - : curveInfoParams.axisMap[curveInfoParams.curves[curveIndex].axisKey].xmin; + : returnCurveInfoParams.axisMap[ + returnCurveInfoParams.curves[curveIndex].axisKey + ].xmin; } // end curves const resultOptions = matsDataPlotOpsUtils.generateEnsembleHistogramPlotOptions( - dataset, - curveInfoParams.curves, - curveInfoParams.axisMap + returnDataset, + returnCurveInfoParams.curves, + returnCurveInfoParams.axisMap ); // add black 0 line curve @@ -1741,26 +1903,29 @@ const processDataEnsembleHistogram = function ( "rgb(0,0,0)", 1 ); - dataset.push(zeroLine); + returnDataset.push(zeroLine); const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -1776,6 +1941,9 @@ const processDataHistogram = function ( binParams, bookkeepingParams ) { + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; let curvesLengthSoFar = 0; let xmax = -1 * Number.MAX_VALUE; @@ -1784,8 +1952,12 @@ const processDataHistogram = function ( let ymin = Number.MAX_VALUE; // flatten all the returned data into one stats array and one secs array in order to calculate histogram bins over the whole range. - const curveSubStats = [].concat.apply([], allReturnedSubStats); - const curveSubSecs = [].concat.apply([], allReturnedSubSecs); + const curveSubStats = allReturnedSubStats.reduce(function (a, b) { + return a.concat(b); + }); + const curveSubSecs = allReturnedSubSecs.reduce(function (a, b) { + return a.concat(b); + }); let binStats; if (binParams.binBounds.length === 0) { @@ -1808,9 +1980,9 @@ const processDataHistogram = function ( const plotBins = {}; plotBins.binMeans = []; plotBins.binLabels = []; - for (let b_idx = 0; b_idx < binStats.binMeans.length; b_idx += 1) { - plotBins.binMeans.push(binStats.binMeans[b_idx]); - plotBins.binLabels.push(binStats.binLabels[b_idx]); + for (let bidx = 0; bidx < binStats.binMeans.length; bidx += 1) { + plotBins.binMeans.push(binStats.binMeans[bidx]); + plotBins.binLabels.push(binStats.binLabels[bidx]); } // post process curves @@ -1818,8 +1990,12 @@ const processDataHistogram = function ( let curve; let diffFrom; let label; - for (var curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - curve = curveInfoParams.curves[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + curve = returnCurveInfoParams.curves[curveIndex]; diffFrom = curve.diffFrom; label = curve.label; @@ -1860,7 +2036,7 @@ const processDataHistogram = function ( }; if (!diffFrom) { - if (curveInfoParams.dataFoundForCurve[curveIndex]) { + if (returnCurveInfoParams.dataFoundForCurve[curveIndex]) { // sort queried data into the full set of histogram bins sortedData = matsDataUtils.sortHistogramBins( allReturnedSubStats[curveIndex], @@ -1876,22 +2052,22 @@ const processDataHistogram = function ( } else { // this is a difference curve, so we're done with regular curves. // do any matching that needs to be done. - if (appParams.matching && !bookkeepingParams.alreadyMatched) { - const originalCurvesLength = curveInfoParams.curvesLength; - curveInfoParams.curvesLength = curvesLengthSoFar; - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (appParams.matching && !returnBookkeepingParams.alreadyMatched) { + const originalCurvesLength = returnCurveInfoParams.curvesLength; + returnCurveInfoParams.curvesLength = curvesLengthSoFar; + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, binStats ); - curveInfoParams.curvesLength = originalCurvesLength; - bookkeepingParams.alreadyMatched = true; + returnCurveInfoParams.curvesLength = originalCurvesLength; + returnBookkeepingParams.alreadyMatched = true; } // then take diffs const diffResult = matsDataDiffUtils.getDataForDiffCurve( - dataset, + returnDataset, diffFrom, appParams, ["histogram", "histogram"] @@ -1904,17 +2080,17 @@ const processDataHistogram = function ( // set curve annotation to be the curve mean -- may be recalculated later // also pass previously calculated axis stats to curve options curve.annotation = ""; - curve.axisKey = curveInfoParams.curves[curveIndex].axisKey; + curve.axisKey = returnCurveInfoParams.curves[curveIndex].axisKey; if (d.length > 0) { d.xmin = d.bin_stats[0].binUpBound; d.xmax = d.bin_stats[d.bin_stats.length - 1].binLowBound; } d.ymin = - curveInfoParams.yAxisFormat === "Relative frequency" + returnCurveInfoParams.yAxisFormat === "Relative frequency" ? (d.ymin / d.glob_stats.glob_n) * 100 : d.ymin; d.ymax = - curveInfoParams.yAxisFormat === "Relative frequency" + returnCurveInfoParams.yAxisFormat === "Relative frequency" ? (d.ymax / d.glob_stats.glob_n) * 100 : d.ymax; xmin = d.xmin < xmin ? d.xmin : xmin; @@ -1924,38 +2100,42 @@ const processDataHistogram = function ( const cOptions = matsDataCurveOpsUtils.generateBarChartCurveOptions( curve, curveIndex, - curveInfoParams.axisMap, + returnCurveInfoParams.axisMap, d, appParams ); // generate plot with data, curve annotation, axis labels, etc. - dataset.push(cOptions); + returnDataset.push(cOptions); curvesLengthSoFar += 1; } // end for curves - // if matching, pare down dataset to only matching data. Only do this if we didn't already do it while calculating diffs. + // if matching, pare down returnDataset to only matching data. Only do this if we didn't already do it while calculating diffs. if ( - curveInfoParams.curvesLength > 1 && + returnCurveInfoParams.curvesLength > 1 && appParams.matching && - !bookkeepingParams.alreadyMatched + !returnBookkeepingParams.alreadyMatched ) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, binStats ); } // calculate data statistics (including error bars) for each curve - for (curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - const statisticSelect = curveInfoParams.curves[curveIndex].statistic; - diffFrom = curveInfoParams.curves[curveIndex].diffFrom; - var data = dataset[curveIndex]; - label = dataset[curveIndex].label; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + const statisticSelect = returnCurveInfoParams.curves[curveIndex].statistic; + diffFrom = returnCurveInfoParams.curves[curveIndex].diffFrom; + const data = returnDataset[curveIndex]; + label = returnDataset[curveIndex].label; let di = 0; while (di < data.x.length) { - if (curveInfoParams.yAxisFormat === "Relative frequency") { + if (returnCurveInfoParams.yAxisFormat === "Relative frequency") { // replace the bin number with the bin relative frequency for the plotted statistic data.y[di] = data.bin_stats[di].bin_rf * 100; } @@ -1995,14 +2175,18 @@ const processDataHistogram = function ( } // end curves // generate plot options - curveInfoParams.axisMap[curveInfoParams.curves[0].axisKey].xmin = xmin; - curveInfoParams.axisMap[curveInfoParams.curves[0].axisKey].xmax = xmax; - curveInfoParams.axisMap[curveInfoParams.curves[0].axisKey].ymin = ymin; - curveInfoParams.axisMap[curveInfoParams.curves[0].axisKey].ymax = ymax; - - for (curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[0].axisKey].xmin = xmin; + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[0].axisKey].xmax = xmax; + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[0].axisKey].ymin = ymin; + returnCurveInfoParams.axisMap[returnCurveInfoParams.curves[0].axisKey].ymax = ymax; + + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { // remove sub values and times to save space - data = dataset[curveIndex]; + const data = returnDataset[curveIndex]; data.subHit = []; data.subFa = []; data.subMiss = []; @@ -2026,29 +2210,32 @@ const processDataHistogram = function ( } const resultOptions = matsDataPlotOpsUtils.generateHistogramPlotOptions( - curveInfoParams.curves, - curveInfoParams.axisMap, - curveInfoParams.varUnits, + returnCurveInfoParams.curves, + returnCurveInfoParams.axisMap, + returnCurveInfoParams.varUnits, plotBins ); const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -2059,12 +2246,17 @@ const processDataContour = function ( plotParams, bookkeepingParams ) { + const returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; const { appName } = matsCollections.Settings.findOne({}); const statisticSelect = - appName.indexOf("anomalycor") !== -1 ? "ACC" : curveInfoParams.curve[0].statistic; - const data = dataset[0]; - const { label } = dataset[0]; + appName.indexOf("anomalycor") !== -1 + ? "ACC" + : returnCurveInfoParams.curve[0].statistic; + const data = returnDataset[0]; + const { label } = returnDataset[0]; // if we have dates on one axis, make sure they're formatted correctly if (data.xAxisKey.indexOf("Date") !== -1) { @@ -2130,26 +2322,29 @@ const processDataContour = function ( data.subLevs = []; // generate plot options - const resultOptions = matsDataPlotOpsUtils.generateContourPlotOptions(dataset); + const resultOptions = matsDataPlotOpsUtils.generateContourPlotOptions(returnDataset); const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; @@ -2161,44 +2356,43 @@ const processDataSimpleScatter = function ( plotParams, bookkeepingParams ) { + let returnDataset = dataset; + const returnCurveInfoParams = curveInfoParams; + const returnBookkeepingParams = bookkeepingParams; const error = ""; const isMetexpress = matsCollections.Settings.findOne({}).appType === matsTypes.AppTypes.metexpress; // if matching, pare down dataset to only matching data. METexpress takes care of matching in its python query code - if (curveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { - dataset = matsDataMatchUtils.getMatchedDataSet( - dataset, - curveInfoParams, + if (returnCurveInfoParams.curvesLength > 1 && appParams.matching && !isMetexpress) { + returnDataset = matsDataMatchUtils.getMatchedDataSet( + returnDataset, + returnCurveInfoParams, appParams, {} ); } // sort data statistics for each curve - for (var curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { - var data = dataset[curveIndex]; - var statType; - if (curveInfoParams.statType === undefined) { - statType = "scalar"; - } else if (Array.isArray(curveInfoParams.statType)) { - statType = curveInfoParams.statType[curveIndex]; - } else { - statType = curveInfoParams.statType; - } - const { label } = dataset[curveIndex]; + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { + const data = returnDataset[curveIndex]; + const { label } = returnDataset[curveIndex]; const statisticXSelect = - curveInfoParams.curves[curveIndex]["x-statistic"] === undefined - ? curveInfoParams.curves[curveIndex].statistic - : curveInfoParams.curves[curveIndex]["x-statistic"]; - const statisticYSelect = curveInfoParams.curves[curveIndex]["y-statistic"]; + returnCurveInfoParams.curves[curveIndex]["x-statistic"] === undefined + ? returnCurveInfoParams.curves[curveIndex].statistic + : returnCurveInfoParams.curves[curveIndex]["x-statistic"]; + const statisticYSelect = returnCurveInfoParams.curves[curveIndex]["y-statistic"]; const variableXSelect = - curveInfoParams.curves[curveIndex]["x-variable"] === undefined - ? curveInfoParams.curves[curveIndex].variable - : curveInfoParams.curves[curveIndex]["x-variable"]; - const variableYSelect = curveInfoParams.curves[curveIndex]["y-variable"]; + returnCurveInfoParams.curves[curveIndex]["x-variable"] === undefined + ? returnCurveInfoParams.curves[curveIndex].variable + : returnCurveInfoParams.curves[curveIndex]["x-variable"]; + const variableYSelect = returnCurveInfoParams.curves[curveIndex]["y-variable"]; let di = 0; while (di < data.x.length) { @@ -2227,12 +2421,16 @@ const processDataSimpleScatter = function ( di += 1; } - dataset[curveIndex].glob_stats = {}; + returnDataset[curveIndex].glob_stats = {}; } - for (curveIndex = 0; curveIndex < curveInfoParams.curvesLength; curveIndex += 1) { + for ( + let curveIndex = 0; + curveIndex < returnCurveInfoParams.curvesLength; + curveIndex += 1 + ) { // remove sub values and times to save space - data = dataset[curveIndex]; + const data = returnDataset[curveIndex]; data.subSquareDiffSumX = []; data.subNSumX = []; data.subObsModelDiffSumX = []; @@ -2259,32 +2457,36 @@ const processDataSimpleScatter = function ( // generate plot options const resultOptions = matsDataPlotOpsUtils.generateScatterPlotOptions( - curveInfoParams.axisXMap, - curveInfoParams.axisYMap + returnCurveInfoParams.axisXMap, + returnCurveInfoParams.axisYMap ); const totalProcessingFinish = moment(); - bookkeepingParams.dataRequests["total retrieval and processing time for curve set"] = - { - begin: bookkeepingParams.totalProcessingStart.format(), - finish: totalProcessingFinish.format(), - duration: `${moment - .duration(totalProcessingFinish.diff(bookkeepingParams.totalProcessingStart)) - .asSeconds()} seconds`, - }; + returnBookkeepingParams.dataRequests[ + "total retrieval and processing time for curve set" + ] = { + begin: returnBookkeepingParams.totalProcessingStart.format(), + finish: totalProcessingFinish.format(), + duration: `${moment + .duration( + totalProcessingFinish.diff(returnBookkeepingParams.totalProcessingStart) + ) + .asSeconds()} seconds`, + }; // pass result to client-side plotting functions return { error, - data: dataset, + data: returnDataset, options: resultOptions, basis: { plotParams, - queries: bookkeepingParams.dataRequests, + queries: returnBookkeepingParams.dataRequests, }, }; }; +// eslint-disable-next-line no-undef export default matsDataProcessUtils = { processDataXYCurve, processDataProfile, diff --git a/meteor_packages/mats-common/imports/startup/server/data_query_util.js b/meteor_packages/mats-common/imports/startup/server/data_query_util.js index a5244335a..29a955763 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_query_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_query_util.js @@ -316,7 +316,7 @@ const parseQueryDataXYCurve = function ( sum: 0 }; */ - const returnD = d; + let returnD = d; const { plotType } = appParams; const { hasLevels } = appParams; const completenessQCParam = Number(appParams.completeness) / 100; @@ -848,7 +848,7 @@ const parseQueryDataXYCurve = function ( thisCadence = Number(thisCadence) - Number(forecastOffset) * 3600 * 1000; // current hour of day (cycle time) } if (cycles.indexOf(thisCadence) !== -1) { - matsDataUtils.addNullPoint( + returnD = matsDataUtils.addNullPoint( returnD, dIdx + 1, matsTypes.PlotTypes.timeSeries, @@ -861,7 +861,7 @@ const parseQueryDataXYCurve = function ( ); } } else { - matsDataUtils.addNullPoint( + returnD = matsDataUtils.addNullPoint( returnD, dIdx + 1, matsTypes.PlotTypes.timeSeries, diff --git a/meteor_packages/mats-common/imports/startup/server/data_util.js b/meteor_packages/mats-common/imports/startup/server/data_util.js index 71bd18c44..9500a50a9 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_util.js @@ -2,7 +2,12 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsTypes, matsCollections, matsPlotUtils } from "meteor/randyp:mats-common"; +import { matsTypes, matsCollections } from "meteor/randyp:mats-common"; +import { Meteor } from "meteor/meteor"; +import { HTTP } from "meteor/jkuester:http"; + +/* eslint-disable global-require */ +/* eslint-disable no-console */ // this function checks if two JSON objects are identical const areObjectsEqual = function (o, p) { @@ -27,10 +32,10 @@ const arrayContainsArray = function (superArray, subArray) { let j; for (i = 0, j = 0; i < superArray.length && j < subArray.length; ) { if (superArray[i] < subArray[j]) { - ++i; + i += 1; } else if (superArray[i] === subArray[j]) { - ++i; - ++j; + i += 1; + j += 1; } else { // subArray[j] not in superArray, so superArray does not contain all elements of subArray return false; @@ -45,10 +50,10 @@ const arrayContainsSubArray = function (superArray, subArray) { let i; let j; let current; - for (i = 0; i < superArray.length; ++i) { + for (i = 0; i < superArray.length; i += 1) { if (subArray.length === superArray[i].length) { current = superArray[i]; - for (j = 0; j < subArray.length && subArray[j] === current[j]; ++j); + for (j = 0; j < subArray.length && subArray[j] === current[j]; j += 1); if (j === subArray.length) return true; } } @@ -60,7 +65,7 @@ const arraysEqual = function (a, b) { if (a === b) return true; if (!a || !b) return false; if (a.length !== b.length) return false; - for (let i = 0; i < a.length; ++i) { + for (let i = 0; i < a.length; i += 1) { if (a[i] !== b[i]) return false; } return true; @@ -68,7 +73,7 @@ const arraysEqual = function (a, b) { const arrayUnique = function (a) { const arr = []; - for (let i = 0; i < a.length; i++) { + for (let i = 0; i < a.length; i += 1) { if (!arr.includes(a[i])) { arr.push(a[i]); } @@ -81,10 +86,10 @@ const findArrayInSubArray = function (superArray, subArray) { let i; let j; let current; - for (i = 0; i < superArray.length; ++i) { + for (i = 0; i < superArray.length; i += 1) { if (subArray.length === superArray[i].length) { current = superArray[i]; - for (j = 0; j < subArray.length && subArray[j] === current[j]; ++j); + for (j = 0; j < subArray.length && subArray[j] === current[j]; j += 1); if (j === subArray.length) return i; } } @@ -95,7 +100,7 @@ const findArrayInSubArray = function (superArray, subArray) { const objectContainsObject = function (superObject, subObject) { const superObjectKeys = Object.keys(superObject); let currentObject; - for (let i = 0; i < superObjectKeys.length; i++) { + for (let i = 0; i < superObjectKeys.length; i += 1) { currentObject = superObject[superObjectKeys[i]]; if (areObjectsEqual(subObject, currentObject)) { return true; @@ -108,8 +113,8 @@ const objectContainsObject = function (superObject, subObject) { // utility for calculating the sum of an array const sum = function (data) { if (data.length === 0) return null; - return data.reduce(function (sum, value) { - return !value ? sum : sum + value; + return data.reduce(function (thisSum, value) { + return !value ? thisSum : thisSum + value; }, 0); }; @@ -154,46 +159,28 @@ const dateConvert = function (dStr) { now.getUTCMinutes(), now.getUTCSeconds() ); - var yr = date.getUTCFullYear(); - var day = date.getUTCDate(); - var month = date.getUTCMonth(); - var hour = date.getUTCHours(); - var minute = date.getUTCMinutes(); + const yr = date.getUTCFullYear(); + const day = date.getUTCDate(); + const month = date.getUTCMonth(); + const hour = date.getUTCHours(); + const minute = date.getUTCMinutes(); return `${month}/${day}/${yr} ${hour}:${minute}`; } const dateParts = dStr.split(" "); - const dateArray = dateParts[0].split(/[\-\/]/); // split on - or / 01-01-2017 OR 01/01/2017 - var month = dateArray[0]; - var day = dateArray[1]; - var yr = dateArray[2]; - var hour = 0; - var minute = 0; + const dateArray = dateParts[0].split(/[-/]/); // split on - or / 01-01-2017 OR 01/01/2017 + const month = dateArray[0]; + const day = dateArray[1]; + const yr = dateArray[2]; + let hour = 0; + let minute = 0; if (dateParts[1]) { const timeArray = dateParts[1].split(":"); - hour = timeArray[0]; - minute = timeArray[1]; + [hour] = timeArray; + [, minute] = timeArray; } return `${month}/${day}/${yr} ${hour}:${minute}`; }; -// splits the date range string from the date selector into standardized fromDate/toDate strings, -// plus the epochs for the fromDate and toDate -const getDateRange = function (dateRange) { - const dates = dateRange.split(" - "); - const fromDateStr = dates[0]; - const fromDate = dateConvert(fromDateStr); - const toDateStr = dates[1]; - const toDate = dateConvert(toDateStr); - const fromSecs = secsConvert(fromDateStr); - const toSecs = secsConvert(toDateStr); - return { - fromDate, - toDate, - fromSeconds: fromSecs, - toSeconds: toSecs, - }; -}; - // this function converts a date string into an epoch const secsConvert = function (dStr) { if (dStr === undefined || dStr === " ") { @@ -201,7 +188,7 @@ const secsConvert = function (dStr) { return now.getTime() / 1000; } const dateParts = dStr.split(" "); - const dateArray = dateParts[0].split(/[\-\/]/); // split on - or / 01-01-2017 OR 01/01/2017 + const dateArray = dateParts[0].split(/[-/]/); // split on - or / 01-01-2017 OR 01/01/2017 const month = dateArray[0]; const day = dateArray[1]; const yr = dateArray[2]; @@ -209,16 +196,34 @@ const secsConvert = function (dStr) { let minute = 0; if (dateParts[1]) { const timeArray = dateParts[1].split(":"); - hour = timeArray[0]; - minute = timeArray[1]; + [hour] = timeArray; + [, minute] = timeArray; } - const my_date = new Date(Date.UTC(yr, month - 1, day, hour, minute, 0)); + const myDate = new Date(Date.UTC(yr, month - 1, day, hour, minute, 0)); // to UTC time, not local time - const date_in_secs = my_date.getTime(); + const dateInSecs = myDate.getTime(); // to UTC time, not local time // return date_in_secs/1000 -3600*6; - return date_in_secs / 1000; + return dateInSecs / 1000; +}; + +// splits the date range string from the date selector into standardized fromDate/toDate strings, +// plus the epochs for the fromDate and toDate +const getDateRange = function (dateRange) { + const dates = dateRange.split(" - "); + const fromDateStr = dates[0]; + const fromDate = dateConvert(fromDateStr); + const toDateStr = dates[1]; + const toDate = dateConvert(toDateStr); + const fromSecs = secsConvert(fromDateStr); + const toSecs = secsConvert(toDateStr); + return { + fromDate, + toDate, + fromSeconds: fromSecs, + toSeconds: toSecs, + }; }; // function to manage authorized logins for MATS @@ -391,6 +396,7 @@ const doSettings = function ( metexpress: "production", }; const settingsId = settings._id; + // eslint-disable-next-line no-undef const os = Npm.require("os"); const hostname = os.hostname().split(".")[0]; settings.appVersion = version; @@ -407,6 +413,9 @@ const callMetadataAPI = function ( fakeMetadata, hideOtherFor ) { + let returnDestinationStructure = destinationStructure; + const returnHideOtherFor = hideOtherFor; + let returnExpectedApps = expectedApps; const Future = require("fibers/future"); const pFuture = new Future(); HTTP.get(queryURL, {}, function (error, response) { @@ -414,37 +423,43 @@ const callMetadataAPI = function ( console.log(error); } else { const metadata = JSON.parse(response.content); - if (Array.isArray(destinationStructure)) { + if (Array.isArray(returnDestinationStructure)) { // this is the list of apps. It's the only array in the API. - destinationStructure = [...destinationStructure, ...metadata]; - expectedApps = metadata; + returnDestinationStructure = [...returnDestinationStructure, ...metadata]; + returnExpectedApps = metadata; } else if (Object.keys(metadata).length === 0) { // this metadata type (e.g. 'threshold') is not valid for this app // we need to add placeholder metadata to the destination structure // and add the selector in question to the hideOtherFor map. const dummyMetadata = {}; if (!selector.includes("values")) { - hideOtherFor[selector] = - hideOtherFor[selector] === undefined ? [] : hideOtherFor[selector]; - for (let eidx = 0; eidx < expectedApps.length; eidx++) { - dummyMetadata[expectedApps[eidx]] = fakeMetadata; - hideOtherFor[selector].push(expectedApps[eidx]); + returnHideOtherFor[selector] = + returnHideOtherFor[selector] === undefined + ? [] + : returnHideOtherFor[selector]; + for (let eidx = 0; eidx < returnExpectedApps.length; eidx += 1) { + dummyMetadata[returnExpectedApps[eidx]] = fakeMetadata; + returnHideOtherFor[selector].push(returnExpectedApps[eidx]); } } - destinationStructure = { ...destinationStructure, ...dummyMetadata }; + returnDestinationStructure = { + ...returnDestinationStructure, + ...dummyMetadata, + }; } else { - destinationStructure = { ...destinationStructure, ...metadata }; + returnDestinationStructure = { ...returnDestinationStructure, ...metadata }; } } pFuture.return(); }); pFuture.wait(); - return [destinationStructure, expectedApps, hideOtherFor]; + return [returnDestinationStructure, returnExpectedApps, returnHideOtherFor]; }; // calculates the statistic for ctc plots const calculateStatCTC = function (hit, fa, miss, cn, n, statistic) { - if (isNaN(hit) || isNaN(fa) || isNaN(miss) || isNaN(cn)) return null; + if (Number.isNaN(hit) || Number.isNaN(fa) || Number.isNaN(miss) || Number.isNaN(cn)) + return null; let queryVal; switch (statistic) { case "TSS (True Skill Score)": @@ -522,20 +537,20 @@ const calculateStatScalar = function ( modelSum, obsSum, absSum, - statistic + statisticAndVariable ) { if ( - isNaN(squareDiffSum) || - isNaN(NSum) || - isNaN(obsModelDiffSum) || - isNaN(modelSum) || - isNaN(obsSum) || - isNaN(absSum) + Number.isNaN(squareDiffSum) || + Number.isNaN(NSum) || + Number.isNaN(obsModelDiffSum) || + Number.isNaN(modelSum) || + Number.isNaN(obsSum) || + Number.isNaN(absSum) ) return null; let queryVal; - const variable = statistic.split("_")[1]; - [statistic] = statistic.split("_"); + const variable = statisticAndVariable.split("_")[1]; + const statistic = statisticAndVariable.split("_"); switch (statistic) { case "RMSE": queryVal = Math.sqrt(squareDiffSum / NSum); @@ -563,7 +578,7 @@ const calculateStatScalar = function ( default: queryVal = null; } - if (isNaN(queryVal)) return null; + if (Number.isNaN(queryVal)) return null; // need to convert to correct units for surface data but not upperair if (statistic !== "N") { if ( @@ -1134,20 +1149,20 @@ const readableAdeckModels = function () { }; // calculates mean, stdev, and other statistics for curve data points in all apps and plot types -const get_err = function (sVals, sSecs, sLevs, appParams) { +const getErr = function (sVals, sSecs, sLevs, appParams) { /* refer to perl error_library.pl sub get_stats to see the perl implementation of these statics calculations. These should match exactly those, except that they are processed in reverse order. */ - const autocorr_limit = 0.95; + const autocorrLimit = 0.95; const { hasLevels } = appParams; let n = sVals.length; - const data_wg = []; - let n_gaps = 0; - let sum = 0; + const dataWg = []; + let nGaps = 0; + let dataSum = 0; let sum2 = 0; - let d_mean = 0; + let dMean = 0; let sd2 = 0; let sd = 0; let error; @@ -1155,10 +1170,10 @@ const get_err = function (sVals, sSecs, sLevs, appParams) { if (n < 1) { return { - d_mean: null, - stde_betsy: null, + dMean: null, + stdeBetsy: null, sd: null, - n_good: n, + nGood: n, lag1: null, min: null, max: null, @@ -1167,47 +1182,48 @@ const get_err = function (sVals, sSecs, sLevs, appParams) { } // find minimum delta_time, if any value missing, set null - let last_secs = 0; + let lastSecs = 0; let minDelta = Number.MAX_VALUE; let minSecs = Number.MAX_VALUE; - let max_secs = Number.MIN_VALUE; + let maxSecs = Number.MIN_VALUE; let minVal = Number.MAX_VALUE; let maxVal = -1 * Number.MAX_VALUE; let secs; let delta; - for (i = 0; i < sSecs.length; i++) { - if (isNaN(sVals[i])) { + for (i = 0; i < sSecs.length; i += 1) { + if (Number.isNaN(sVals[i])) { n -= 1; - continue; - } - secs = sSecs[i]; - delta = Math.abs(secs - last_secs); - if (delta > 0 && delta < minDelta) { - minDelta = delta; - } - if (secs < minSecs) { - minSecs = secs; - } - if (secs > max_secs) { - max_secs = secs; + } else { + secs = sSecs[i]; + delta = Math.abs(secs - lastSecs); + if (delta > 0 && delta < minDelta) { + minDelta = delta; + } + if (secs < minSecs) { + minSecs = secs; + } + if (secs > maxSecs) { + maxSecs = secs; + } + lastSecs = secs; } - last_secs = secs; } if (minDelta < 0) { error = `Invalid time interval - minDelta: ${minDelta}`; console.log(`matsDataUtil.getErr: ${error}`); } - for (i = 0; i < sVals.length; i++) { - if (isNaN(sVals[i])) continue; - minVal = minVal < sVals[i] ? minVal : sVals[i]; - maxVal = maxVal > sVals[i] ? maxVal : sVals[i]; - sum += sVals[i]; - sum2 += sVals[i] * sVals[i]; + for (i = 0; i < sVals.length; i += 1) { + if (!Number.isNaN(sVals[i])) { + minVal = minVal < sVals[i] ? minVal : sVals[i]; + maxVal = maxVal > sVals[i] ? maxVal : sVals[i]; + dataSum += sVals[i]; + sum2 += sVals[i] * sVals[i]; + } } - d_mean = sum / n; - sd2 = sum2 / n - d_mean * d_mean; + dMean = dataSum / n; + sd2 = sum2 / n - dMean * dMean; if (sd2 > 0) { sd = Math.sqrt(sd2); } @@ -1215,178 +1231,196 @@ const get_err = function (sVals, sSecs, sLevs, appParams) { // look for gaps let lastSecond = -1 * Number.MAX_VALUE; let lastPressure = -1 * Number.MAX_VALUE; - let n_pressures; + let nPressures; if (hasLevels) { - n_pressures = arrayUnique(sLevs).length; + nPressures = arrayUnique(sLevs).length; } else { - n_pressures = 1; + nPressures = 1; } // set lag1_t to the first time the time changes from its initial value + 1 (data zero based) // set lag1_p to the first time the pressure changes from its initial value + 1 (data zero based) - let lag1_t = 0; - let lag1_p = 0; - let r1_t = 0; // autocorrelation for time - let r1_p = 0; // autocorrelation for pressure + let lag1T = 0; + let lag1P = 0; + let r1T = 0; // autocorrelation for time + let r1P = 0; // autocorrelation for pressure let j = 0; // i is loop index without gaps; j is loop index with gaps - let n_deltas = 0; - - for (i = 0; i < sSecs.length; i++) { - if (isNaN(sVals[i])) continue; - let sec = sSecs[i]; - if (typeof sec === "string" || sec instanceof String) sec = Number(sec); - var lev; - if (hasLevels) { - lev = sLevs[i]; - if (typeof lev === "string" || lev instanceof String) { - if (lev[0] === "P") { - lev = Number(lev.substring(1)); - } else { - lev = Number(lev); + let nDeltas = 0; + + for (i = 0; i < sSecs.length; i += 1) { + if (!Number.isNaN(sVals[i])) { + let sec = sSecs[i]; + if (typeof sec === "string" || sec instanceof String) sec = Number(sec); + let lev; + if (hasLevels) { + lev = sLevs[i]; + if (typeof lev === "string" || lev instanceof String) { + if (lev[0] === "P") { + lev = Number(lev.substring(1)); + } else { + lev = Number(lev); + } } - } - // find first time the pressure changes - if (lag1_p === 0 && lastPressure > 0) { - if (lev !== lastPressure) { - lag1_p = j; + // find first time the pressure changes + if (lag1P === 0 && lastPressure > 0) { + if (lev !== lastPressure) { + lag1P = j; + } } } - } - if (lastSecond >= 0) { - if (lag1_t === 0 && sec !== lastSecond) { - lag1_t = j; - } - if (Math.abs(sec - lastSecond) > minDelta) { - n_deltas = (Math.abs(sec - lastSecond) / minDelta - 1) * n_pressures; - // for the Autocorrelation at lag 1, it doesn't matter how many missing - // data we put in within gaps! (But for the other AC's it does.) - // since we're using only the AC at lag 1 for calculating std err, let's - // save cpu time and only put in one missing datum per gap, no matter - // how long. WRM 2/22/2019 - // but if we're using a different lag, which could happen, we'll need - // to insert all the missing data in each gap. WRM 2/22/2019 - // $n_deltas=1; - for (let count = 0; count < n_deltas; count++) { - data_wg.push(null); - n_gaps++; - j++; + if (lastSecond >= 0) { + if (lag1T === 0 && sec !== lastSecond) { + lag1T = j; + } + if (Math.abs(sec - lastSecond) > minDelta) { + nDeltas = (Math.abs(sec - lastSecond) / minDelta - 1) * nPressures; + // for the Autocorrelation at lag 1, it doesn't matter how many missing + // data we put in within gaps! (But for the other AC's it does.) + // since we're using only the AC at lag 1 for calculating std err, let's + // save cpu time and only put in one missing datum per gap, no matter + // how long. WRM 2/22/2019 + // but if we're using a different lag, which could happen, we'll need + // to insert all the missing data in each gap. WRM 2/22/2019 + // $n_deltas=1; + for (let count = 0; count < nDeltas; count += 1) { + dataWg.push(null); + nGaps += 1; + j += 1; + } } } + lastSecond = sec; + if (hasLevels) { + lastPressure = lev; + } + dataWg.push(sVals[i]); + j += 1; } - lastSecond = sec; - if (hasLevels) { - lastPressure = lev; - } - data_wg.push(sVals[i]); - j++; } // from http://www.itl.nist.gov/div898/handbook/eda/section3/eda35c.htm const r = []; - const lag_by_r = {}; - const lag1_max = lag1_p > lag1_t ? lag1_p : lag1_t; - let r_sum = 0; - let n_r = 0; - let n_in_lag; + const lagByR = {}; + const lag1Max = lag1P > lag1T ? lag1P : lag1T; + // let rSum = 0; + // let nR = 0; + let nInLag; let lag; let t; - for (lag = 0; lag <= lag1_max; lag++) { + for (lag = 0; lag <= lag1Max; lag += 1) { r[lag] = 0; - n_in_lag = 0; - for (t = 0; t < n + n_gaps - lag; t++) { - if (data_wg[t] && data_wg[t + lag]) { - r[lag] += +(data_wg[t] - d_mean) * (data_wg[t + lag] - d_mean); - n_in_lag++; + nInLag = 0; + for (t = 0; t < n + nGaps - lag; t += 1) { + if (dataWg[t] && dataWg[t + lag]) { + r[lag] += +(dataWg[t] - dMean) * (dataWg[t + lag] - dMean); + nInLag += 1; } } - if (n_in_lag > 0 && sd > 0) { - r[lag] /= n_in_lag * sd * sd; - r_sum += r[lag]; - n_r++; + if (nInLag > 0 && sd > 0) { + r[lag] /= nInLag * sd * sd; + // rSum += r[lag]; + // nR++; } else { r[lag] = null; } - if (lag >= 1 && lag < (n + n_gaps) / 2) { - lag_by_r[r[lag]] = lag; + if (lag >= 1 && lag < (n + nGaps) / 2) { + lagByR[r[lag]] = lag; } } - if (lag1_t > 0) { - r1_t = r[lag1_t] !== undefined ? r[lag1_t] : 0; + if (lag1T > 0) { + r1T = r[lag1T] !== undefined ? r[lag1T] : 0; } - if (lag1_p > 0) { - r1_p = r[lag1_p] !== undefined ? r[lag1_p] : 0; + if (lag1P > 0) { + r1P = r[lag1P] !== undefined ? r[lag1P] : 0; } // Betsy Weatherhead's correction, based on lag 1, augmented by the highest // lag > 1 and < n/2 - if (r1_p >= autocorr_limit) { - r1_p = autocorr_limit; + if (r1P >= autocorrLimit) { + r1P = autocorrLimit; } - if (r1_t >= autocorr_limit) { - r1_t = autocorr_limit; + if (r1T >= autocorrLimit) { + r1T = autocorrLimit; } - const betsy = Math.sqrt((n - 1) * (1 - r1_p) * (1 - r1_t)); - let stde_betsy; + const betsy = Math.sqrt((n - 1) * (1 - r1P) * (1 - r1T)); + let stdeBetsy; if (betsy !== 0) { - stde_betsy = sd / betsy; + stdeBetsy = sd / betsy; } else { - stde_betsy = null; + stdeBetsy = null; } const stats = { - d_mean, - stde_betsy, + dMean, + stdeBetsy, sd, - n_good: n, + nGood: n, lag1: r[1], min: minSecs, - max: max_secs, + max: maxSecs, minVal, maxVal, - sum, + sum: dataSum, }; // console.log("stats are " + JSON.stringify(stats)); - // stde_betsy is standard error with auto correlation + // stdeBetsy is standard error with auto correlation // console.log("---------\n\n"); return stats; }; -// find the p-value or significance for this -const checkDiffContourSignificanceCTC = function ( - diffValue, - mH, - mF, - mM, - mC, - sH, - sF, - sM, - sC, - sigType, - statistic -) { - const minuendData = { - hit: mH, - fa: mF, - miss: mM, - cn: mC, - }; - const subtrahendData = { - hit: sH, - fa: sF, - miss: sM, - cn: sC, - }; - const errorLength = ctcErrorPython(statistic, minuendData, subtrahendData); - const upperBound = diffValue + errorLength; - const lowerBound = diffValue - errorLength; - return (upperBound > 0 && lowerBound > 0) || (upperBound < 0 && lowerBound < 0); -}; +// utility for getting CTC error bar length via Python +const ctcErrorPython = function (statistic, minuendData, subtrahendData) { + if (Meteor.isServer) { + // send the data to the python script + const pyOptions = { + mode: "text", + pythonPath: Meteor.settings.private.PYTHON_PATH, + pythonOptions: ["-u"], // get print results in real-time + scriptPath: + process.env.NODE_ENV === "development" + ? `${process.env.PWD}/.meteor/local/build/programs/server/assets/packages/randyp_mats-common/public/python/` + : `${process.env.PWD}/programs/server/assets/packages/randyp_mats-common/public/python/`, + args: [ + "-S", + statistic, + "-m", + JSON.stringify(minuendData), + "-s", + JSON.stringify(subtrahendData), + ], + }; + const pyShell = require("python-shell"); + const Future = require("fibers/future"); -// use a student's t-test to see if a point on a contour diff is statistically significant -const checkDiffContourSignificance = function (x1, x2, s1, s2, n1, n2, sigType) { - const t = getTValue(x1, x2, s1, s2, n1, n2); - const df = getDfValue(x1, x2, s1, s2, n1, n2); - return isStudentTTestValueSignificant(t, df, sigType); + const future = new Future(); + let error; + let errorLength = 0; + pyShell.PythonShell.run("python_ctc_error.py", pyOptions) + .then((results) => { + // parse the results or set an error + if (results === undefined || results === "undefined") { + error = + "Error thrown by python_ctc_error.py. Please write down exactly how you produced this error, and submit a ticket at mats.gsl@noaa.gov."; + } else { + // get the data back from the query + const parsedData = JSON.parse(results); + errorLength = Number(parsedData.error_length); + } + // done waiting - have results + future.return(); + }) + .catch((err) => { + error = err.message; + future.return(); + }); + + // wait for future to finish + future.wait(); + if (error) { + throw new Error(`Error when calculating CTC errorbars: ${error}`); + } + return errorLength; + } + return null; }; // calculate the t value for a student's t-test @@ -1457,6 +1491,45 @@ const isStudentTTestValueSignificant = function (t, df, sigType) { return t > sigThresh; }; +// find the p-value or significance for this +const checkDiffContourSignificanceCTC = function ( + diffValue, + mH, + mF, + mM, + mC, + sH, + sF, + sM, + sC, + sigType, + statistic +) { + const minuendData = { + hit: mH, + fa: mF, + miss: mM, + cn: mC, + }; + const subtrahendData = { + hit: sH, + fa: sF, + miss: sM, + cn: sC, + }; + const errorLength = ctcErrorPython(statistic, minuendData, subtrahendData); + const upperBound = diffValue + errorLength; + const lowerBound = diffValue - errorLength; + return (upperBound > 0 && lowerBound > 0) || (upperBound < 0 && lowerBound < 0); +}; + +// use a student's t-test to see if a point on a contour diff is statistically significant +const checkDiffContourSignificance = function (x1, x2, s1, s2, n1, n2, sigType) { + const t = getTValue(x1, x2, s1, s2, n1, n2); + const df = getDfValue(x1, x2, s1, s2, n1, n2); + return isStudentTTestValueSignificant(t, df, sigType); +}; + // utility to process the user-input histogram customization controls const setHistogramParameters = function (plotParams) { const yAxisFormat = plotParams["histogram-yaxis-controls"]; @@ -1471,7 +1544,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins": // get the user's chosen number of bins binNum = Number(plotParams["bin-number"]); - if (isNaN(binNum)) { + if (Number.isNaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1486,7 +1559,7 @@ const setHistogramParameters = function (plotParams) { case "Choose a bin bound": // let the histogram routine know that we want the bins shifted over to whatever was input pivotVal = Number(plotParams["bin-pivot"]); - if (isNaN(pivotVal)) { + if (Number.isNaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1494,7 +1567,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and make zero a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to zero binNum = Number(plotParams["bin-number"]); - if (isNaN(binNum)) { + if (Number.isNaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1505,13 +1578,13 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and choose a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to whatever was input binNum = Number(plotParams["bin-number"]); - if (isNaN(binNum)) { + if (Number.isNaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } pivotVal = Number(plotParams["bin-pivot"]); - if (isNaN(pivotVal)) { + if (Number.isNaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1520,10 +1593,10 @@ const setHistogramParameters = function (plotParams) { // try to parse whatever we've been given for bin bounds. Throw an error if they didn't follow directions to enter a comma-separated list of numbers. try { binBounds = plotParams["bin-bounds"].split(",").map(function (item) { - item.trim(); - item = Number(item); - if (!isNaN(item)) { - return item; + let thisItem = item.trim(); + thisItem = Number(thisItem); + if (!Number.isNaN(thisItem)) { + return thisItem; } throw new Error( "Error parsing bin bounds: please enter at least two numbers delimited by commas." @@ -1546,17 +1619,17 @@ const setHistogramParameters = function (plotParams) { case "Manual bin start, number, and stride": // get the bin start, number, and stride. binNum = Number(plotParams["bin-number"]); - if (isNaN(binNum)) { + if (Number.isNaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } binStart = Number(plotParams["bin-start"]); - if (isNaN(binStart)) { + if (Number.isNaN(binStart)) { throw new Error("Error parsing bin start: please enter the desired bin start."); } binStride = Number(plotParams["bin-stride"]); - if (isNaN(binStride)) { + if (Number.isNaN(binStride)) { throw new Error( "Error parsing bin stride: please enter the desired bin stride." ); @@ -1596,12 +1669,12 @@ const calculateHistogramBins = function ( let binMeans = []; // calculate the global stats across all of the data - const globalStats = get_err(curveSubStats, curveSubSecs, [], { + const globalStats = getErr(curveSubStats, curveSubSecs, [], { hasLevels: false, outliers: appParams.outliers, }); // we don't need levels for the mean or sd calculations, so just pass in an empty array - const glob_mean = globalStats.d_mean; - const glob_sd = globalStats.sd; + const globMean = globalStats.dMean; + const globSd = globalStats.sd; let fullLowBound; let fullUpBound; @@ -1610,9 +1683,9 @@ const calculateHistogramBins = function ( if (binParams.binStart === undefined || binParams.binStride === undefined) { // use the global stats to determine the bin bounds -- should be based on dividing up +/- 3*sd from the mean into requested number of bins - fullLowBound = glob_mean - 3 * glob_sd; - fullUpBound = glob_mean + 3 * glob_sd; - fullRange = 6 * glob_sd; + fullLowBound = globMean - 3 * globSd; + fullUpBound = globMean + 3 * globSd; + fullRange = 6 * globSd; binInterval = fullRange / (binParams.binNum - 2); // take off two bins from the total number of requested bins to represent values either less than - 3*sd from the mean or greater than 3*sd from the mean } else { // use the user-defined start, number, and stride to determine the bin bounds @@ -1626,16 +1699,16 @@ const calculateHistogramBins = function ( binUpBounds[0] = fullLowBound; // the first upper bound should be exactly - 3*sd from the mean, or the previously calculated fullLowBound binLowBounds[0] = -1 * Number.MAX_VALUE; binMeans[0] = fullLowBound - binInterval / 2; - for (var b_idx = 1; b_idx < binParams.binNum - 1; b_idx++) { - binUpBounds[b_idx] = binUpBounds[b_idx - 1] + binInterval; // increment from fullLowBound to get the rest of the bin upper limits - binLowBounds[b_idx] = binUpBounds[b_idx - 1]; - binMeans[b_idx] = binUpBounds[b_idx - 1] + binInterval / 2; + for (let bIdx = 1; bIdx < binParams.binNum - 1; bIdx += 1) { + binUpBounds[bIdx] = binUpBounds[bIdx - 1] + binInterval; // increment from fullLowBound to get the rest of the bin upper limits + binLowBounds[bIdx] = binUpBounds[bIdx - 1]; + binMeans[bIdx] = binUpBounds[bIdx - 1] + binInterval / 2; } binUpBounds[binParams.binNum - 1] = Number.MAX_VALUE; // the last bin should have everything too large to fit into the previous bins, so make its upper bound the max number value binLowBounds[binParams.binNum - 1] = fullUpBound; binMeans[binParams.binNum - 1] = fullUpBound + binInterval / 2; - if (binParams.pivotVal !== undefined && !isNaN(binParams.pivotVal)) { + if (binParams.pivotVal !== undefined && !Number.isNaN(binParams.pivotVal)) { // need to shift the bounds and means over so that one of the bounds is on the chosen pivot const closestBoundToPivot = binLowBounds.reduce(function (prev, curr) { return Math.abs(curr - binParams.pivotVal) < Math.abs(prev - binParams.pivotVal) @@ -1657,20 +1730,20 @@ const calculateHistogramBins = function ( const binLabels = []; let lowSdFromMean; let upSdFromMean; - for (b_idx = 0; b_idx < binParams.binNum; b_idx++) { - lowSdFromMean = binLowBounds[b_idx].toFixed(2); - upSdFromMean = binUpBounds[b_idx].toFixed(2); - if (b_idx === 0) { - binLabels[b_idx] = `< ${upSdFromMean}`; - } else if (b_idx === binParams.binNum - 1) { - binLabels[b_idx] = `> ${lowSdFromMean}`; + for (let bIdx = 0; bIdx < binParams.binNum; bIdx += 1) { + lowSdFromMean = binLowBounds[bIdx].toFixed(2); + upSdFromMean = binUpBounds[bIdx].toFixed(2); + if (bIdx === 0) { + binLabels[bIdx] = `< ${upSdFromMean}`; + } else if (bIdx === binParams.binNum - 1) { + binLabels[bIdx] = `> ${lowSdFromMean}`; } else { - binLabels[b_idx] = `${lowSdFromMean}-${upSdFromMean}`; + binLabels[bIdx] = `${lowSdFromMean}-${upSdFromMean}`; } } - binStats.glob_mean = glob_mean; - binStats.glob_sd = glob_sd; + binStats.glob_mean = globMean; + binStats.glob_sd = globSd; binStats.binUpBounds = binUpBounds; binStats.binLowBounds = binLowBounds; binStats.binMeans = binMeans; @@ -1687,17 +1760,18 @@ const prescribeHistogramBins = function ( appParams ) { const binStats = {}; + const returnBinParams = binParams; // calculate the global stats across all of the data - const globalStats = get_err(curveSubStats, curveSubSecs, [], { + const globalStats = getErr(curveSubStats, curveSubSecs, [], { hasLevels: false, outliers: appParams.outliers, }); // we don't need levels for the mean or sd calculations, so just pass in an empty array - const glob_mean = globalStats.d_mean; - const glob_sd = globalStats.sd; + const globMean = globalStats.dMean; + const globSd = globalStats.sd; // make sure the user-defined bins are in order from least to greatest - binParams.binBounds = binParams.binBounds.sort(function (a, b) { + returnBinParams.binBounds = returnBinParams.binBounds.sort(function (a, b) { return Number(a) - Number(b); }); @@ -1706,14 +1780,14 @@ const prescribeHistogramBins = function ( const binLowBounds = []; const binMeans = []; let binIntervalSum = 0; - for (var b_idx = 1; b_idx < binParams.binNum - 1; b_idx++) { - binUpBounds[b_idx] = binParams.binBounds[b_idx]; - binLowBounds[b_idx] = binParams.binBounds[b_idx - 1]; - binMeans[b_idx] = (binUpBounds[b_idx] + binLowBounds[b_idx]) / 2; - binIntervalSum += binUpBounds[b_idx] - binLowBounds[b_idx]; + for (let bIdx = 1; bIdx < binParams.binNum - 1; bIdx += 1) { + binUpBounds[bIdx] = binParams.binBounds[bIdx]; + binLowBounds[bIdx] = binParams.binBounds[bIdx - 1]; + binMeans[bIdx] = (binUpBounds[bIdx] + binLowBounds[bIdx]) / 2; + binIntervalSum += binUpBounds[bIdx] - binLowBounds[bIdx]; } const binIntervalAverage = binIntervalSum / (binParams.binNum - 2); - binUpBounds[0] = binLowBounds[1]; + [, binUpBounds[0]] = binLowBounds; binLowBounds[0] = -1 * Number.MAX_VALUE; // the first bin should have everything too small to fit into the other bins, so make its lower bound -1 * the max number value binMeans[0] = binLowBounds[1] - binIntervalAverage / 2; // the bin means for the edge bins is a little arbitrary, so base it on the average bin width binUpBounds[binParams.binNum - 1] = Number.MAX_VALUE; // the last bin should have everything too large to fit into the previous bins, so make its upper bound the max number value @@ -1725,20 +1799,20 @@ const prescribeHistogramBins = function ( const binLabels = []; let lowSdFromMean; let upSdFromMean; - for (b_idx = 0; b_idx < binParams.binNum; b_idx++) { - lowSdFromMean = binLowBounds[b_idx].toFixed(2); - upSdFromMean = binUpBounds[b_idx].toFixed(2); - if (b_idx === 0) { - binLabels[b_idx] = `< ${upSdFromMean}`; - } else if (b_idx === binParams.binNum - 1) { - binLabels[b_idx] = `> ${lowSdFromMean}`; + for (let bIdx = 0; bIdx < binParams.binNum; bIdx += 1) { + lowSdFromMean = binLowBounds[bIdx].toFixed(2); + upSdFromMean = binUpBounds[bIdx].toFixed(2); + if (bIdx === 0) { + binLabels[bIdx] = `< ${upSdFromMean}`; + } else if (bIdx === binParams.binNum - 1) { + binLabels[bIdx] = `> ${lowSdFromMean}`; } else { - binLabels[b_idx] = `${lowSdFromMean}-${upSdFromMean}`; + binLabels[bIdx] = `${lowSdFromMean}-${upSdFromMean}`; } } - binStats.glob_mean = glob_mean; - binStats.glob_sd = glob_sd; + binStats.glob_mean = globMean; + binStats.glob_sd = globSd; binStats.binUpBounds = binUpBounds; binStats.binLowBounds = binLowBounds; binStats.binMeans = binMeans; @@ -1762,21 +1836,21 @@ const sortHistogramBins = function ( const binSubStats = {}; const binSubSecs = {}; const binSubLevs = {}; + const returnD = d; - for (var b_idx = 0; b_idx < binNum; b_idx++) { - binSubStats[b_idx] = []; - binSubSecs[b_idx] = []; - binSubLevs[b_idx] = []; + for (let bIdx = 0; bIdx < binNum; bIdx += 1) { + binSubStats[bIdx] = []; + binSubSecs[bIdx] = []; + binSubLevs[bIdx] = []; } // calculate the global stats across all of the data - let globalStats; - globalStats = get_err(curveSubStats, curveSubSecs, curveSubLevs, appParams); - const glob_mean = globalStats.d_mean; - const glob_sd = globalStats.sd; - const glob_n = globalStats.n_good; - const glob_max = globalStats.maxVal; - const glob_min = globalStats.minVal; + const globalStats = getErr(curveSubStats, curveSubSecs, curveSubLevs, appParams); + const globMean = globalStats.dMean; + const globSd = globalStats.sd; + const globN = globalStats.nGood; + const globMax = globalStats.maxVal; + const globMin = globalStats.minVal; // sort data into bins const { binUpBounds } = masterBinStats; @@ -1784,14 +1858,14 @@ const sortHistogramBins = function ( const { binMeans } = masterBinStats; const { binLabels } = masterBinStats; - for (let d_idx = 0; d_idx < curveSubStats.length; d_idx++) { + for (let dIdx = 0; dIdx < curveSubStats.length; dIdx += 1) { // iterate through all of the bins until we find one where the upper limit is greater than our datum. - for (b_idx = 0; b_idx < binNum; b_idx++) { - if (curveSubStats[d_idx] <= binUpBounds[b_idx]) { - binSubStats[b_idx].push(curveSubStats[d_idx]); - binSubSecs[b_idx].push(curveSubSecs[d_idx]); + for (let bIdx = 0; bIdx < binNum; bIdx += 1) { + if (curveSubStats[dIdx] <= binUpBounds[bIdx]) { + binSubStats[bIdx].push(curveSubStats[dIdx]); + binSubSecs[bIdx].push(curveSubSecs[dIdx]); if (appParams.hasLevels) { - binSubLevs[b_idx].push(curveSubLevs[d_idx]); + binSubLevs[bIdx].push(curveSubLevs[dIdx]); } break; } @@ -1801,24 +1875,17 @@ const sortHistogramBins = function ( // calculate the statistics for each bin // we are especially interested in the number of values in each bin, as that is the plotted stat in a histogram let binStats; - let bin_mean; - let bin_sd; - let bin_n; - let bin_rf; - - let sum = 0; - let count = 0; - for (b_idx = 0; b_idx < binNum; b_idx++) { - binStats = get_err( - binSubStats[b_idx], - binSubSecs[b_idx], - binSubLevs[b_idx], - appParams - ); - bin_mean = binStats.d_mean; - bin_sd = binStats.sd; - bin_n = binStats.n_good; - bin_rf = bin_n / glob_n; + let binMean; + let binSd; + let binN; + let binRf; + + for (let bIdx = 0; bIdx < binNum; bIdx += 1) { + binStats = getErr(binSubStats[bIdx], binSubSecs[bIdx], binSubLevs[bIdx], appParams); + binMean = binStats.dMean; + binSd = binStats.sd; + binN = binStats.nGood; + binRf = binN / globN; /* var d = {// d will contain the curve data @@ -1847,44 +1914,42 @@ const sortHistogramBins = function ( }; */ - d.x.push(binMeans[b_idx]); - d.y.push(bin_n); - d.subVals.push(binSubStats[b_idx]); - d.subSecs.push(binSubSecs[b_idx]); - d.bin_stats.push({ - bin_mean, - bin_sd, - bin_n, - bin_rf, - binLowBound: binLowBounds[b_idx], - binUpBound: binUpBounds[b_idx], - binLabel: binLabels[b_idx], + returnD.x.push(binMeans[bIdx]); + returnD.y.push(binN); + returnD.subVals.push(binSubStats[bIdx]); + returnD.subSecs.push(binSubSecs[bIdx]); + returnD.bin_stats.push({ + bin_mean: binMean, + bin_sd: binSd, + bin_n: binN, + bin_rf: binRf, + binLowBound: binLowBounds[bIdx], + binUpBound: binUpBounds[bIdx], + binLabel: binLabels[bIdx], }); - d.text.push(null); + returnD.text.push(null); if (appParams.hasLevels) { - d.subLevs.push(binSubLevs[b_idx]); + returnD.subLevs.push(binSubLevs[bIdx]); } // set axis limits based on returned data - if (d.y[b_idx] !== null) { - sum += d.y[b_idx]; - count++; - d.ymin = d.ymin < d.y[b_idx] ? d.ymin : d.y[b_idx]; - d.ymax = d.ymax > d.y[b_idx] ? d.ymax : d.y[b_idx]; + if (returnD.y[bIdx] !== null) { + returnD.ymin = returnD.ymin < returnD.y[bIdx] ? returnD.ymin : returnD.y[bIdx]; + returnD.ymax = returnD.ymax > returnD.y[bIdx] ? returnD.ymax : returnD.y[bIdx]; } } - d.glob_stats = { - glob_mean, - glob_sd, - glob_n, - glob_max, - glob_min, + returnD.glob_stats = { + glob_mean: globMean, + glob_sd: globSd, + glob_n: globN, + glob_max: globMax, + glob_min: globMin, }; - d.xmin = d.x[0]; - d.xmax = d.x[binNum - 1]; + [returnD.xmin] = returnD.x; + returnD.xmax = returnD.x[binNum - 1]; - return { d }; + return { d: returnD }; }; // utility that takes the curve params for two contour plots and collapses them into the curve params for one diff contour. @@ -1892,7 +1957,7 @@ const getDiffContourCurveParams = function (curves) { const newCurve = {}; const curveKeys = Object.keys(curves[0]); let currKey; - for (let ckidx = 0; ckidx < curveKeys.length; ckidx++) { + for (let ckidx = 0; ckidx < curveKeys.length; ckidx += 1) { currKey = curveKeys[ckidx]; if (currKey === "color") { newCurve.color = "rgb(255,165,0)"; @@ -1905,61 +1970,6 @@ const getDiffContourCurveParams = function (curves) { return [newCurve]; }; -// utility for getting CTC error bar length via Python -const ctcErrorPython = function (statistic, minuendData, subtrahendData) { - if (Meteor.isServer) { - // send the data to the python script - const pyOptions = { - mode: "text", - pythonPath: Meteor.settings.private.PYTHON_PATH, - pythonOptions: ["-u"], // get print results in real-time - scriptPath: - process.env.NODE_ENV === "development" - ? `${process.env.PWD}/.meteor/local/build/programs/server/assets/packages/randyp_mats-common/public/python/` - : `${process.env.PWD}/programs/server/assets/packages/randyp_mats-common/public/python/`, - args: [ - "-S", - statistic, - "-m", - JSON.stringify(minuendData), - "-s", - JSON.stringify(subtrahendData), - ], - }; - const pyShell = require("python-shell"); - const Future = require("fibers/future"); - - const future = new Future(); - let error; - let errorLength = 0; - pyShell.PythonShell.run("python_ctc_error.py", pyOptions) - .then((results) => { - // parse the results or set an error - if (results === undefined || results === "undefined") { - error = - "Error thrown by python_ctc_error.py. Please write down exactly how you produced this error, and submit a ticket at mats.gsl@noaa.gov."; - } else { - // get the data back from the query - const parsedData = JSON.parse(results); - errorLength = Number(parsedData.error_length); - } - // done waiting - have results - future.return(); - }) - .catch((err) => { - error = err.message; - future.return(); - }); - - // wait for future to finish - future.wait(); - if (error) { - throw new Error(`Error when calculating CTC errorbars: ${error}`); - } - return errorLength; - } -}; - // utility to remove a point on a graph const removePoint = function ( data, @@ -1970,62 +1980,64 @@ const removePoint = function ( isScalar, hasLevels ) { - data.x.splice(di, 1); - data.y.splice(di, 1); + const returnData = data; + returnData.x.splice(di, 1); + returnData.y.splice(di, 1); if ( plotType === matsTypes.PlotTypes.performanceDiagram || plotType === matsTypes.PlotTypes.roc ) { - data.oy_all.splice(di, 1); - data.on_all.splice(di, 1); + returnData.oy_all.splice(di, 1); + returnData.on_all.splice(di, 1); } - if (data[`error_${statVarName}`].array !== undefined) { - data[`error_${statVarName}`].array.splice(di, 1); + if (returnData[`error_${statVarName}`].array !== undefined) { + returnData[`error_${statVarName}`].array.splice(di, 1); } if (isCTC) { - data.subHit.splice(di, 1); - data.subFa.splice(di, 1); - data.subMiss.splice(di, 1); - data.subCn.splice(di, 1); + returnData.subHit.splice(di, 1); + returnData.subFa.splice(di, 1); + returnData.subMiss.splice(di, 1); + returnData.subCn.splice(di, 1); } else if (isScalar) { - if (data.subSquareDiffSumX !== undefined) { - data.subSquareDiffSumX.splice(di, 1); - data.subNSumX.splice(di, 1); - data.subObsModelDiffSumX.splice(di, 1); - data.subModelSumX.splice(di, 1); - data.subObsSumX.splice(di, 1); - data.subAbsSumX.splice(di, 1); - data.subSquareDiffSumY.splice(di, 1); - data.subNSumY.splice(di, 1); - data.subObsModelDiffSumY.splice(di, 1); - data.subModelSumY.splice(di, 1); - data.subObsSumY.splice(di, 1); - data.subAbsSumY.splice(di, 1); + if (returnData.subSquareDiffSumX !== undefined) { + returnData.subSquareDiffSumX.splice(di, 1); + returnData.subNSumX.splice(di, 1); + returnData.subObsModelDiffSumX.splice(di, 1); + returnData.subModelSumX.splice(di, 1); + returnData.subObsSumX.splice(di, 1); + returnData.subAbsSumX.splice(di, 1); + returnData.subSquareDiffSumY.splice(di, 1); + returnData.subNSumY.splice(di, 1); + returnData.subObsModelDiffSumY.splice(di, 1); + returnData.subModelSumY.splice(di, 1); + returnData.subObsSumY.splice(di, 1); + returnData.subAbsSumY.splice(di, 1); } else { - data.subSquareDiffSum.splice(di, 1); - data.subNSum.splice(di, 1); - data.subObsModelDiffSum.splice(di, 1); - data.subModelSum.splice(di, 1); - data.subObsSum.splice(di, 1); - data.subAbsSum.splice(di, 1); + returnData.subSquareDiffSum.splice(di, 1); + returnData.subNSum.splice(di, 1); + returnData.subObsModelDiffSum.splice(di, 1); + returnData.subModelSum.splice(di, 1); + returnData.subObsSum.splice(di, 1); + returnData.subAbsSum.splice(di, 1); } - } else if (data.subRelHit !== undefined) { - data.subRelHit.splice(di, 1); - data.subRelCount.splice(di, 1); - data.subRelRawCount.splice(di, 1); - } - if (data.subValsX !== undefined) { - data.subValsX.splice(di, 1); - data.subValsY.splice(di, 1); + } else if (returnData.subRelHit !== undefined) { + returnData.subRelHit.splice(di, 1); + returnData.subRelCount.splice(di, 1); + returnData.subRelRawCount.splice(di, 1); + } + if (returnData.subValsX !== undefined) { + returnData.subValsX.splice(di, 1); + returnData.subValsY.splice(di, 1); } else { - data.subVals.splice(di, 1); + returnData.subVals.splice(di, 1); } - data.subSecs.splice(di, 1); + returnData.subSecs.splice(di, 1); if (hasLevels) { - data.subLevs.splice(di, 1); + returnData.subLevs.splice(di, 1); } - data.stats.splice(di, 1); - data.text.splice(di, 1); + returnData.stats.splice(di, 1); + returnData.text.splice(di, 1); + return returnData; }; // utility to add an additional null point on a graph @@ -2040,107 +2052,111 @@ const addNullPoint = function ( isScalar, hasLevels ) { - data[indVarName].splice(di, 0, newIndVar); - data[statVarName].splice(di, 0, null); + const returnData = data; + returnData[indVarName].splice(di, 0, newIndVar); + returnData[statVarName].splice(di, 0, null); if ( plotType === matsTypes.PlotTypes.performanceDiagram || plotType === matsTypes.PlotTypes.roc ) { - data.oy_all.splice(di, 0, null); - data.on_all.splice(di, 0, null); + returnData.oy_all.splice(di, 0, null); + returnData.on_all.splice(di, 0, null); } - data[`error_${statVarName}`].splice(di, 0, null); + returnData[`error_${statVarName}`].splice(di, 0, null); if (isCTC) { - data.subHit.splice(di, 0, []); - data.subFa.splice(di, 0, []); - data.subMiss.splice(di, 0, []); - data.subCn.splice(di, 0, []); + returnData.subHit.splice(di, 0, []); + returnData.subFa.splice(di, 0, []); + returnData.subMiss.splice(di, 0, []); + returnData.subCn.splice(di, 0, []); } else if (isScalar) { - if (data.subSquareDiffSumX !== undefined) { - data.subSquareDiffSumX.splice(di, 0, []); - data.subNSumX.splice(di, 0, []); - data.subObsModelDiffSumX.splice(di, 0, []); - data.subModelSumX.splice(di, 0, []); - data.subObsSumX.splice(di, 0, []); - data.subAbsSumX.splice(di, 0, []); - data.subSquareDiffSumY.splice(di, 0, []); - data.subNSumY.splice(di, 0, []); - data.subObsModelDiffSumY.splice(di, 0, []); - data.subModelSumY.splice(di, 0, []); - data.subObsSumY.splice(di, 0, []); - data.subAbsSumY.splice(di, 0, []); + if (returnData.subSquareDiffSumX !== undefined) { + returnData.subSquareDiffSumX.splice(di, 0, []); + returnData.subNSumX.splice(di, 0, []); + returnData.subObsModelDiffSumX.splice(di, 0, []); + returnData.subModelSumX.splice(di, 0, []); + returnData.subObsSumX.splice(di, 0, []); + returnData.subAbsSumX.splice(di, 0, []); + returnData.subSquareDiffSumY.splice(di, 0, []); + returnData.subNSumY.splice(di, 0, []); + returnData.subObsModelDiffSumY.splice(di, 0, []); + returnData.subModelSumY.splice(di, 0, []); + returnData.subObsSumY.splice(di, 0, []); + returnData.subAbsSumY.splice(di, 0, []); } else { - data.subSquareDiffSum.splice(di, 0, []); - data.subNSum.splice(di, 0, []); - data.subObsModelDiffSum.splice(di, 0, []); - data.subModelSum.splice(di, 0, []); - data.subObsSum.splice(di, 0, []); - data.subAbsSum.splice(di, 0, []); + returnData.subSquareDiffSum.splice(di, 0, []); + returnData.subNSum.splice(di, 0, []); + returnData.subObsModelDiffSum.splice(di, 0, []); + returnData.subModelSum.splice(di, 0, []); + returnData.subObsSum.splice(di, 0, []); + returnData.subAbsSum.splice(di, 0, []); } - } else if (data.subRelHit !== undefined) { - data.subRelHit.splice(di, 0, []); - data.subRelCount.splice(di, 0, []); - data.subRelRawCount.splice(di, 0, []); - } - if (data.subValsX !== undefined) { - data.subValsX.splice(di, 0, []); - data.subValsY.splice(di, 0, []); + } else if (returnData.subRelHit !== undefined) { + returnData.subRelHit.splice(di, 0, []); + returnData.subRelCount.splice(di, 0, []); + returnData.subRelRawCount.splice(di, 0, []); + } + if (returnData.subValsX !== undefined) { + returnData.subValsX.splice(di, 0, []); + returnData.subValsY.splice(di, 0, []); } else { - data.subVals.splice(di, 0, []); + returnData.subVals.splice(di, 0, []); } - data.subSecs.splice(di, 0, []); + returnData.subSecs.splice(di, 0, []); if (hasLevels) { - data.subLevs.splice(di, 0, []); + returnData.subLevs.splice(di, 0, []); } - data.stats.splice(di, 0, []); - data.text.splice(di, 0, []); + returnData.stats.splice(di, 0, []); + returnData.text.splice(di, 0, []); + return returnData; }; // utility to make null an existing point on a graph const nullPoint = function (data, di, statVarName, isCTC, isScalar, hasLevels) { - data[statVarName][di] = null; + const returnData = data; + returnData[statVarName][di] = null; if (isCTC) { - data.subHit[di] = []; - data.subFa[di] = []; - data.subMiss[di] = []; - data.subCn[di] = []; + returnData.subHit[di] = []; + returnData.subFa[di] = []; + returnData.subMiss[di] = []; + returnData.subCn[di] = []; } else if (isScalar) { - if (data.subSquareDiffSumX !== undefined) { - data.subSquareDiffSumX[di] = []; - data.subNSumX[di] = []; - data.subObsModelDiffSumX[di] = []; - data.subModelSumX[di] = []; - data.subObsSumX[di] = []; - data.subAbsSumX[di] = []; - data.subSquareDiffSumY[di] = []; - data.subNSumY[di] = []; - data.subObsModelDiffSumY[di] = []; - data.subModelSumY[di] = []; - data.subObsSumY[di] = []; - data.subAbsSumY[di] = []; + if (returnData.subSquareDiffSumX !== undefined) { + returnData.subSquareDiffSumX[di] = []; + returnData.subNSumX[di] = []; + returnData.subObsModelDiffSumX[di] = []; + returnData.subModelSumX[di] = []; + returnData.subObsSumX[di] = []; + returnData.subAbsSumX[di] = []; + returnData.subSquareDiffSumY[di] = []; + returnData.subNSumY[di] = []; + returnData.subObsModelDiffSumY[di] = []; + returnData.subModelSumY[di] = []; + returnData.subObsSumY[di] = []; + returnData.subAbsSumY[di] = []; } else { - data.subSquareDiffSum[di] = []; - data.subNSum[di] = []; - data.subObsModelDiffSum[di] = []; - data.subModelSum[di] = []; - data.subObsSum[di] = []; - data.subAbsSum[di] = []; + returnData.subSquareDiffSum[di] = []; + returnData.subNSum[di] = []; + returnData.subObsModelDiffSum[di] = []; + returnData.subModelSum[di] = []; + returnData.subObsSum[di] = []; + returnData.subAbsSum[di] = []; } - } else if (data.subRelHit !== undefined) { - data.subRelHit[di] = []; - data.subRelCount[di] = []; - data.subRelRawCount[di] = []; - } - if (data.subValsX !== undefined) { - data.subValsX[di] = []; - data.subValsY[di] = []; + } else if (returnData.subRelHit !== undefined) { + returnData.subRelHit[di] = []; + returnData.subRelCount[di] = []; + returnData.subRelRawCount[di] = []; + } + if (returnData.subValsX !== undefined) { + returnData.subValsX[di] = []; + returnData.subValsY[di] = []; } else { - data.subVals[di] = []; + returnData.subVals[di] = []; } - data.subSecs[di] = []; + returnData.subSecs[di] = []; if (hasLevels) { - data.subLevs[di] = []; + returnData.subLevs[di] = []; } + return returnData; }; // used for sorting arrays @@ -2151,6 +2167,7 @@ const sortFunction = function (a, b) { return a[0] < b[0] ? -1 : 1; }; +// eslint-disable-next-line no-undef export default matsDataUtils = { areObjectsEqual, arrayContainsArray, @@ -2179,7 +2196,7 @@ export default matsDataUtils = { readableStandardRegions, readableTCCategories, readableAdeckModels, - get_err, + getErr, ctcErrorPython, checkDiffContourSignificance, checkDiffContourSignificanceCTC, diff --git a/meteor_packages/mats-common/imports/startup/server/publications.js b/meteor_packages/mats-common/imports/startup/server/publications.js index a447690d5..bf4fe60d5 100644 --- a/meteor_packages/mats-common/imports/startup/server/publications.js +++ b/meteor_packages/mats-common/imports/startup/server/publications.js @@ -6,6 +6,8 @@ import { Meteor } from "meteor/meteor"; import { matsCollections } from "meteor/randyp:mats-common"; import { curveParamsByApp } from "../both/mats-curve-params"; +/* eslint-disable no-console */ + const _publishField = function (field) { Meteor.publish(field, function () { const data = matsCollections[field].find({}); @@ -27,7 +29,7 @@ if (Meteor.isServer) { ); } let currParam; - for (let i = 0; i < params.length; i++) { + for (let i = 0; i < params.length; i += 1) { currParam = params[i]; _publishField(currParam); } diff --git a/meteor_packages/mats-common/templates/textOutput/textOutput.js b/meteor_packages/mats-common/templates/textOutput/textOutput.js index cccf039d7..9747642e1 100644 --- a/meteor_packages/mats-common/templates/textOutput/textOutput.js +++ b/meteor_packages/mats-common/templates/textOutput/textOutput.js @@ -397,8 +397,7 @@ Template.textOutput.helpers({ const elementLabel = ""; let line = ""; const isCTC = element.hit !== undefined && element.hit !== null; - const isModeSingle = - element.nForecast !== undefined && element.nForecast !== null; + const isModeSingle = element.nForecast !== undefined && element.nForecast !== null; const isModePairs = element.avgInterest !== undefined && element.avgInterest !== null; const plotType = Session.get("plotType"); From 8861fbf5f053ddbd655f52e18cc2711a63e3e214 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Thu, 15 Aug 2024 13:53:11 -0600 Subject: [PATCH 35/55] Linted matsMethods --- .eslintrc.json | 3 +- .../.npm/package/npm-shrinkwrap.json | 62 +- .../imports/startup/api/matsMethods.js | 7049 +++++++++-------- .../startup/server/data_plot_ops_util.js | 4 +- .../startup/server/data_process_util.js | 68 +- .../imports/startup/server/data_query_util.js | 52 +- .../imports/startup/server/data_util.js | 50 +- meteor_packages/mats-common/package.js | 52 +- 8 files changed, 3701 insertions(+), 3639 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 0d777e35d..3c5c1d052 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -24,7 +24,7 @@ // our dependencies. // XXX: this *should* be taken care of by eslint-import-resolver-meteor, investigate. // "import/no-extraneous-dependencies": "off", - // "no-underscore-dangle": ["error", { "allow": ["_id", "_ensureIndex"] }], + "no-underscore-dangle": ["error", { "allow": ["_id", "_ensureIndex"] }], "object-shorthand": ["error", "always", { "avoidQuotes": false }], "space-before-function-paren": "off", // for Meteor API's that rely on `this` context, e.g. Template.onCreated and publications @@ -51,7 +51,6 @@ "radix": "warn", "global-require": "warn", "no-lonely-if": "warn", - "no-underscore-dangle": ["off", { "allow": ["_id", "_ensureIndex"] }], // Uncomment line 27 with Meteor's recommended "no-underscore-dangle" policy when resolving this "no-return-assign": "warn", "no-shadow": "warn", "new-cap": "warn", diff --git a/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json b/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json index 2cfe6142f..e10d6f950 100644 --- a/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json +++ b/meteor_packages/mats-common/.npm/package/npm-shrinkwrap.json @@ -42,9 +42,9 @@ "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "axios": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", - "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==" + "version": "1.7.4", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.4.tgz", + "integrity": "sha512-DukmaFRnY6AzAALSH4J2M3k6PkaC+MfaAGdEERRWcC9q3/TWQwLpHR8ZRLKTdQ3aBDL64EdluRDjJqKw+BPZEw==" }, "call-bind": { "version": "1.0.7", @@ -61,11 +61,6 @@ "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==" }, - "clone": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz", - "integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==" - }, "cmake-js": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/cmake-js/-/cmake-js-7.3.0.tgz", @@ -127,9 +122,9 @@ "integrity": "sha512-DxWXvvPq4srWLCqFugqSV+6CBt/CvQ0dnpXhQ3gl0autcIDAruG1PuGG3gC7yPRNytAD1oU1AcUOzaYhOawhTw==" }, "debug": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.5.tgz", - "integrity": "sha512-pt0bNEmneDIvdL1Xsd9oDQ/wrQRkXDT4AUWlNZNPKvW5x/jyO9VFXkJUP07vQ2upmw5PlaITaPKc31jK13V+jg==" + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.6.tgz", + "integrity": "sha512-O/09Bd4Z1fBrU4VzkhFqVgpPzaGbw6Sm9FEkBT1A/YBXQFGuuSxa1dN2nxgxS34JmKXqYx8CZAwEVoJFImUXIg==" }, "deep-extend": { "version": "0.6.0", @@ -717,40 +712,40 @@ "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==" }, - "mongo-object": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mongo-object/-/mongo-object-3.0.1.tgz", - "integrity": "sha512-EbiwWHvKOF9xhIzuwaqknwPISdkHMipjMs6DiJFicupgBBLEhUs0OOro9MuPkFogB17DZlsV4KJhhxfqZ7ZRMQ==" - }, "ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node-addon-api": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.0.tgz", - "integrity": "sha512-mNcltoe1R8o7STTegSOHdnJNN7s5EUvhoS7ShnTHDyOSd+8H+UdWODq6qSv67PjC8Zc5JRT8+oLAMCr0SIXw7g==" + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-7.1.1.tgz", + "integrity": "sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==" }, "node-api-headers": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/node-api-headers/-/node-api-headers-1.1.0.tgz", - "integrity": "sha512-ucQW+SbYCUPfprvmzBsnjT034IGRB2XK8rRc78BgjNKhTdFKgAwAmgW704bKIBmcYW48it0Gkjpkd39Azrwquw==" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/node-api-headers/-/node-api-headers-1.2.0.tgz", + "integrity": "sha512-L9AiEkBfgupC0D/LsudLPOhzy/EdObsp+FHyL1zSK0kKv5FDA9rJMoRz8xd+ojxzlqfg0tTZm2h8ot2nS7bgRA==" }, "node-file-cache": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/node-file-cache/-/node-file-cache-1.0.2.tgz", "integrity": "sha512-X+u705v8tTdOnNhTkkSdb36QbuILf1MjSmSb8JlrpzLIzlLFu7qmCF3ezaAFPz9w5vFe0NpZfa7kn2v6eW+SAg==" }, + "nodemailer": { + "version": "6.9.14", + "resolved": "https://registry.npmjs.org/nodemailer/-/nodemailer-6.9.14.tgz", + "integrity": "sha512-Dobp/ebDKBvz91sbtRKhcznLThrKxKt97GI2FAlAyy+fk19j73Uz3sBXolVtmcXjaorivqsbbbjDY+Jkt4/bQA==" + }, "npmlog": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-6.0.2.tgz", "integrity": "sha512-/vBvz5Jfr9dT/aFWd0FIRf+T/Q2WBsLENygUaFUqstqsycmZAP/t5BvFJTK0viFmSUxiUKTUplWy5vt+rvKIxg==" }, "object-inspect": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.1.tgz", - "integrity": "sha512-5qoj1RUiKOMsCCNLV1CBiPYE10sziTsnmNxkAI/rZhiD63CF7IqdFGC/XzjWjpSgLf0LxXX3bDFIh0E18f6UhQ==" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.2.tgz", + "integrity": "sha512-IRZSRuzJiynemAXPYtPe5BoI/RESNYR7TYm50MC5Mqbd3Jmw5y790sErYw3V6SryFJD64b74qQQs9wn5Bg/k3g==" }, "proxy-from-env": { "version": "1.1.0", @@ -768,9 +763,9 @@ "integrity": "sha512-RUOOOjHLhgR1MIQrCtnEqz/HJ1RMZBIN+REnpSUrfft2bXqXy69fwJASVziWExfFXsR1bCY0TznnHooNsCo0/w==" }, "qs": { - "version": "6.12.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.12.1.tgz", - "integrity": "sha512-zWmv4RSuB9r2mYQw3zxQuHWeU+42aKi1wWig/j4ele4ygELZ7PEO6MM7rim9oAQH2A5MWfsAVf/jPvTPgCbvUQ==" + "version": "6.13.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.13.0.tgz", + "integrity": "sha512-+38qI9SOr8tfZ4QmJNplMUxqjbe7LKvvZgWdExBOmd+egZTtjLB67Gu0HRX3u/XOq7UU2Nx6nsjvS16Z9uwfpg==" }, "rc": { "version": "1.2.8", @@ -798,9 +793,9 @@ "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==" + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==" }, "set-blocking": { "version": "2.0.0", @@ -822,11 +817,6 @@ "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==" }, - "simpl-schema": { - "version": "3.4.6", - "resolved": "https://registry.npmjs.org/simpl-schema/-/simpl-schema-3.4.6.tgz", - "integrity": "sha512-xgShTrNzktC1TTgizSjyDHrxs0bmZa1b9sso54cL8xwO2OloVhtHjfO73/dAK9OFzUIWCBTpKMpD12JPTgVimA==" - }, "steno": { "version": "0.4.4", "resolved": "https://registry.npmjs.org/steno/-/steno-0.4.4.tgz", diff --git a/meteor_packages/mats-common/imports/startup/api/matsMethods.js b/meteor_packages/mats-common/imports/startup/api/matsMethods.js index bb8e5d4e0..38a70de68 100644 --- a/meteor_packages/mats-common/imports/startup/api/matsMethods.js +++ b/meteor_packages/mats-common/imports/startup/api/matsMethods.js @@ -4,7 +4,8 @@ import { Meteor } from "meteor/meteor"; import { ValidatedMethod } from "meteor/mdg:validated-method"; -import SimpleSchema from "simpl-schema"; +import SimpleSchema from "meteor/aldeed:simple-schema"; +import { Picker } from "meteor/meteorhacks:picker"; import { matsCache, matsCollections, @@ -15,10 +16,17 @@ import { versionInfo, } from "meteor/randyp:mats-common"; import { mysql } from "meteor/pcel:mysql"; -import { url } from "url"; +import { moment } from "meteor/momentjs:moment"; +import { _ } from "meteor/underscore"; import { Mongo } from "meteor/mongo"; import { curveParamsByApp } from "../both/mats-curve-params"; +/* global cbPool, cbScorecardPool, cbScorecardSettingsPool, appSpecificResetRoutines */ + +/* eslint-disable no-await-in-loop */ +/* eslint-disable no-console */ +/* eslint-disable global-require */ + // PRIVATE // local collection used to keep the table update times for refresh - won't ever be synchronized or persisted. @@ -30,714 +38,1676 @@ const DownSampleResults = new Mongo.Collection("DownSampleResults"); // utility to check for empty object const isEmpty = function (map) { - for (const key in map) { - if (map.hasOwnProperty(key)) { - return false; - } - } - return true; + const mapKeys = Object.keys(map); + return mapKeys.length === 0; }; -// Define routes for server -if (Meteor.isServer) { - // add indexes to result and axes collections - DownSampleResults.rawCollection().createIndex( - { - createdAt: 1, - }, - { - expireAfterSeconds: 3600 * 8, - } - ); // 8 hour expiration - LayoutStoreCollection.rawCollection().createIndex( - { - createdAt: 1, - }, - { - expireAfterSeconds: 900, - } - ); // 15 min expiration - // set the default proxy prefix path to "" - // If the settings are not complete, they will be set by the configuration and written out, which will cause the app to reset - if (Meteor.settings.public && !Meteor.settings.public.proxy_prefix_path) { - Meteor.settings.public.proxy_prefix_path = ""; +// private middleware for getting the status - think health check +const status = function (res) { + if (Meteor.isServer) { + const settings = matsCollections.Settings.findOne(); + res.end( + `
Running: version - ${settings.appVersion}
` + ); } +}; - Picker.route("/status", function (params, req, res, next) { - Picker.middleware(_status(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/status`, - function (params, req, res, next) { - Picker.middleware(_status(params, req, res, next)); +// private - used to see if the main page needs to update its selectors +const checkMetaDataRefresh = async function () { + // This routine compares the current last modified time of the tables (MYSQL) or documents (Couchbase) + // used for curveParameter metadata with the last update time to determine if an update is necessary. + // We really only do this for Curveparams + /* + metaDataTableUpdates: + { + name: dataBaseName(MYSQL) or bucketName(couchbase), + (for couchbase tables are documents) + tables: [tableName1, tableName2 ..], + lastRefreshed : timestamp + } + */ + let refresh = false; + const tableUpdates = metaDataTableUpdates.find({}).fetch(); + const dbType = + matsCollections.Settings.findOne() !== undefined + ? matsCollections.Settings.findOne().dbType + : matsTypes.DbTypes.mysql; + for (let tui = 0; tui < tableUpdates.length; tui += 1) { + const id = tableUpdates[tui]._id; + const poolName = tableUpdates[tui].pool; + const dbName = tableUpdates[tui].name; + const tableNames = tableUpdates[tui].tables; + const { lastRefreshed } = tableUpdates[tui]; + let updatedEpoch = Number.MAX_VALUE; + let rows; + let doc; + for (let ti = 0; ti < tableNames.length; ti += 1) { + const tName = tableNames[ti]; + try { + if (Meteor.isServer) { + switch (dbType) { + case matsTypes.DbTypes.mysql: + rows = matsDataQueryUtils.simplePoolQueryWrapSynchronous( + global[poolName], + `SELECT UNIX_TIMESTAMP(UPDATE_TIME)` + + ` FROM information_schema.tables` + + ` WHERE TABLE_SCHEMA = '${dbName}'` + + ` AND TABLE_NAME = '${tName}'` + ); + updatedEpoch = rows[0]["UNIX_TIMESTAMP(UPDATE_TIME)"]; + break; + case matsTypes.DbTypes.couchbase: + // the tName for couchbase is supposed to be the document id + doc = await cbPool.getCB(tName); + updatedEpoch = doc.updated; + break; + default: + throw new Meteor.Error("resetApp: undefined DbType"); + } + } + // console.log("DB says metadata for table " + dbName + "." + tName + " was updated at " + updatedEpoch); + if ( + updatedEpoch === undefined || + updatedEpoch === null || + updatedEpoch === "NULL" || + updatedEpoch === Number.MAX_VALUE + ) { + // if time of last update isn't stored by the database (thanks, Aurora DB), refresh automatically + // console.log("_checkMetaDataRefresh - cannot find last update time for database: " + dbName + " and table: " + tName); + refresh = true; + // console.log("FORCED Refreshing the metadata for table because updatedEpoch is undefined" + dbName + "." + tName + " : updated at " + updatedEpoch); + break; + } + } catch (e) { + throw new Error( + `_checkMetaDataRefresh - error finding last update time for database: ${dbName} and table: ${tName}, ERROR:${e.message}` + ); + } + const lastRefreshedEpoch = moment.utc(lastRefreshed).valueOf() / 1000; + const updatedEpochMoment = moment.utc(updatedEpoch).valueOf(); + // console.log("Epoch of when this app last refreshed metadata for table " + dbName + "." + tName + " is " + lastRefreshedEpoch); + // console.log("Epoch of when the DB says table " + dbName + "." + tName + " was last updated is " + updatedEpochMoment); + if (lastRefreshedEpoch < updatedEpochMoment || updatedEpochMoment === 0) { + // Aurora DB sometimes returns a 0 for last updated. In that case, do refresh the metadata. + refresh = true; + // console.log("Refreshing the metadata in the app selectors because table " + dbName + "." + tName + " was updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " while the metadata was last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); + break; + } else { + // console.log("NOT Refreshing the metadata for table " + dbName + "." + tName + " : updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " : metadata last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); + } } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/status`, - function (params, req, res, next) { - Picker.middleware(_status(params, req, res, next)); + if (refresh === true) { + // refresh the app metadata + // app specific routines + for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { + await global.appSpecificResetRoutines[ai](); + } + // remember that we updated ALL the metadata tables just now + metaDataTableUpdates.update( + { + _id: id, + }, + { + $set: { + lastRefreshed: moment().format(), + }, + } + ); } - ); - - Picker.route("/_getCSV/:key", function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - }); + } + return true; +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/_getCSV/:key`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - } - ); +// private middleware for clearing the cache +const clearCache = function (res) { + if (Meteor.isServer) { + matsCache.clear(); + res.end("

clearCache Done!

"); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/app:/_getCSV/:key`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); +// private middleware for dropping a distinct instance (a single run) of a scorecard +const dropThisScorecardInstance = async function ( + userName, + name, + submittedTime, + processedAt +) { + try { + if (cbScorecardPool === undefined) { + throw new Meteor.Error("dropThisScorecardInstance: No cbScorecardPool defined"); } - ); - - Picker.route("/CSV/:f/:key/:m/:a", function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - }); + const statement = `DELETE + From + vxdata._default.SCORECARD sc + WHERE + sc.type='SC' + AND sc.userName='${userName}' + AND sc.name='${name}' + AND sc.processedAt=${processedAt} + AND sc.submitted=${submittedTime};`; + return await cbScorecardPool.queryCB(statement); + // delete this result from the mongo Scorecard collection + } catch (err) { + console.log(`dropThisScorecardInstance error : ${err.message}`); + return { + error: err.message, + }; + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/CSV/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); - } - ); +// helper function for returning an array of database-distinct apps contained within a larger MATS app +function getListOfApps() { + let apps; + if ( + matsCollections.database !== undefined && + matsCollections.database.findOne({ name: "database" }) !== undefined + ) { + // get list of databases (one per app) + apps = matsCollections.database.findOne({ + name: "database", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + } else if ( + matsCollections.variable !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }) !== undefined && + matsCollections.threshold !== undefined && + matsCollections.threshold.findOne({ + name: "threshold", + }) !== undefined + ) { + // get list of apps (variables in apps that also have thresholds) + apps = matsCollections.variable.findOne({ + name: "variable", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + } else { + apps = [matsCollections.Settings.findOne().Title]; + } + return apps; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/CSV/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getCSV(params, req, res, next)); +// helper function to map a results array to specific apps +function mapArrayToApps(result) { + // put results in a map keyed by app + const newResult = {}; + const apps = getListOfApps(); + for (let aidx = 0; aidx < apps.length; aidx += 1) { + if (result[aidx] === apps[aidx]) { + newResult[apps[aidx]] = [result[aidx]]; + } else { + newResult[apps[aidx]] = result; } - ); - - Picker.route("/_getJSON/:key", function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - }); + } + return newResult; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/_getJSON/:key`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); +// helper function to map a results map to specific apps +function mapMapToApps(result) { + // put results in a map keyed by app + let newResult = {}; + let tempResult; + const apps = getListOfApps(); + const resultKeys = Object.keys(result); + if (!matsDataUtils.arraysEqual(apps.sort(), resultKeys.sort())) { + if (resultKeys.includes("Predefined region")) { + tempResult = result["Predefined region"]; + } else { + tempResult = result; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/app:/_getJSON/:key`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); + for (let aidx = 0; aidx < apps.length; aidx += 1) { + newResult[apps[aidx]] = tempResult; } - ); + } else { + newResult = result; + } + return newResult; +} - Picker.route("/JSON/:f/:key/:m/:a", function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/JSON/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/JSON/:f/:key/:m/:a`, - function (params, req, res, next) { - Picker.middleware(_getJSON(params, req, res, next)); - } - ); - - Picker.route("/clearCache", function (params, req, res, next) { - Picker.middleware(_clearCache(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/clearCache`, - function (params, req, res, next) { - Picker.middleware(_clearCache(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/clearCache`, - function (params, req, res, next) { - Picker.middleware(_clearCache(params, req, res, next)); +// helper function for returning a map of database-distinct apps contained within a larger MATS app and their DBs +function getListOfAppDBs() { + let apps; + const result = {}; + let aidx; + if ( + matsCollections.database !== undefined && + matsCollections.database.findOne({ name: "database" }) !== undefined + ) { + // get list of databases (one per app) + apps = matsCollections.database.findOne({ + name: "database", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + for (aidx = 0; aidx < apps.length; aidx += 1) { + result[apps[aidx]] = matsCollections.database.findOne({ + name: "database", + }).optionsMap[apps[aidx]].sumsDB; } - ); - - Picker.route("/getApps", function (params, req, res, next) { - Picker.middleware(_getApps(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getApps`, - function (params, req, res, next) { - Picker.middleware(_getApps(params, req, res, next)); + } else if ( + matsCollections.variable !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }) !== undefined && + matsCollections.threshold !== undefined && + matsCollections.threshold.findOne({ + name: "threshold", + }) !== undefined + ) { + // get list of apps (variables in apps that also have thresholds) + apps = matsCollections.variable.findOne({ + name: "variable", + }).options; + if (!Array.isArray(apps)) apps = Object.keys(apps); + for (aidx = 0; aidx < apps.length; aidx += 1) { + result[apps[aidx]] = matsCollections.variable.findOne({ + name: "variable", + }).optionsMap[apps[aidx]]; + if ( + typeof result[apps[aidx]] !== "string" && + !(result[apps[aidx]] instanceof String) + ) + result[apps[aidx]] = result[apps[aidx]].sumsDB; } - ); + } else { + result[matsCollections.Settings.findOne().Title] = + matsCollections.Databases.findOne({ + role: matsTypes.DatabaseRoles.SUMS_DATA, + status: "active", + }).database; + } + return result; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getApps`, - function (params, req, res, next) { - Picker.middleware(_getApps(params, req, res, next)); +// helper function for getting a metadata map from a MATS selector, keyed by app title and model display text +function getMapByAppAndModel(selector, mapType) { + let flatJSON = ""; + try { + let result; + if ( + matsCollections[selector] !== undefined && + matsCollections[selector].findOne({ name: selector }) !== undefined && + matsCollections[selector].findOne({ name: selector })[mapType] !== undefined + ) { + // get map of requested selector's metadata + result = matsCollections[selector].findOne({ + name: selector, + })[mapType]; + let newResult = {}; + if ( + mapType === "valuesMap" || + selector === "variable" || + selector === "statistic" + ) { + // valueMaps always need to be re-keyed by app (statistic and variable get their valuesMaps from optionsMaps) + newResult = mapMapToApps(result); + result = newResult; + } else if ( + matsCollections.database === undefined && + !( + matsCollections.variable !== undefined && + matsCollections.threshold !== undefined + ) + ) { + // key by app title if we're not already + const appTitle = matsCollections.Settings.findOne().Title; + newResult[appTitle] = result; + result = newResult; + } + } else { + result = {}; } - ); - - Picker.route("/getAppSumsDBs", function (params, req, res, next) { - Picker.middleware(_getAppSumsDBs(params, req, res, next)); - }); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log(`error retrieving metadata from ${selector}: `, e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getAppSumsDBs`, - function (params, req, res, next) { - Picker.middleware(_getAppSumsDBs(params, req, res, next)); +// helper function for getting a date metadata map from a MATS selector, keyed by app title and model display text +function getDateMapByAppAndModel() { + let flatJSON = ""; + try { + let result; + // the date map can be in a few places. we have to hunt for it. + if ( + matsCollections.database !== undefined && + matsCollections.database.findOne({ + name: "database", + }) !== undefined && + matsCollections.database.findOne({ + name: "database", + }).dates !== undefined + ) { + result = matsCollections.database.findOne({ + name: "database", + }).dates; + } else if ( + matsCollections.variable !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }) !== undefined && + matsCollections.variable.findOne({ + name: "variable", + }).dates !== undefined + ) { + result = matsCollections.variable.findOne({ + name: "variable", + }).dates; + } else if ( + matsCollections["data-source"] !== undefined && + matsCollections["data-source"].findOne({ + name: "data-source", + }) !== undefined && + matsCollections["data-source"].findOne({ + name: "data-source", + }).dates !== undefined + ) { + result = matsCollections["data-source"].findOne({ + name: "data-source", + }).dates; + } else { + result = {}; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getAppSumsDBs`, - function (params, req, res, next) { - Picker.middleware(_getAppSumsDBs(params, req, res, next)); + if ( + matsCollections.database === undefined && + !( + matsCollections.variable !== undefined && + matsCollections.threshold !== undefined + ) + ) { + // key by app title if we're not already + const appTitle = matsCollections.Settings.findOne().Title; + const newResult = {}; + newResult[appTitle] = result; + result = newResult; } - ); - - Picker.route("/getModels", function (params, req, res, next) { - Picker.middleware(_getModels(params, req, res, next)); - }); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving datemap", e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getModels`, - function (params, req, res, next) { - Picker.middleware(_getModels(params, req, res, next)); +// helper function for getting a metadata map from a MATS selector, keyed by app title +function getMapByApp(selector) { + let flatJSON = ""; + try { + let result; + if ( + matsCollections[selector] !== undefined && + matsCollections[selector].findOne({ name: selector }) !== undefined + ) { + // get array of requested selector's metadata + result = matsCollections[selector].findOne({ + name: selector, + }).options; + if (!Array.isArray(result)) result = Object.keys(result); + } else if (selector === "statistic") { + result = ["ACC"]; + } else if (selector === "variable") { + result = [matsCollections.Settings.findOne().Title]; + } else { + result = []; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getModels`, - function (params, req, res, next) { - Picker.middleware(_getModels(params, req, res, next)); + // put results in a map keyed by app + let newResult; + if (result.length === 0) { + newResult = {}; + } else { + newResult = mapArrayToApps(result); } - ); - - Picker.route("/getRegions", function (params, req, res, next) { - Picker.middleware(_getRegions(params, req, res, next)); - }); + flatJSON = JSON.stringify(newResult); + } catch (e) { + console.log(`error retrieving metadata from ${selector}: `, e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getRegions`, - function (params, req, res, next) { - Picker.middleware(_getRegions(params, req, res, next)); +// helper function for populating the levels in a MATS selector +function getlevelsByApp() { + let flatJSON = ""; + try { + let result; + if ( + matsCollections.level !== undefined && + matsCollections.level.findOne({ name: "level" }) !== undefined + ) { + // we have levels already defined + result = matsCollections.level.findOne({ + name: "level", + }).options; + if (!Array.isArray(result)) result = Object.keys(result); + } else if ( + matsCollections.top !== undefined && + matsCollections.top.findOne({ name: "top" }) !== undefined + ) { + // use the MATS mandatory levels + result = _.range(100, 1050, 50); + if (!Array.isArray(result)) result = Object.keys(result); + } else { + result = []; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getRegions`, - function (params, req, res, next) { - Picker.middleware(_getRegions(params, req, res, next)); + let newResult; + if (result.length === 0) { + newResult = {}; + } else { + newResult = mapArrayToApps(result); } - ); - - Picker.route("/getRegionsValuesMap", function (params, req, res, next) { - Picker.middleware(_getRegionsValuesMap(params, req, res, next)); - }); + flatJSON = JSON.stringify(newResult); + } catch (e) { + console.log("error retrieving levels: ", e); + flatJSON = JSON.stringify({ + error: e, + }); + } + return flatJSON; +} - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getRegionsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getRegionsValuesMap(params, req, res, next)); +// private middleware for getApps route +const getApps = function (res) { + // this function returns an array of apps. + if (Meteor.isServer) { + let flatJSON = ""; + try { + const result = getListOfApps(); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving apps: ", e); + flatJSON = JSON.stringify({ + error: e, + }); } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getRegionsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getRegionsValuesMap(params, req, res, next)); +// private middleware for getAppSumsDBs route +const getAppSumsDBs = function (res) { + // this function returns map of apps and appRefs. + if (Meteor.isServer) { + let flatJSON = ""; + try { + const result = getListOfAppDBs(); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving apps: ", e); + flatJSON = JSON.stringify({ + error: e, + }); } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getStatistics", function (params, req, res, next) { - Picker.middleware(_getStatistics(params, req, res, next)); - }); +// private middleware for getModels route +const getModels = function (res) { + // this function returns a map of models keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("data-source", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getStatistics`, - function (params, req, res, next) { - Picker.middleware(_getStatistics(params, req, res, next)); +// private middleware for getRegions route +const getRegions = function (res) { + // this function returns a map of regions keyed by app title and model display text + if (Meteor.isServer) { + let flatJSON = getMapByAppAndModel("region", "optionsMap"); + if (flatJSON === "{}") { + flatJSON = getMapByAppAndModel("vgtyp", "optionsMap"); } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getStatistics`, - function (params, req, res, next) { - Picker.middleware(_getStatistics(params, req, res, next)); +// private middleware for getRegionsValuesMap route +const getRegionsValuesMap = function (res) { + // this function returns a map of regions values keyed by app title + if (Meteor.isServer) { + let flatJSON = getMapByAppAndModel("region", "valuesMap"); + if (flatJSON === "{}") { + flatJSON = getMapByAppAndModel("vgtyp", "valuesMap"); } - ); - - Picker.route("/getStatisticsValuesMap", function (params, req, res, next) { - Picker.middleware(_getStatisticsValuesMap(params, req, res, next)); - }); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getStatisticsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getStatisticsValuesMap(params, req, res, next)); - } - ); +// private middleware for getStatistics route +const getStatistics = function (res) { + // this function returns an map of statistics keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByApp("statistic"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getStatisticsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getStatisticsValuesMap(params, req, res, next)); - } - ); +// private middleware for getStatisticsValuesMap route +const getStatisticsValuesMap = function (res) { + // this function returns a map of statistic values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("statistic", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getVariables", function (params, req, res, next) { - Picker.middleware(_getVariables(params, req, res, next)); - }); +// private middleware for getVariables route +const getVariables = function (res) { + // this function returns an map of variables keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByApp("variable"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getVariables`, - function (params, req, res, next) { - Picker.middleware(_getVariables(params, req, res, next)); - } - ); +// private middleware for getVariablesValuesMap route +const getVariablesValuesMap = function (res) { + // this function returns a map of variable values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("variable", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getVariables`, - function (params, req, res, next) { - Picker.middleware(_getVariables(params, req, res, next)); - } - ); +// private middleware for getThresholds route +const getThresholds = function (res) { + // this function returns a map of thresholds keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("threshold", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getVariablesValuesMap", function (params, req, res, next) { - Picker.middleware(_getVariablesValuesMap(params, req, res, next)); - }); +// private middleware for getThresholdsValuesMap route +const getThresholdsValuesMap = function (res) { + // this function returns a map of threshold values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("threshold", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getVariablesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getVariablesValuesMap(params, req, res, next)); - } - ); +// private middleware for getScales route +const getScales = function (res) { + // this function returns a map of scales keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("scale", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getVariablesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getVariablesValuesMap(params, req, res, next)); - } - ); +// private middleware for getScalesValuesMap route +const getScalesValuesMap = function (res) { + // this function returns a map of scale values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("scale", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getThresholds", function (params, req, res, next) { - Picker.middleware(_getThresholds(params, req, res, next)); - }); +// private middleware for getTruth route +const getTruths = function (res) { + // this function returns a map of truths keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("truth", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getThresholds`, - function (params, req, res, next) { - Picker.middleware(_getThresholds(params, req, res, next)); - } - ); +// private middleware for getTruthValuesMap route +const getTruthsValuesMap = function (res) { + // this function returns a map of truth values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("truth", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholds`, - function (params, req, res, next) { - Picker.middleware(_getThresholds(params, req, res, next)); - } - ); +// private middleware for getFcstLengths route +const getFcstLengths = function (res) { + // this function returns a map of forecast lengths keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("forecast-length", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getThresholdsValuesMap", function (params, req, res, next) { - Picker.middleware(_getThresholdsValuesMap(params, req, res, next)); - }); +// private middleware for getFcstTypes route +const getFcstTypes = function (res) { + // this function returns a map of forecast types keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("forecast-type", "optionsMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getThresholdsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getThresholdsValuesMap(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholdsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getThresholdsValuesMap(params, req, res, next)); - } - ); - - Picker.route("/getScales", function (params, req, res, next) { - Picker.middleware(_getScales(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getScales`, - function (params, req, res, next) { - Picker.middleware(_getScales(params, req, res, next)); - } - ); +// private middleware for getFcstTypesValuesMap route +const getFcstTypesValuesMap = function (res) { + // this function returns a map of forecast type values keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByAppAndModel("forecast-type", "valuesMap"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getScales`, - function (params, req, res, next) { - Picker.middleware(_getScales(params, req, res, next)); - } - ); +// private middleware for getValidTimes route +const getValidTimes = function (res) { + // this function returns an map of valid times keyed by app title + if (Meteor.isServer) { + const flatJSON = getMapByApp("valid-time"); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getScalesValuesMap", function (params, req, res, next) { - Picker.middleware(_getScalesValuesMap(params, req, res, next)); - }); +// private middleware for getValidTimes route +const getLevels = function (res) { + // this function returns an map of pressure levels keyed by app title + if (Meteor.isServer) { + const flatJSON = getlevelsByApp(); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getScalesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getScalesValuesMap(params, req, res, next)); - } - ); +// private middleware for getDates route +const getDates = function (res) { + // this function returns a map of dates keyed by app title and model display text + if (Meteor.isServer) { + const flatJSON = getDateMapByAppAndModel(); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getScalesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getScalesValuesMap(params, req, res, next)); +// helper function for getCSV +const stringifyCurveData = function (stringify, dataArray, res) { + const thisDataArray = dataArray[0]; + stringify.stringify( + thisDataArray, + { + header: true, + }, + function (err, output) { + if (err) { + console.log("error in getCSV:", err); + res.write(`error,${err.toLocaleString()}`); + res.end(`

getCSV Error! ${err.toLocaleString()}

`); + return; + } + res.write(output); + if (dataArray.length > 1) { + const newDataArray = dataArray.slice(1); + stringifyCurveData(stringify, newDataArray, res); + } else { + res.end(); + } } ); +}; - Picker.route("/getTruths", function (params, req, res, next) { - Picker.middleware(_getTruths(params, req, res, next)); - }); +// private method for getting pagenated data +// a newPageIndex of -1000 means get all the data (used for export) +// a newPageIndex of -2000 means get just the last page +const getPagenatedData = function (rky, p, np) { + if (Meteor.isServer) { + const key = rky; + const myPageIndex = p; + const newPageIndex = np; + let rawReturn; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getTruths`, - function (params, req, res, next) { - Picker.middleware(_getTruths(params, req, res, next)); + try { + const result = matsCache.getResult(key); + rawReturn = result === undefined ? undefined : result.result; // getResult structure is {key:something, result:resultObject} + } catch (e) { + console.log("getPagenatedData: Error - ", e); + return undefined; } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getTruths`, - function (params, req, res, next) { - Picker.middleware(_getTruths(params, req, res, next)); + const ret = + rawReturn === undefined ? undefined : JSON.parse(JSON.stringify(rawReturn)); + let start; + let end; + let direction = 1; + if (newPageIndex === -1000) { + // all the data + start = 0; + end = Number.MAX_VALUE; + } else if (newPageIndex === -2000) { + // just the last page + start = -2000; + direction = -1; + } else if (myPageIndex <= newPageIndex) { + // proceed forward + start = (newPageIndex - 1) * 100; + end = newPageIndex * 100; + } else { + // move back + direction = -1; + start = newPageIndex * 100; + end = (newPageIndex + 1) * 100; } - ); - Picker.route("/getTruthsValuesMap", function (params, req, res, next) { - Picker.middleware(_getTruthsValuesMap(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getTruthsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getTruthsValuesMap(params, req, res, next)); + let dsiStart; + let dsiEnd; + for (let csi = 0; csi < ret.data.length; csi += 1) { + if (ret.data[csi].x && ret.data[csi].x.length > 100) { + dsiStart = start; + dsiEnd = end; + if (dsiStart > ret.data[csi].x.length || dsiStart === -2000) { + // show the last page if we either requested it specifically or are trying to navigate past it + dsiStart = Math.floor(rawReturn.data[csi].x.length / 100) * 100; + dsiEnd = rawReturn.data[csi].x.length; + if (dsiEnd === dsiStart) { + // make sure the last page isn't empty -= 1if rawReturn.data[csi].data.length/100 produces a whole number, + // dsiStart and dsiEnd would be the same. This makes sure that the last full page is indeed the last page, without a phantom empty page afterwards + dsiStart = dsiEnd - 100; + } + } + if (dsiStart < 0) { + // show the first page if we are trying to navigate before it + dsiStart = 0; + dsiEnd = 100; + } + if (dsiEnd < dsiStart) { + // make sure that the end is after the start + dsiEnd = dsiStart + 100; + } + if (dsiEnd > ret.data[csi].x.length) { + // make sure we don't request past the end -= 1 if results are one page, this should convert the + // start and end from 0 and 100 to 0 and whatever the end is. + dsiEnd = ret.data[csi].x.length; + } + ret.data[csi].x = rawReturn.data[csi].x.slice(dsiStart, dsiEnd); + ret.data[csi].y = rawReturn.data[csi].y.slice(dsiStart, dsiEnd); + ret.data[csi].stats = rawReturn.data[csi].stats.slice(dsiStart, dsiEnd); + ret.data[csi].glob_stats = rawReturn.data[csi].glob_stats; + } } - ); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getTruthsValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getTruthsValuesMap(params, req, res, next)); + if (direction === 1) { + ret.dsiRealPageIndex = Math.floor(dsiEnd / 100); + } else { + ret.dsiRealPageIndex = Math.floor(dsiStart / 100); } - ); + ret.dsiTextDirection = direction; + return ret; + } + return null; +}; - Picker.route("/getFcstLengths", function (params, req, res, next) { - Picker.middleware(_getFcstLengths(params, req, res, next)); - }); +// private method for getting pagenated results and flattening them in order to be appropriate for text display. +const getFlattenedResultData = function (rk, p, np) { + if (Meteor.isServer) { + try { + const r = rk; + const thisP = p; + const thisNP = np; + // get the pagenated data + const result = getPagenatedData(r, thisP, thisNP); + // find the type + const { plotTypes } = result.basis.plotParams; + const plotType = _.invert(plotTypes).true; + // extract data + let isCTC = false; + let isModeSingle = false; + let isModePairs = false; + let labelSuffix; + const returnData = {}; + const stats = {}; + let curveData = []; // array of maps + const { data } = result; + const { dsiRealPageIndex } = result; + const { dsiTextDirection } = result; + let firstBestFitIndex = -1; + let bestFitIndexes = {}; + switch (plotType) { + case matsTypes.PlotTypes.timeSeries: + case matsTypes.PlotTypes.profile: + case matsTypes.PlotTypes.dieoff: + case matsTypes.PlotTypes.threshold: + case matsTypes.PlotTypes.validtime: + case matsTypes.PlotTypes.dailyModelCycle: + case matsTypes.PlotTypes.gridscale: + case matsTypes.PlotTypes.yearToYear: + switch (plotType) { + case matsTypes.PlotTypes.timeSeries: + case matsTypes.PlotTypes.dailyModelCycle: + labelSuffix = " time"; + break; + case matsTypes.PlotTypes.profile: + labelSuffix = " level"; + break; + case matsTypes.PlotTypes.dieoff: + labelSuffix = " forecast lead time"; + break; + case matsTypes.PlotTypes.validtime: + labelSuffix = " hour of day"; + break; + case matsTypes.PlotTypes.threshold: + labelSuffix = " threshold"; + break; + case matsTypes.PlotTypes.gridscale: + labelSuffix = " grid scale"; + break; + case matsTypes.PlotTypes.yearToYear: + labelSuffix = " year"; + break; + default: + labelSuffix = "x-value"; + } + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + // for each curve + isCTC = + data[ci] !== undefined && + ((data[ci].stats !== undefined && + data[ci].stats[0] !== undefined && + data[ci].stats[0].hit !== undefined) || + (data[ci].hitTextOutput !== undefined && + data[ci].hitTextOutput.length > 0)); + isModePairs = + data[ci] !== undefined && + data[ci].stats !== undefined && + data[ci].stats[0] !== undefined && + data[ci].stats[0].avgInterest !== undefined; + isModeSingle = + data[ci] !== undefined && + data[ci].stats !== undefined && + data[ci].stats[0] !== undefined && + data[ci].stats[0].nForecast !== undefined; + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + stats.label = data[ci].label; + stats.mean = data[ci].glob_stats.dMean; + stats["standard deviation"] = data[ci].glob_stats.sd; + stats.n = data[ci].glob_stats.nGood; + if ( + plotType === matsTypes.PlotTypes.timeSeries || + plotType === matsTypes.PlotTypes.profile + ) { + stats["standard error"] = data[ci].glob_stats.stdeBetsy; + stats.lag1 = data[ci].glob_stats.lag1; + } + stats.minimum = data[ci].glob_stats.minVal; + stats.maximum = data[ci].glob_stats.maxVal; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].x.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + if (plotType === matsTypes.PlotTypes.profile) { + curveDataElement[data[ci].label + labelSuffix] = data[ci].y[cdi]; + } else { + curveDataElement[data[ci].label + labelSuffix] = data[ci].x[cdi]; + } + if (isCTC) { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.n = data[ci].stats[cdi].n; + curveDataElement.hit = data[ci].stats[cdi].hit; + curveDataElement.fa = data[ci].stats[cdi].fa; + curveDataElement.miss = data[ci].stats[cdi].miss; + curveDataElement.cn = data[ci].stats[cdi].cn; + } else if (isModeSingle) { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.nForecast = data[ci].stats[cdi].nForecast; + curveDataElement.nMatched = data[ci].stats[cdi].nMatched; + curveDataElement.nSimple = data[ci].stats[cdi].nSimple; + curveDataElement.nTotal = data[ci].stats[cdi].nTotal; + } else if (isModePairs) { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.n = data[ci].stats[cdi].n; + curveDataElement.avgInterest = data[ci].stats[cdi].avgInterest; + } else { + curveDataElement.stat = data[ci].stats[cdi].stat; + curveDataElement.mean = data[ci].stats[cdi].mean; + curveDataElement["std dev"] = data[ci].stats[cdi].sd; + if ( + plotType === matsTypes.PlotTypes.timeSeries || + plotType === matsTypes.PlotTypes.profile + ) { + curveDataElement["std error"] = data[ci].stats[cdi].stdeBetsy; + curveDataElement.lag1 = data[ci].stats[cdi].lag1; + } + curveDataElement.n = data[ci].stats[cdi].nGood; + } + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.reliability: + case matsTypes.PlotTypes.roc: + case matsTypes.PlotTypes.performanceDiagram: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if ( + reservedWords.indexOf(data[ci].label) === -1 && + !data[ci].label.includes(matsTypes.ReservedWords.noSkill) + ) { + stats.label = data[ci].label; + if (plotType === matsTypes.PlotTypes.reliability) { + stats["sample climo"] = data[ci].glob_stats.sample_climo; + } else if (plotType === matsTypes.PlotTypes.roc) { + stats.auc = data[ci].glob_stats.auc; + } + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].y.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + if (plotType === matsTypes.PlotTypes.reliability) { + curveDataElement[`${data[ci].label} probability bin`] = + data[ci].stats[cdi].prob_bin; + if (data[ci].stats[cdi].hit_rate) { + curveDataElement["hit rate"] = data[ci].stats[cdi].hit_rate; + } else { + curveDataElement["observed frequency"] = + data[ci].stats[cdi].obs_freq; + } + } else { + curveDataElement[`${data[ci].label} bin value`] = + data[ci].stats[cdi].bin_value; + curveDataElement["probability of detection"] = + data[ci].stats[cdi].pody; + if (plotType === matsTypes.PlotTypes.roc) { + curveDataElement["probability of false detection"] = + data[ci].stats[cdi].pofd; + } else { + curveDataElement["success ratio"] = data[ci].stats[cdi].fa; + } + curveDataElement.n = data[ci].stats[cdi].n; + } + if (data[ci].stats[cdi].obs_y) { + curveDataElement.oy = data[ci].stats[cdi].obs_y; + curveDataElement.on = data[ci].stats[cdi].obs_n; + } else { + curveDataElement.hitcount = data[ci].stats[cdi].hit_count; + curveDataElement.fcstcount = data[ci].stats[cdi].fcst_count; + } + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.gridscaleProb: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].y.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} probability bin`] = + data[ci].stats[cdi].bin_value; + curveDataElement["number of grid points"] = data[ci].stats[cdi].n_grid; + curveDataElement.n = data[ci].stats[cdi].n; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.simpleScatter: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getFcstLengths`, - function (params, req, res, next) { - Picker.middleware(_getFcstLengths(params, req, res, next)); - } - ); + for (let cdi = 0; cdi < data[ci].y.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} bin value`] = + data[ci].stats[cdi].bin_value; + curveDataElement["x-stat"] = data[ci].stats[cdi].xstat; + curveDataElement["y-stat"] = data[ci].stats[cdi].ystat; + curveDataElement.n = data[ci].stats[cdi].n; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.map: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + stats.label = data[0].label; + stats["total number of obs"] = data[0].stats.reduce(function (prev, curr) { + return prev + curr.nTimes; + }, 0); + stats["mean difference"] = matsDataUtils.average(data[0].queryVal); + stats["standard deviation"] = matsDataUtils.stdev(data[0].queryVal); + stats["minimum time"] = data[0].stats.reduce(function (prev, curr) { + return prev < curr.min_time ? prev : curr.min_time; + }); + stats["minimum time"] = moment + .utc(stats["minimum time"] * 1000) + .format("YYYY-MM-DD HH:mm"); + stats["maximum time"] = data[0].stats.reduce(function (prev, curr) { + return prev > curr.max_time ? prev : curr.max_time; + }); + stats["maximum time"] = moment + .utc(stats["maximum time"] * 1000) + .format("YYYY-MM-DD HH:mm"); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstLengths`, - function (params, req, res, next) { - Picker.middleware(_getFcstLengths(params, req, res, next)); - } - ); + returnData.stats[data[0].label] = stats; - Picker.route("/getFcstTypes", function (params, req, res, next) { - Picker.middleware(_getFcstTypes(params, req, res, next)); - }); + isCTC = + data[0] !== undefined && + data[0].stats !== undefined && + data[0].stats[0] !== undefined && + data[0].stats[0].hit !== undefined; + for (let si = 0; si < data[0].siteName.length; si += 1) { + const curveDataElement = {}; + curveDataElement["site name"] = data[0].siteName[si]; + curveDataElement["number of times"] = data[0].stats[si].nTimes; + if (isCTC) { + curveDataElement.stat = data[0].queryVal[si]; + curveDataElement.hit = data[0].stats[si].hit; + curveDataElement.fa = data[0].stats[si].fa; + curveDataElement.miss = data[0].stats[si].miss; + curveDataElement.cn = data[0].stats[si].cn; + } else { + curveDataElement["start date"] = moment + .utc(data[0].stats[si].min_time * 1000) + .format("YYYY-MM-DD HH:mm"); + curveDataElement["end date"] = moment + .utc(data[0].stats[si].max_time * 1000) + .format("YYYY-MM-DD HH:mm"); + curveDataElement.stat = data[0].queryVal[si]; + } + curveData.push(curveDataElement); + } + returnData.data[data[0].label] = curveData; + break; + case matsTypes.PlotTypes.histogram: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; + stats.mean = data[ci].glob_stats.glob_mean; + stats["standard deviation"] = data[ci].glob_stats.glob_sd; + stats.n = data[ci].glob_stats.glob_n; + stats.minimum = data[ci].glob_stats.glob_min; + stats.maximum = data[ci].glob_stats.glob_max; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].x.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} bin range`] = + data[ci].bin_stats[cdi].binLabel; + curveDataElement.n = data[ci].bin_stats[cdi].bin_n; + curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; + curveDataElement["bin lower bound"] = + data[ci].bin_stats[cdi].binLowBound; + curveDataElement["bin upper bound"] = + data[ci].bin_stats[cdi].binUpBound; + curveDataElement["bin mean"] = data[ci].bin_stats[cdi].bin_mean; + curveDataElement["bin std dev"] = data[ci].bin_stats[cdi].bin_sd; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.ensembleHistogram: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + for (let ci = 0; ci < data.length; ci += 1) { + // for each curve + // if the curve label is a reserved word do not process the curve (its a zero or max curve) + const reservedWords = Object.values(matsTypes.ReservedWords); + if (reservedWords.indexOf(data[ci].label) === -1) { + stats.label = data[ci].label; + stats.mean = data[ci].glob_stats.dMean; + stats["standard deviation"] = data[ci].glob_stats.sd; + stats.n = data[ci].glob_stats.nGood; + stats.minimum = data[ci].glob_stats.minVal; + stats.maximum = data[ci].glob_stats.maxVal; + returnData.stats[data[ci].label] = stats; + + for (let cdi = 0; cdi < data[ci].x.length; cdi += 1) { + // for each datapoint + const curveDataElement = {}; + curveDataElement[`${data[ci].label} bin`] = data[ci].x[cdi]; + curveDataElement.n = data[ci].bin_stats[cdi].bin_n; + curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; + curveData.push(curveDataElement); + } + returnData.data[data[ci].label] = curveData; + } + } + break; + case matsTypes.PlotTypes.contour: + case matsTypes.PlotTypes.contourDiff: + returnData.stats = {}; // map of maps + returnData.data = {}; // map of arrays of maps + stats.label = data[0].label; + stats["total number of points"] = data[0].glob_stats.n; + stats["mean stat"] = data[0].glob_stats.mean; + stats["minimum time"] = moment + .utc(data[0].glob_stats.minDate * 1000) + .format("YYYY-MM-DD HH:mm"); + stats["maximum time"] = moment + .utc(data[0].glob_stats.maxDate * 1000) + .format("YYYY-MM-DD HH:mm"); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getFcstTypes`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypes(params, req, res, next)); - } - ); + returnData.stats[data[0].label] = stats; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypes`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypes(params, req, res, next)); + isCTC = + data[0] !== undefined && + data[0].hitTextOutput !== undefined && + data[0].hitTextOutput.length > 0; + for (let si = 0; si < data[0].xTextOutput.length; si += 1) { + const curveDataElement = {}; + curveDataElement.xVal = data[0].xTextOutput[si]; + curveDataElement.yVal = data[0].yTextOutput[si]; + curveDataElement.stat = data[0].zTextOutput[si]; + curveDataElement.N = data[0].nTextOutput[si]; + if (isCTC) { + curveDataElement.hit = data[0].hitTextOutput[si]; + curveDataElement.fa = data[0].faTextOutput[si]; + curveDataElement.miss = data[0].missTextOutput[si]; + curveDataElement.cn = data[0].cnTextOutput[si]; + } else { + curveDataElement["Start Date"] = moment + .utc(data[0].minDateTextOutput[si] * 1000) + .format("YYYY-MM-DD HH:mm"); + curveDataElement["End Date"] = moment + .utc(data[0].maxDateTextOutput[si] * 1000) + .format("YYYY-MM-DD HH:mm"); + } + curveData.push(curveDataElement); + } + returnData.data[data[0].label] = curveData; + break; + case matsTypes.PlotTypes.scatter2d: + firstBestFitIndex = -1; + bestFitIndexes = {}; + for (let ci = 0; ci < data.length; ci += 1) { + if (ci === firstBestFitIndex) { + break; // best fit curves are at the end so do not do further processing + } + curveData = data[ci]; + // look for a best fit curve - only have to look at curves with higher index than this one + for (let cbi = ci + 1; cbi < data.length; cbi += 1) { + if ( + data[cbi].label.indexOf(curveData.label) !== -1 && + data[cbi].label.indexOf("-best fit") !== -1 + ) { + bestFitIndexes[ci] = cbi; + if (firstBestFitIndex === -1) { + firstBestFitIndex = cbi; + } + break; + } + } + const curveTextData = []; + for (let cdi = 0; cdi < curveData.data.length; cdi += 1) { + const element = {}; + [element.xAxis] = curveData.data[cdi]; + [, element.yAxis] = curveData.data[cdi]; + if (bestFitIndexes[ci] === undefined) { + element["best fit"] = "none;"; + } else { + [, element["best fit"]] = data[bestFitIndexes[ci]].data[cdi]; + } + curveTextData.push(element); + } + returnData[curveData.label] = curveTextData; + } + break; + default: + return undefined; + } + returnData.dsiRealPageIndex = dsiRealPageIndex; + returnData.dsiTextDirection = dsiTextDirection; + return returnData; + } catch (error) { + throw new Meteor.Error( + `Error in getFlattenedResultData function: ${error.message}` + ); } - ); + } + return null; +}; - Picker.route("/getFcstTypesValuesMap", function (params, req, res, next) { - Picker.middleware(_getFcstTypesValuesMap(params, req, res, next)); - }); +// private middleware for getCSV route +const getCSV = function (params, res) { + if (Meteor.isServer) { + const stringify = require("csv-stringify"); + let csv = ""; + try { + const result = getFlattenedResultData(params.key, 0, -1000); + const statArray = Object.values(result.stats); + const dataArray = Object.values(result.data); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getFcstTypesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypesValuesMap(params, req, res, next)); + const fileName = `matsplot-${moment.utc().format("YYYYMMDD-HH.mm.ss")}.csv`; + res.setHeader("Content-disposition", `attachment; filename=${fileName}`); + res.setHeader("Content-Type", "attachment.ContentType"); + stringify.stringify( + statArray, + { + header: true, + }, + function (err, output) { + if (err) { + console.log("error in getCSV:", err); + res.write(`error,${err.toLocaleString()}`); + res.end(`

getCSV Error! ${err.toLocaleString()}

`); + return; + } + res.write(output); + stringifyCurveData(stringify, dataArray, res); + } + ); + } catch (e) { + console.log("error retrieving data: ", e); + csv = `error,${e.toLocaleString()}`; + res.setHeader("Content-disposition", "attachment; filename=matsplot.csv"); + res.setHeader("Content-Type", "attachment.ContentType"); + res.end(`

getCSV Error! ${csv}

`); } - ); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypesValuesMap`, - function (params, req, res, next) { - Picker.middleware(_getFcstTypesValuesMap(params, req, res, next)); +// private middleware for getJSON route +const getJSON = function (params, res) { + if (Meteor.isServer) { + let flatJSON = ""; + try { + const result = getPagenatedData(params.key, 0, -1000); + flatJSON = JSON.stringify(result); + } catch (e) { + console.log("error retrieving data: ", e); + flatJSON = JSON.stringify({ + error: e, + }); + delete flatJSON.dsiRealPageIndex; + delete flatJSON.dsiTextDirection; } - ); + res.setHeader("Content-Type", "application/json"); + res.write(flatJSON); + res.end(); + } +}; - Picker.route("/getValidTimes", function (params, req, res, next) { - Picker.middleware(_getValidTimes(params, req, res, next)); - }); +// private define a middleware for refreshing the metadata +const refreshMetadataMWltData = function (res) { + if (Meteor.isServer) { + console.log("Server route asked to refresh metadata"); - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getValidTimes`, - function (params, req, res, next) { - Picker.middleware(_getValidTimes(params, req, res, next)); + try { + console.log("GUI asked to refresh metadata"); + checkMetaDataRefresh(); + } catch (e) { + console.log(e); + res.end( + `` + + `

refreshMetadata Failed!

` + + `

${e.message}

` + + `` + ); } - ); + res.end("

refreshMetadata Done!

"); + } +}; - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getValidTimes`, - function (params, req, res, next) { - Picker.middleware(_getValidTimes(params, req, res, next)); +// private middleware for causing the scorecard to refresh its mongo collection for a given document +const refreshScorecard = function (params, res) { + if (Meteor.isServer) { + const docId = decodeURIComponent(params.docId); + // get userName, name, submitted, processedAt from id + // SC:anonymous -= 1submitted:20230322230435 -= 11block:0:02/19/2023_20_00_-_03/21/2023_20_00 + if (cbScorecardPool === undefined) { + throw new Meteor.Error("getScorecardData: No cbScorecardPool defined"); } - ); - - Picker.route("/getLevels", function (params, req, res, next) { - Picker.middleware(_getLevels(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getLevels`, - function (params, req, res, next) { - Picker.middleware(_getLevels(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getLevels`, - function (params, req, res, next) { - Picker.middleware(_getLevels(params, req, res, next)); - } - ); - - Picker.route("/getDates", function (params, req, res, next) { - Picker.middleware(_getDates(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/getDates`, - function (params, req, res, next) { - Picker.middleware(_getDates(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/getDates`, - function (params, req, res, next) { - Picker.middleware(_getDates(params, req, res, next)); - } - ); - - // create picker routes for refreshMetaData - Picker.route("/refreshMetadata", function (params, req, res, next) { - Picker.middleware(_refreshMetadataMWltData(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/refreshMetadata`, - function (params, req, res, next) { - Picker.middleware(_refreshMetadataMWltData(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/refreshMetadata`, - function (params, req, res, next) { - Picker.middleware(_refreshMetadataMWltData(params, req, res, next)); - } - ); - Picker.route("/refreshScorecard/:docId", function (params, req, res, next) { - Picker.middleware(_refreshScorecard(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/refreshScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_refreshScorecard(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/refreshScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_refreshScorecard(params, req, res, next)); - } - ); - - Picker.route("/setStatusScorecard/:docId", function (params, req, res, next) { - Picker.middleware(_setStatusScorecard(params, req, res, next)); - }); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/setStatusScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_setStatusScorecard(params, req, res, next)); - } - ); - - Picker.route( - `${Meteor.settings.public.proxy_prefix_path}/:app/setStatusScorecard/:docId`, - function (params, req, res, next) { - Picker.middleware(_setStatusScorecard(params, req, res, next)); - } - ); -} - -// private - used to see if the main page needs to update its selectors -const _checkMetaDataRefresh = async function () { - // This routine compares the current last modified time of the tables (MYSQL) or documents (Couchbase) - // used for curveParameter metadata with the last update time to determine if an update is necessary. - // We really only do this for Curveparams - /* - metaDataTableUpdates: - { - name: dataBaseName(MYSQL) or bucketName(couchbase), - (for couchbase tables are documents) - tables: [tableName1, tableName2 ..], - lastRefreshed : timestamp - } - */ - let refresh = false; - const tableUpdates = metaDataTableUpdates.find({}).fetch(); - const dbType = - matsCollections.Settings.findOne() !== undefined - ? matsCollections.Settings.findOne().dbType - : matsTypes.DbTypes.mysql; - for (let tui = 0; tui < tableUpdates.length; tui++) { - const id = tableUpdates[tui]._id; - const poolName = tableUpdates[tui].pool; - const dbName = tableUpdates[tui].name; - const tableNames = tableUpdates[tui].tables; - const { lastRefreshed } = tableUpdates[tui]; - let updatedEpoch = Number.MAX_VALUE; - for (let ti = 0; ti < tableNames.length; ti++) { - const tName = tableNames[ti]; - try { - if (Meteor.isServer) { - switch (dbType) { - case matsTypes.DbTypes.mysql: - var rows = matsDataQueryUtils.simplePoolQueryWrapSynchronous( - global[poolName], - `SELECT UNIX_TIMESTAMP(UPDATE_TIME)` + - ` FROM information_schema.tables` + - ` WHERE TABLE_SCHEMA = '${dbName}'` + - ` AND TABLE_NAME = '${tName}'` - ); - updatedEpoch = rows[0]["UNIX_TIMESTAMP(UPDATE_TIME)"]; - break; - case matsTypes.DbTypes.couchbase: - // the tName for couchbase is supposed to be the document id - var doc = await cbPool.getCB(tName); - updatedEpoch = doc.updated; - break; - default: - throw new Meteor.Error("resetApp: undefined DbType"); - } - } - // console.log("DB says metadata for table " + dbName + "." + tName + " was updated at " + updatedEpoch); - if ( - updatedEpoch === undefined || - updatedEpoch === null || - updatedEpoch === "NULL" || - updatedEpoch === Number.MAX_VALUE - ) { - // if time of last update isn't stored by the database (thanks, Aurora DB), refresh automatically - // console.log("_checkMetaDataRefresh - cannot find last update time for database: " + dbName + " and table: " + tName); - refresh = true; - // console.log("FORCED Refreshing the metadata for table because updatedEpoch is undefined" + dbName + "." + tName + " : updated at " + updatedEpoch); - break; + const statement = `SELECT sc.* + From + vxdata._default.SCORECARD sc + WHERE + sc.id='${docId}';`; + cbScorecardPool + .queryCB(statement) + .then((result) => { + // insert this result into the mongo Scorecard collection - createdAt is used for TTL + // created at gets updated each display even if it already existed. + // TTL is 24 hours + if (typeof result === "string") { + throw new Error(`Error from couchbase query - ${result}`); + } else if (result[0] === undefined) { + throw new Error("Error from couchbase query - document not found"); + } else { + matsCollections.Scorecard.upsert( + { + "scorecard.userName": result[0].userName, + "scorecard.name": result[0].name, + "scorecard.submitted": result[0].submitted, + "scorecard.processedAt": result[0].processedAt, + }, + { + $set: { + createdAt: new Date(), + scorecard: result[0], + }, + } + ); } - } catch (e) { - throw new Error( - `_checkMetaDataRefresh - error finding last update time for database: ${dbName} and table: ${tName}, ERROR:${e.message}` + res.end("

refreshScorecard Done!

"); + }) + .catch((err) => { + res.end( + `` + + `

refreshScorecard Failed!

` + + `

${err.message}

` + + `` ); - } - const lastRefreshedEpoch = moment.utc(lastRefreshed).valueOf() / 1000; - const updatedEpochMoment = moment.utc(updatedEpoch).valueOf(); - // console.log("Epoch of when this app last refreshed metadata for table " + dbName + "." + tName + " is " + lastRefreshedEpoch); - // console.log("Epoch of when the DB says table " + dbName + "." + tName + " was last updated is " + updatedEpochMoment); - if (lastRefreshedEpoch < updatedEpochMoment || updatedEpochMoment === 0) { - // Aurora DB sometimes returns a 0 for last updated. In that case, do refresh the metadata. - refresh = true; - // console.log("Refreshing the metadata in the app selectors because table " + dbName + "." + tName + " was updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " while the metadata was last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); - break; - } else { - // console.log("NOT Refreshing the metadata for table " + dbName + "." + tName + " : updated at " + moment.utc(updatedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss") + " : metadata last refreshed at " + moment.utc(lastRefreshedEpoch * 1000).format("YYYY-MM-DD HH:mm:ss")); - } - } - if (refresh === true) { - // refresh the app metadata - // app specific routines - for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { - await global.appSpecificResetRoutines[ai](); - } - // remember that we updated ALL the metadata tables just now - metaDataTableUpdates.update( - { - _id: id, - }, - { - $set: { - lastRefreshed: moment().format(), - }, - } - ); - } + }); } - return true; }; -// private middleware for getting the status - think health check -const _status = function (params, req, res, next) { +const setStatusScorecard = function (params, req, res) { if (Meteor.isServer) { - const settings = matsCollections.Settings.findOne(); - res.end( - `
Running: version - ${settings.appVersion}
` + const docId = decodeURIComponent(params.docId); + let body = ""; + req.on( + "data", + Meteor.bindEnvironment(function (data) { + body += data; + }) + ); + + req.on( + "end", + Meteor.bindEnvironment(function () { + // console.log(body); + try { + const doc = JSON.parse(body); + const docStatus = doc.status; + const found = matsCollections.Scorecard.find({ id: docId }).fetch(); + if (found.length === 0) { + throw new Error("Error from scorecard lookup - document not found"); + } + matsCollections.Scorecard.upsert( + { + id: docId, + }, + { + $set: { + docStatus, + }, + } + ); + // set error if there is one somehow. (use the session?) + res.end("

setScorecardStatus Done!

"); + } catch (err) { + res.statusCode = 400; + res.end( + `` + + `

setScorecardStatus Failed!

` + + `

${err.message}

` + + `` + ); + } + }) ); } }; -// private middleware for clearing the cache -const _clearCache = function (params, req, res, next) { +// private save the result from the query into mongo and downsample if that result's size is greater than 1.2Mb +const saveResultData = function (result) { if (Meteor.isServer) { - matsCache.clear(); - res.end("

clearCache Done!

"); + const storedResult = result; + const sizeof = require("object-sizeof"); + const hash = require("object-hash"); + const key = hash(storedResult.basis.plotParams); + const threshold = 1200000; + let ret = {}; + try { + const dSize = sizeof(storedResult.data); + // console.log("storedResult.basis.data size is ", dSize); + // TimeSeries and DailyModelCycle are the only plot types that require downSampling + if ( + dSize > threshold && + (storedResult.basis.plotParams.plotTypes.TimeSeries || + storedResult.basis.plotParams.plotTypes.DailyModelCycle) + ) { + // greater than threshold need to downsample + // downsample and save it in DownSampleResult + console.log("DownSampling"); + const downsampler = require("downsample-lttb"); + let totalPoints = 0; + for (let di = 0; di < storedResult.data.length; di += 1) { + totalPoints += storedResult.data[di].x_epoch.length; + } + const allowedNumberOfPoints = (threshold / dSize) * totalPoints; + const downSampleResult = + storedResult === undefined + ? undefined + : JSON.parse(JSON.stringify(storedResult)); + for (let ci = 0; ci < storedResult.data.length; ci += 1) { + const dsData = {}; + const xyDataset = storedResult.data[ci].x_epoch.map(function (d, index) { + return [ + storedResult.data[ci].x_epoch[index], + storedResult.data[ci].y[index], + ]; + }); + const ratioTotalPoints = xyDataset.length / totalPoints; + const myAllowedPoints = Math.round(ratioTotalPoints * allowedNumberOfPoints); + // downsample the array + let downsampledSeries; + if (myAllowedPoints < xyDataset.length && xyDataset.length > 2) { + downsampledSeries = downsampler.processData(xyDataset, myAllowedPoints); + // replace the y attributes (tooltips etc.) with the y attributes from the nearest x + let originalIndex = 0; + // skip through the original dataset capturing each downSampled data point + const arrayKeys = []; + const nonArrayKeys = []; + const keys = Object.keys(storedResult.data[ci]); + for (let ki = 0; ki < keys.length; ki += 1) { + if (keys[ki] !== "x_epoch") { + if (Array.isArray(storedResult.data[ci][keys[ki]])) { + arrayKeys.push(keys[ki]); + dsData[keys[ki]] = []; + } else { + nonArrayKeys.push(keys[ki]); + } + } + } + // We only ever downsample series plots - never profiles and series plots only ever have error_y arrays. + // This is a little hacky but what is happening is we putting error_y.array on the arrayKeys list so that it gets its + // downsampled equivalent values. + for (let ki = 0; ki < nonArrayKeys.length; ki += 1) { + dsData[nonArrayKeys[ki]] = JSON.parse( + JSON.stringify(storedResult.data[ci][nonArrayKeys[ki]]) + ); + } + // remove the original error_y array data. + dsData.error_y.array = []; + for (let dsi = 0; dsi < downsampledSeries.length; dsi += 1) { + while ( + originalIndex < storedResult.data[ci].x_epoch.length && + storedResult.data[ci].x_epoch[originalIndex] < downsampledSeries[dsi][0] + ) { + originalIndex += 1; + } + // capture the stuff related to this downSampled data point (downSampled data points are always a subset of original data points) + for (let ki = 0; ki < arrayKeys.length; ki += 1) { + dsData[arrayKeys[ki]][dsi] = + storedResult.data[ci][arrayKeys[ki]][originalIndex]; + } + dsData.error_y.array[dsi] = + storedResult.data[ci].error_y.array[originalIndex]; + } + // add downsampled annotation to curve options + downSampleResult[ci] = dsData; + downSampleResult[ci].annotation += " **DOWNSAMPLED**"; + } else { + downSampleResult[ci] = storedResult.data[ci]; + } + downSampleResult.data[ci] = downSampleResult[ci]; + } + DownSampleResults.rawCollection().insert({ + createdAt: new Date(), + key, + result: downSampleResult, + }); // createdAt ensures expiration set in mats-collections + ret = { + key, + result: downSampleResult, + }; + } else { + ret = { + key, + result: storedResult, + }; + } + // save original dataset in the matsCache + if ( + storedResult.basis.plotParams.plotTypes.TimeSeries || + storedResult.basis.plotParams.plotTypes.DailyModelCycle + ) { + for (let ci = 0; ci < storedResult.data.length; ci += 1) { + delete storedResult.data[ci].x_epoch; // we only needed this as an index for downsampling + } + } + matsCache.storeResult(key, { + key, + result: storedResult, + }); // lifespan is handled by lowDb (internally) in matscache + } catch (error) { + if (error.toLocaleString().indexOf("larger than the maximum size") !== -1) { + throw new Meteor.Error(": Requesting too much data... try averaging"); + } + } + return ret; } + return null; }; -// private middleware for dropping a distinct instance (a single run) of a scorecard -const _dropScorecardInstance = async function ( - userName, - name, - submittedTime, - processedAt -) { +// Utility method for writing out the meteor.settings file +const writeSettings = function (settings, appName) { + const fs = require("fs"); + let settingsPath = process.env.METEOR_SETTINGS_DIR; + if (!settingsPath) { + console.log( + "environment var METEOR_SETTINGS_DIR is undefined: setting it to /usr/app/settings" + ); + settingsPath = "/usr/app/settings"; + } + if (!fs.existsSync(settingsPath)) { + fs.mkdirSync(settingsPath, { + recursive: true, + }); + } + let appSettings = {}; + let newSettings = {}; + try { + const appSettingsData = fs.readFileSync(`${settingsPath}/${appName}/settings.json`); + appSettings = JSON.parse(appSettingsData); + } catch (e) { + appSettings = { + private: {}, + public: {}, + }; + } + newSettings = settings; + // Merge settings into appSettings + newSettings.private = { + ...appSettings.private, + ...settings.private, + }; + newSettings.public = { + ...appSettings.public, + ...settings.public, + }; + // write the settings file + const jsonSettings = JSON.stringify(newSettings, null, 2); + // console.log (jsonSettings); + fs.writeFileSync(`${settingsPath}/${appName}/settings.json`, jsonSettings, { + encoding: "utf8", + flag: "w", + }); +}; +// return the scorecard for the provided selectors +const getThisScorecardData = async function (userName, name, submitted, processedAt) { try { if (cbScorecardPool === undefined) { - throw new Meteor.Error("_dropScorecardInstance: No cbScorecardPool defined"); + throw new Meteor.Error("getThisScorecardData: No cbScorecardPool defined"); } - const statement = `DELETE + const statement = `SELECT sc.* From vxdata._default.SCORECARD sc WHERE @@ -745,3095 +1715,2236 @@ const _dropScorecardInstance = async function ( AND sc.userName='${userName}' AND sc.name='${name}' AND sc.processedAt=${processedAt} - AND sc.submitted=${submittedTime};`; - const result = await cbScorecardPool.queryCB(statement); - // delete this result from the mongo Scorecard collection + AND sc.submitted=${submitted};`; + const result = await cbScorecardPool.queryCBWithConsistency(statement); + if (typeof result === "string" && result.indexOf("ERROR")) { + throw new Meteor.Error(result); + } + // insert this result into the mongo Scorecard collection - createdAt is used for TTL + // created at gets updated each display even if it already existed. + // TTL is 24 hours + matsCollections.Scorecard.upsert( + { + "scorecard.userName": result[0].userName, + "scorecard.name": result[0].name, + "scorecard.submitted": result[0].submitted, + "scorecard.processedAt": result[0].processedAt, + }, + { + $set: { + createdAt: new Date(), + scorecard: result[0], + }, + } + ); + const docID = matsCollections.Scorecard.findOne( + { + "scorecard.userName": result[0].userName, + "scorecard.name": result[0].name, + "scorecard.submitted": result[0].submitted, + "scorecard.processedAt": result[0].processedAt, + }, + { _id: 1 } + )._id; + // no need to return the whole thing, just the identifying fields + // and the ID. The app will find the whole thing in the mongo collection. + return { scorecard: result[0], docID }; } catch (err) { - console.log(`_dropScorecardInstance error : ${err.message}`); + console.log(`getThisScorecardData error : ${err.message}`); return { error: err.message, }; } }; -// helper function to map a results array to specific apps -function _mapArrayToApps(result) { - // put results in a map keyed by app - const newResult = {}; - const apps = _getListOfApps(); - for (let aidx = 0; aidx < apps.length; aidx++) { - if (result[aidx] === apps[aidx]) { - newResult[apps[aidx]] = [result[aidx]]; - } else { - newResult[apps[aidx]] = result; +// return the scorecard status information from the couchbase database +const getThisScorecardInfo = async function () { + try { + if (cbScorecardPool === undefined) { + throw new Meteor.Error("getThisScorecardInfo: No cbScorecardPool defined"); } - } - return newResult; -} -// helper function to map a results map to specific apps -function _mapMapToApps(result) { - // put results in a map keyed by app - let newResult = {}; - const apps = _getListOfApps(); - const resultKeys = Object.keys(result); - if (!matsDataUtils.arraysEqual(apps.sort(), resultKeys.sort())) { - if (resultKeys.includes("Predefined region")) result = result["Predefined region"]; - for (let aidx = 0; aidx < apps.length; aidx++) { - newResult[apps[aidx]] = result; - } - } else { - newResult = result; + const statement = `SELECT + sc.id, + sc.userName, + sc.name, + sc.status, + sc.processedAt as processedAt, + sc.submitted, + sc.dateRange + From + vxdata._default.SCORECARD sc + WHERE + sc.type='SC';`; + const result = await cbScorecardPool.queryCBWithConsistency(statement); + const scMap = {}; + result.forEach(function (elem) { + if (!Object.keys(scMap).includes(elem.userName)) { + scMap[elem.userName] = {}; + } + const userElem = scMap[elem.userName]; + if (!Object.keys(userElem).includes(elem.name)) { + userElem[elem.name] = {}; + } + const nameElem = userElem[elem.name]; + if (!Object.keys(nameElem).includes(elem.submited)) { + nameElem[elem.submitted] = {}; + } + const submittedElem = nameElem[elem.submitted]; + submittedElem[elem.processedAt] = { + id: elem.id, + status: elem.status, + submitted: elem.submitted, + }; + }); + return scMap; + } catch (err) { + console.log(`getThisScorecardInfo error : ${err.message}`); + return { + error: err.message, + }; } - return newResult; -} +}; -// helper function for returning an array of database-distinct apps contained within a larger MATS app -function _getListOfApps() { - let apps; - if ( - matsCollections.database !== undefined && - matsCollections.database.findOne({ name: "database" }) !== undefined - ) { - // get list of databases (one per app) - apps = matsCollections.database.findOne({ - name: "database", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - } else if ( - matsCollections.variable !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }) !== undefined && - matsCollections.threshold !== undefined && - matsCollections.threshold.findOne({ - name: "threshold", - }) !== undefined - ) { - // get list of apps (variables in apps that also have thresholds) - apps = matsCollections.variable.findOne({ - name: "variable", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - } else { - apps = [matsCollections.Settings.findOne().Title]; - } - return apps; -} - -// helper function for returning a map of database-distinct apps contained within a larger MATS app and their DBs -function _getListOfAppDBs() { - let apps; - const result = {}; - let aidx; - if ( - matsCollections.database !== undefined && - matsCollections.database.findOne({ name: "database" }) !== undefined - ) { - // get list of databases (one per app) - apps = matsCollections.database.findOne({ - name: "database", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - for (aidx = 0; aidx < apps.length; aidx++) { - result[apps[aidx]] = matsCollections.database.findOne({ - name: "database", - }).optionsMap[apps[aidx]].sumsDB; - } - } else if ( - matsCollections.variable !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }) !== undefined && - matsCollections.threshold !== undefined && - matsCollections.threshold.findOne({ - name: "threshold", - }) !== undefined - ) { - // get list of apps (variables in apps that also have thresholds) - apps = matsCollections.variable.findOne({ - name: "variable", - }).options; - if (!Array.isArray(apps)) apps = Object.keys(apps); - for (aidx = 0; aidx < apps.length; aidx++) { - result[apps[aidx]] = matsCollections.variable.findOne({ - name: "variable", - }).optionsMap[apps[aidx]]; - if ( - typeof result[apps[aidx]] !== "string" && - !(result[apps[aidx]] instanceof String) - ) - result[apps[aidx]] = result[apps[aidx]].sumsDB; - } - } else { - result[matsCollections.Settings.findOne().Title] = - matsCollections.Databases.findOne({ - role: matsTypes.DatabaseRoles.SUMS_DATA, - status: "active", - }).database; - } - return result; -} - -// helper function for getting a metadata map from a MATS selector, keyed by app title and model display text -function _getMapByAppAndModel(selector, mapType) { - let flatJSON = ""; - try { - let result; - if ( - matsCollections[selector] !== undefined && - matsCollections[selector].findOne({ name: selector }) !== undefined && - matsCollections[selector].findOne({ name: selector })[mapType] !== undefined - ) { - // get map of requested selector's metadata - result = matsCollections[selector].findOne({ - name: selector, - })[mapType]; - let newResult = {}; - if ( - mapType === "valuesMap" || - selector === "variable" || - selector === "statistic" - ) { - // valueMaps always need to be re-keyed by app (statistic and variable get their valuesMaps from optionsMaps) - newResult = _mapMapToApps(result); - result = newResult; - } else if ( - matsCollections.database === undefined && - !( - matsCollections.variable !== undefined && - matsCollections.threshold !== undefined - ) - ) { - // key by app title if we're not already - const appTitle = matsCollections.Settings.findOne().Title; - newResult[appTitle] = result; - result = newResult; - } - } else { - result = {}; - } - flatJSON = JSON.stringify(result); - } catch (e) { - console.log(`error retrieving metadata from ${selector}: `, e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// helper function for getting a date metadata map from a MATS selector, keyed by app title and model display text -function _getDateMapByAppAndModel() { - let flatJSON = ""; - try { - let result; - // the date map can be in a few places. we have to hunt for it. - if ( - matsCollections.database !== undefined && - matsCollections.database.findOne({ - name: "database", - }) !== undefined && - matsCollections.database.findOne({ - name: "database", - }).dates !== undefined - ) { - result = matsCollections.database.findOne({ - name: "database", - }).dates; - } else if ( - matsCollections.variable !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }) !== undefined && - matsCollections.variable.findOne({ - name: "variable", - }).dates !== undefined - ) { - result = matsCollections.variable.findOne({ - name: "variable", - }).dates; - } else if ( - matsCollections["data-source"] !== undefined && - matsCollections["data-source"].findOne({ - name: "data-source", - }) !== undefined && - matsCollections["data-source"].findOne({ - name: "data-source", - }).dates !== undefined - ) { - result = matsCollections["data-source"].findOne({ - name: "data-source", - }).dates; - } else { - result = {}; - } - if ( - matsCollections.database === undefined && - !( - matsCollections.variable !== undefined && - matsCollections.threshold !== undefined - ) - ) { - // key by app title if we're not already - const appTitle = matsCollections.Settings.findOne().Title; - const newResult = {}; - newResult[appTitle] = result; - result = newResult; - } - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving datemap", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// helper function for getting a metadata map from a MATS selector, keyed by app title -function _getMapByApp(selector) { - let flatJSON = ""; - try { - let result; - if ( - matsCollections[selector] !== undefined && - matsCollections[selector].findOne({ name: selector }) !== undefined - ) { - // get array of requested selector's metadata - result = matsCollections[selector].findOne({ - name: selector, - }).options; - if (!Array.isArray(result)) result = Object.keys(result); - } else if (selector === "statistic") { - result = ["ACC"]; - } else if (selector === "variable") { - result = [matsCollections.Settings.findOne().Title]; - } else { - result = []; - } - // put results in a map keyed by app - let newResult; - if (result.length === 0) { - newResult = {}; - } else { - newResult = _mapArrayToApps(result); - } - flatJSON = JSON.stringify(newResult); - } catch (e) { - console.log(`error retrieving metadata from ${selector}: `, e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// helper function for populating the levels in a MATS selector -function _getlevelsByApp() { - let flatJSON = ""; - try { - let result; - if ( - matsCollections.level !== undefined && - matsCollections.level.findOne({ name: "level" }) !== undefined - ) { - // we have levels already defined - result = matsCollections.level.findOne({ - name: "level", - }).options; - if (!Array.isArray(result)) result = Object.keys(result); - } else if ( - matsCollections.top !== undefined && - matsCollections.top.findOne({ name: "top" }) !== undefined - ) { - // use the MATS mandatory levels - result = _.range(100, 1050, 50); - if (!Array.isArray(result)) result = Object.keys(result); - } else { - result = []; - } - let newResult; - if (result.length === 0) { - newResult = {}; - } else { - newResult = _mapArrayToApps(result); - } - flatJSON = JSON.stringify(newResult); - } catch (e) { - console.log("error retrieving levels: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - return flatJSON; -} - -// private middleware for _getApps route -const _getApps = function (params, req, res, next) { - // this function returns an array of apps. - if (Meteor.isServer) { - let flatJSON = ""; - try { - const result = _getListOfApps(); - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving apps: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getAppSumsDBs route -const _getAppSumsDBs = function (params, req, res, next) { - // this function returns map of apps and appRefs. - if (Meteor.isServer) { - let flatJSON = ""; - try { - const result = _getListOfAppDBs(); - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving apps: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getModels route -const _getModels = function (params, req, res, next) { - // this function returns a map of models keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("data-source", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getRegions route -const _getRegions = function (params, req, res, next) { - // this function returns a map of regions keyed by app title and model display text - if (Meteor.isServer) { - let flatJSON = _getMapByAppAndModel("region", "optionsMap"); - if (flatJSON === "{}") { - flatJSON = _getMapByAppAndModel("vgtyp", "optionsMap"); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getRegionsValuesMap route -const _getRegionsValuesMap = function (params, req, res, next) { - // this function returns a map of regions values keyed by app title - if (Meteor.isServer) { - let flatJSON = _getMapByAppAndModel("region", "valuesMap"); - if (flatJSON === "{}") { - flatJSON = _getMapByAppAndModel("vgtyp", "valuesMap"); - } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getStatistics route -const _getStatistics = function (params, req, res, next) { - // this function returns an map of statistics keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByApp("statistic"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getStatisticsValuesMap route -const _getStatisticsValuesMap = function (params, req, res, next) { - // this function returns a map of statistic values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("statistic", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getVariables route -const _getVariables = function (params, req, res, next) { - // this function returns an map of variables keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByApp("variable"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getVariablesValuesMap route -const _getVariablesValuesMap = function (params, req, res, next) { - // this function returns a map of variable values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("variable", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getThresholds route -const _getThresholds = function (params, req, res, next) { - // this function returns a map of thresholds keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("threshold", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getThresholdsValuesMap route -const _getThresholdsValuesMap = function (params, req, res, next) { - // this function returns a map of threshold values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("threshold", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getScales route -const _getScales = function (params, req, res, next) { - // this function returns a map of scales keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("scale", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getScalesValuesMap route -const _getScalesValuesMap = function (params, req, res, next) { - // this function returns a map of scale values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("scale", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getTruth route -const _getTruths = function (params, req, res, next) { - // this function returns a map of truths keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("truth", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getTruthValuesMap route -const _getTruthsValuesMap = function (params, req, res, next) { - // this function returns a map of truth values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("truth", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getFcstLengths route -const _getFcstLengths = function (params, req, res, next) { - // this function returns a map of forecast lengths keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("forecast-length", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getFcstTypes route -const _getFcstTypes = function (params, req, res, next) { - // this function returns a map of forecast types keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("forecast-type", "optionsMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getFcstTypesValuesMap route -const _getFcstTypesValuesMap = function (params, req, res, next) { - // this function returns a map of forecast type values keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByAppAndModel("forecast-type", "valuesMap"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getValidTimes route -const _getValidTimes = function (params, req, res, next) { - // this function returns an map of valid times keyed by app title - if (Meteor.isServer) { - const flatJSON = _getMapByApp("valid-time"); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getValidTimes route -const _getLevels = function (params, req, res, next) { - // this function returns an map of pressure levels keyed by app title - if (Meteor.isServer) { - const flatJSON = _getlevelsByApp(); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// private middleware for _getDates route -const _getDates = function (params, req, res, next) { - // this function returns a map of dates keyed by app title and model display text - if (Meteor.isServer) { - const flatJSON = _getDateMapByAppAndModel(); - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); - } -}; - -// helper function for _getCSV -const stringifyCurveData = function (stringify, dataArray, res) { - const thisDataArray = dataArray[0]; - stringify.stringify( - thisDataArray, - { - header: true, - }, - function (err, output) { - if (err) { - console.log("error in _getCSV:", err); - res.write(`error,${err.toLocaleString()}`); - res.end(`

_getCSV Error! ${err.toLocaleString()}

`); - return; - } - res.write(output); - if (dataArray.length > 1) { - const newDataArray = dataArray.slice(1); - stringifyCurveData(stringify, newDataArray, res); - } else { - res.end(); - } - } - ); -}; - -// private middleware for _getCSV route -const _getCSV = function (params, req, res, next) { - if (Meteor.isServer) { - const stringify = require("csv-stringify"); - let csv = ""; - try { - const result = _getFlattenedResultData(params.key, 0, -1000); - const statArray = Object.values(result.stats); - const dataArray = Object.values(result.data); - - const fileName = `matsplot-${moment.utc().format("YYYYMMDD-HH.mm.ss")}.csv`; - res.setHeader("Content-disposition", `attachment; filename=${fileName}`); - res.setHeader("Content-Type", "attachment.ContentType"); - stringify.stringify( - statArray, - { - header: true, - }, - function (err, output) { - if (err) { - console.log("error in _getCSV:", err); - res.write(`error,${err.toLocaleString()}`); - res.end(`

_getCSV Error! ${err.toLocaleString()}

`); - return; - } - res.write(output); - stringifyCurveData(stringify, dataArray, res); - } +const getThesePlotParamsFromScorecardInstance = async function ( + userName, + name, + submitted, + processedAt +) { + try { + if (cbScorecardPool === undefined) { + throw new Meteor.Error( + "getThesePlotParamsFromScorecardInstance: No cbScorecardPool defined" ); - } catch (e) { - console.log("error retrieving data: ", e); - csv = `error,${e.toLocaleString()}`; - res.setHeader("Content-disposition", "attachment; filename=matsplot.csv"); - res.setHeader("Content-Type", "attachment.ContentType"); - res.end(`

_getCSV Error! ${csv}

`); } - } -}; - -// private middleware for _getJSON route -const _getJSON = function (params, req, res, next) { - if (Meteor.isServer) { - let flatJSON = ""; - try { - const result = _getPagenatedData(params.key, 0, -1000); - flatJSON = JSON.stringify(result); - } catch (e) { - console.log("error retrieving data: ", e); - flatJSON = JSON.stringify({ - error: e, - }); - delete flatJSON.dsiRealPageIndex; - delete flatJSON.dsiTextDirection; + const statement = `SELECT sc.plotParams + From + vxdata._default.SCORECARD sc + WHERE + sc.type='SC' + AND sc.userName='${userName}' + AND sc.name='${name}' + AND sc.processedAt=${processedAt} + AND sc.submitted=${submitted};`; + const result = await cbScorecardPool.queryCBWithConsistency(statement); + if (typeof result === "string" && result.indexOf("ERROR")) { + throw new Meteor.Error(result); } - res.setHeader("Content-Type", "application/json"); - res.write(flatJSON); - res.end(); + return result[0]; + } catch (err) { + console.log(`getThesePlotParamsFromScorecardInstance error : ${err.message}`); + return { + error: err.message, + }; } }; -// private method for getting pagenated results and flattening them in order to be appropriate for text display. -const _getFlattenedResultData = function (rk, p, np) { - if (Meteor.isServer) { - let resp; - try { - const r = rk; - var p = p; - var np = np; - // get the pagenated data - const result = _getPagenatedData(r, p, np); - // find the type - const { plotTypes } = result.basis.plotParams; - const plotType = _.invert(plotTypes).true; - // extract data - let isCTC = false; - let isModeSingle = false; - let isModePairs = false; - const { data } = result; - const { dsiRealPageIndex } = result; - const { dsiTextDirection } = result; - switch (plotType) { - case matsTypes.PlotTypes.timeSeries: - case matsTypes.PlotTypes.profile: - case matsTypes.PlotTypes.dieoff: - case matsTypes.PlotTypes.threshold: - case matsTypes.PlotTypes.validtime: - case matsTypes.PlotTypes.dailyModelCycle: - case matsTypes.PlotTypes.gridscale: - case matsTypes.PlotTypes.yearToYear: - var labelSuffix; - switch (plotType) { - case matsTypes.PlotTypes.timeSeries: - case matsTypes.PlotTypes.dailyModelCycle: - labelSuffix = " time"; - break; - case matsTypes.PlotTypes.profile: - labelSuffix = " level"; - break; - case matsTypes.PlotTypes.dieoff: - labelSuffix = " forecast lead time"; - break; - case matsTypes.PlotTypes.validtime: - labelSuffix = " hour of day"; - break; - case matsTypes.PlotTypes.threshold: - labelSuffix = " threshold"; - break; - case matsTypes.PlotTypes.gridscale: - labelSuffix = " grid scale"; - break; - case matsTypes.PlotTypes.yearToYear: - labelSuffix = " year"; - break; - } - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - isCTC = - data[ci] !== undefined && - ((data[ci].stats !== undefined && - data[ci].stats[0] !== undefined && - data[ci].stats[0].hit !== undefined) || - (data[ci].hitTextOutput !== undefined && - data[ci].hitTextOutput.length > 0)); - isModePairs = - data[ci] !== undefined && - data[ci].stats !== undefined && - data[ci].stats[0] !== undefined && - data[ci].stats[0].avgInterest !== undefined; - isModeSingle = - data[ci] !== undefined && - data[ci].stats !== undefined && - data[ci].stats[0] !== undefined && - data[ci].stats[0].nForecast !== undefined; - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.dMean; - stats["standard deviation"] = data[ci].glob_stats.sd; - stats.n = data[ci].glob_stats.nGood; - if ( - plotType === matsTypes.PlotTypes.timeSeries || - plotType === matsTypes.PlotTypes.profile - ) { - stats["standard error"] = data[ci].glob_stats.stdeBetsy; - stats.lag1 = data[ci].glob_stats.lag1; - } - stats.minimum = data[ci].glob_stats.minVal; - stats.maximum = data[ci].glob_stats.maxVal; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].x.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - if (plotType === matsTypes.PlotTypes.profile) { - curveDataElement[data[ci].label + labelSuffix] = data[ci].y[cdi]; - } else { - curveDataElement[data[ci].label + labelSuffix] = data[ci].x[cdi]; - } - if (isCTC) { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.n = data[ci].stats[cdi].n; - curveDataElement.hit = data[ci].stats[cdi].hit; - curveDataElement.fa = data[ci].stats[cdi].fa; - curveDataElement.miss = data[ci].stats[cdi].miss; - curveDataElement.cn = data[ci].stats[cdi].cn; - } else if (isModeSingle) { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.nForecast = data[ci].stats[cdi].nForecast; - curveDataElement.nMatched = data[ci].stats[cdi].nMatched; - curveDataElement.nSimple = data[ci].stats[cdi].nSimple; - curveDataElement.nTotal = data[ci].stats[cdi].nTotal; - } else if (isModePairs) { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.n = data[ci].stats[cdi].n; - curveDataElement.avgInterest = data[ci].stats[cdi].avgInterest; - } else { - curveDataElement.stat = data[ci].stats[cdi].stat; - curveDataElement.mean = data[ci].stats[cdi].mean; - curveDataElement["std dev"] = data[ci].stats[cdi].sd; - if ( - plotType === matsTypes.PlotTypes.timeSeries || - plotType === matsTypes.PlotTypes.profile - ) { - curveDataElement["std error"] = data[ci].stats[cdi].stdeBetsy; - curveDataElement.lag1 = data[ci].stats[cdi].lag1; - } - curveDataElement.n = data[ci].stats[cdi].nGood; - } - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.reliability: - case matsTypes.PlotTypes.roc: - case matsTypes.PlotTypes.performanceDiagram: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if ( - reservedWords.indexOf(data[ci].label) >= 0 || - data[ci].label.includes(matsTypes.ReservedWords.noSkill) - ) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - if (plotType === matsTypes.PlotTypes.reliability) { - stats["sample climo"] = data[ci].glob_stats.sample_climo; - } else if (plotType === matsTypes.PlotTypes.roc) { - stats.auc = data[ci].glob_stats.auc; - } - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].y.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - if (plotType === matsTypes.PlotTypes.reliability) { - curveDataElement[`${data[ci].label} probability bin`] = - data[ci].stats[cdi].prob_bin; - if (data[ci].stats[cdi].hit_rate) { - curveDataElement["hit rate"] = data[ci].stats[cdi].hit_rate; - } else { - curveDataElement["observed frequency"] = data[ci].stats[cdi].obs_freq; - } - } else { - curveDataElement[`${data[ci].label} bin value`] = - data[ci].stats[cdi].bin_value; - curveDataElement["probability of detection"] = data[ci].stats[cdi].pody; - if (plotType === matsTypes.PlotTypes.roc) { - curveDataElement["probability of false detection"] = - data[ci].stats[cdi].pofd; - } else { - curveDataElement["success ratio"] = data[ci].stats[cdi].fa; - } - curveDataElement.n = data[ci].stats[cdi].n; - } - if (data[ci].stats[cdi].obs_y) { - curveDataElement.oy = data[ci].stats[cdi].obs_y; - curveDataElement.on = data[ci].stats[cdi].obs_n; - } else { - curveDataElement.hitcount = data[ci].stats[cdi].hit_count; - curveDataElement.fcstcount = data[ci].stats[cdi].fcst_count; - } - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.gridscaleProb: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].y.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} probability bin`] = - data[ci].stats[cdi].bin_value; - curveDataElement["number of grid points"] = data[ci].stats[cdi].n_grid; - curveDataElement.n = data[ci].stats[cdi].n; - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.simpleScatter: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].y.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} bin value`] = - data[ci].stats[cdi].bin_value; - curveDataElement["x-stat"] = data[ci].stats[cdi].xstat; - curveDataElement["y-stat"] = data[ci].stats[cdi].ystat; - curveDataElement.n = data[ci].stats[cdi].n; - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; - } - break; - case matsTypes.PlotTypes.map: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - - var stats = {}; - stats.label = data[0].label; - stats["total number of obs"] = data[0].stats.reduce(function (prev, curr) { - return prev + curr.nTimes; - }, 0); - stats["mean difference"] = matsDataUtils.average(data[0].queryVal); - stats["standard deviation"] = matsDataUtils.stdev(data[0].queryVal); - stats["minimum time"] = data[0].stats.reduce(function (prev, curr) { - return prev < curr.min_time ? prev : curr.min_time; - }); - stats["minimum time"] = moment - .utc(stats["minimum time"] * 1000) - .format("YYYY-MM-DD HH:mm"); - stats["maximum time"] = data[0].stats.reduce(function (prev, curr) { - return prev > curr.max_time ? prev : curr.max_time; - }); - stats["maximum time"] = moment - .utc(stats["maximum time"] * 1000) - .format("YYYY-MM-DD HH:mm"); +// PUBLIC METHODS +// administration tools +const addSentAddress = new ValidatedMethod({ + name: "matsMethods.addSentAddress", + validate: new SimpleSchema({ + toAddress: { + type: String, + }, + }).validator(), + run(toAddress) { + if (!Meteor.userId()) { + throw new Meteor.Error(401, "not-logged-in"); + } + matsCollections.SentAddresses.upsert( + { + address: toAddress, + }, + { + address: toAddress, + userId: Meteor.userId(), + } + ); + return false; + }, +}); - returnData.stats[data[0].label] = stats; +// administation tool +const applyAuthorization = new ValidatedMethod({ + name: "matsMethods.applyAuthorization", + validate: new SimpleSchema({ + settings: { + type: Object, + blackbox: true, + }, + }).validator(), + run(settings) { + if (Meteor.isServer) { + let roles; + let roleName; + let authorization; - var curveData = []; // map of maps - isCTC = - data[0] !== undefined && - data[0].stats !== undefined && - data[0].stats[0] !== undefined && - data[0].stats[0].hit !== undefined; - for (var si = 0; si < data[0].siteName.length; si++) { - var curveDataElement = {}; - curveDataElement["site name"] = data[0].siteName[si]; - curveDataElement["number of times"] = data[0].stats[si].nTimes; - if (isCTC) { - curveDataElement.stat = data[0].queryVal[si]; - curveDataElement.hit = data[0].stats[si].hit; - curveDataElement.fa = data[0].stats[si].fa; - curveDataElement.miss = data[0].stats[si].miss; - curveDataElement.cn = data[0].stats[si].cn; - } else { - curveDataElement["start date"] = moment - .utc(data[0].stats[si].min_time * 1000) - .format("YYYY-MM-DD HH:mm"); - curveDataElement["end date"] = moment - .utc(data[0].stats[si].max_time * 1000) - .format("YYYY-MM-DD HH:mm"); - curveDataElement.stat = data[0].queryVal[si]; + const { userRoleName } = settings; + const { userRoleDescription } = settings; + const { authorizationRole } = settings; + const { newUserEmail } = settings; + const { existingUserEmail } = settings; + + if (authorizationRole) { + // existing role - the role roleName - no need to verify as the selection list came from the database + roleName = authorizationRole; + } else if (userRoleName && userRoleDescription) { + // possible new role - see if it happens to already exist + const role = matsCollections.Roles.findOne({ + name: userRoleName, + }); + if (role === undefined) { + // need to add new role using description + matsCollections.Roles.upsert( + { + name: userRoleName, + }, + { + $set: { + description: userRoleDescription, + }, } - curveData.push(curveDataElement); + ); + roleName = userRoleName; + } else { + // see if the description matches... + roleName = role.name; + const { description } = role; + if (description !== userRoleDescription) { + // have to update the description + matsCollections.Roles.upsert( + { + name: userRoleName, + }, + { + $set: { + description: userRoleDescription, + }, + } + ); } - returnData.data[data[0].label] = curveData; - break; - case matsTypes.PlotTypes.histogram: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves - } - var stats = {}; - stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.glob_mean; - stats["standard deviation"] = data[ci].glob_stats.glob_sd; - stats.n = data[ci].glob_stats.glob_n; - stats.minimum = data[ci].glob_stats.glob_min; - stats.maximum = data[ci].glob_stats.glob_max; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].x.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} bin range`] = - data[ci].bin_stats[cdi].binLabel; - curveDataElement.n = data[ci].bin_stats[cdi].bin_n; - curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; - curveDataElement["bin lower bound"] = data[ci].bin_stats[cdi].binLowBound; - curveDataElement["bin upper bound"] = data[ci].bin_stats[cdi].binUpBound; - curveDataElement["bin mean"] = data[ci].bin_stats[cdi].bin_mean; - curveDataElement["bin std dev"] = data[ci].bin_stats[cdi].bin_sd; - curveData.push(curveDataElement); - } - returnData.data[data[ci].label] = curveData; + } + } + // now we have a role roleName - now we need an email + if (existingUserEmail) { + // existing user - no need to verify as the selection list came from the database + // see if it already has the role + authorization = matsCollections.Authorization.findOne({ + email: existingUserEmail, + }); + roles = authorization.roles; + if (roles.indexOf(roleName) === -1) { + // have to add the role + if (roleName) { + roles.push(roleName); } - break; - case matsTypes.PlotTypes.ensembleHistogram: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - for (var ci = 0; ci < data.length; ci++) { - // for each curve - // if the curve label is a reserved word do not process the curve (its a zero or max curve) - var reservedWords = Object.values(matsTypes.ReservedWords); - if (reservedWords.indexOf(data[ci].label) >= 0) { - continue; // don't process the zero or max curves + matsCollections.Authorization.upsert( + { + email: existingUserEmail, + }, + { + $set: { + roles, + }, } - var stats = {}; - stats.label = data[ci].label; - stats.mean = data[ci].glob_stats.dMean; - stats["standard deviation"] = data[ci].glob_stats.sd; - stats.n = data[ci].glob_stats.nGood; - stats.minimum = data[ci].glob_stats.minVal; - stats.maximum = data[ci].glob_stats.maxVal; - returnData.stats[data[ci].label] = stats; - - var curveData = []; // array of maps - for (var cdi = 0; cdi < data[ci].x.length; cdi++) { - // for each datapoint - var curveDataElement = {}; - curveDataElement[`${data[ci].label} bin`] = data[ci].x[cdi]; - curveDataElement.n = data[ci].bin_stats[cdi].bin_n; - curveDataElement["bin rel freq"] = data[ci].bin_stats[cdi].bin_rf; - curveData.push(curveDataElement); + ); + } + } else if (newUserEmail) { + // possible new authorization - see if it happens to exist + authorization = matsCollections.Authorization.findOne({ + email: newUserEmail, + }); + if (authorization !== undefined) { + // authorization exists - add role to roles if necessary + roles = authorization.roles; + if (roles.indexOf(roleName) === -1) { + // have to add the role + if (roleName) { + roles.push(roleName); } - returnData.data[data[ci].label] = curveData; + matsCollections.Authorization.upsert( + { + email: existingUserEmail, + }, + { + $set: { + roles, + }, + } + ); } - break; - case matsTypes.PlotTypes.contour: - case matsTypes.PlotTypes.contourDiff: - var returnData = {}; - returnData.stats = {}; // map of maps - returnData.data = {}; // map of arrays of maps - var stats = {}; - stats.label = data[0].label; - stats["total number of points"] = data[0].glob_stats.n; - stats["mean stat"] = data[0].glob_stats.mean; - stats["minimum time"] = moment - .utc(data[0].glob_stats.minDate * 1000) - .format("YYYY-MM-DD HH:mm"); - stats["maximum time"] = moment - .utc(data[0].glob_stats.maxDate * 1000) - .format("YYYY-MM-DD HH:mm"); - - returnData.stats[data[0].label] = stats; - - var curveData = []; // array of maps - isCTC = - data[0] !== undefined && - data[0].hitTextOutput !== undefined && - data[0].hitTextOutput.length > 0; - for (var si = 0; si < data[0].xTextOutput.length; si++) { - var curveDataElement = {}; - curveDataElement.xVal = data[0].xTextOutput[si]; - curveDataElement.yVal = data[0].yTextOutput[si]; - curveDataElement.stat = data[0].zTextOutput[si]; - curveDataElement.N = data[0].nTextOutput[si]; - if (isCTC) { - curveDataElement.hit = data[0].hitTextOutput[si]; - curveDataElement.fa = data[0].faTextOutput[si]; - curveDataElement.miss = data[0].missTextOutput[si]; - curveDataElement.cn = data[0].cnTextOutput[si]; - } else { - curveDataElement["Start Date"] = moment - .utc(data[0].minDateTextOutput[si] * 1000) - .format("YYYY-MM-DD HH:mm"); - curveDataElement["End Date"] = moment - .utc(data[0].maxDateTextOutput[si] * 1000) - .format("YYYY-MM-DD HH:mm"); - } - curveData.push(curveDataElement); + } else { + // need a new authorization + roles = []; + if (roleName) { + roles.push(roleName); } - returnData.data[data[0].label] = curveData; - break; - case matsTypes.PlotTypes.scatter2d: - var returnData = {}; // returns a map of arrays of maps - var firstBestFitIndex = -1; - var bestFitIndexes = {}; - for (var ci = 0; ci < data.length; ci++) { - if (ci === firstBestFitIndex) { - break; // best fit curves are at the end so do not do further processing - } - var curveData = data[ci]; - // look for a best fit curve - only have to look at curves with higher index than this one - const bestFitIndex = -1; - for (let cbi = ci + 1; cbi < data.length; cbi++) { - if ( - data[cbi].label.indexOf(curveData.label) !== -1 && - data[cbi].label.indexOf("-best fit") !== -1 - ) { - bestFitIndexes[ci] = cbi; - if (firstBestFitIndex === -1) { - firstBestFitIndex = cbi; - } - break; - } - } - const curveTextData = []; - for (var cdi = 0; cdi < curveData.data.length; cdi++) { - const element = {}; - element.xAxis = curveData.data[cdi][0]; - element.yAxis = curveData.data[cdi][1]; - if (bestFitIndexes[ci] === undefined) { - element["best fit"] = "none;"; - } else { - element["best fit"] = data[bestFitIndexes[ci]].data[cdi][1]; + if (newUserEmail) { + matsCollections.Authorization.upsert( + { + email: newUserEmail, + }, + { + $set: { + roles, + }, } - curveTextData.push(element); - } - returnData[curveData.label] = curveTextData; + ); + } + } + } + } + return false; + }, +}); + +// database controls +const applyDatabaseSettings = new ValidatedMethod({ + name: "matsMethods.applyDatabaseSettings", + validate: new SimpleSchema({ + settings: { + type: Object, + blackbox: true, + }, + }).validator(), + + run(settings) { + if (Meteor.isServer) { + if (settings.name) { + matsCollections.Databases.upsert( + { + name: settings.name, + }, + { + $set: { + name: settings.name, + role: settings.role, + status: settings.status, + host: settings.host, + database: settings.database, + user: settings.user, + password: settings.password, + }, } - break; - default: - return undefined; + ); } - returnData.dsiRealPageIndex = dsiRealPageIndex; - returnData.dsiTextDirection = dsiTextDirection; - return returnData; - } catch (error) { - throw new Meteor.Error( - `Error in _getFlattenedResultData function: ${error.message}` - ); } - } -}; - -// private method for getting pagenated data -// a newPageIndex of -1000 means get all the data (used for export) -// a newPageIndex of -2000 means get just the last page -const _getPagenatedData = function (rky, p, np) { - if (Meteor.isServer) { - const key = rky; - const myPageIndex = p; - const newPageIndex = np; - let ret; - let rawReturn; + return false; + }, +}); - try { - const result = matsCache.getResult(key); - rawReturn = result === undefined ? undefined : result.result; // getResult structure is {key:something, result:resultObject} - } catch (e) { - console.log("_getPagenatedData: Error - ", e); - return undefined; +// administration tools +const deleteSettings = new ValidatedMethod({ + name: "matsMethods.deleteSettings", + validate: new SimpleSchema({ + name: { + type: String, + }, + }).validator(), + run(params) { + if (!Meteor.userId()) { + throw new Meteor.Error("not-logged-in"); } - ret = rawReturn === undefined ? undefined : JSON.parse(JSON.stringify(rawReturn)); - let start; - let end; - let direction = 1; - if (newPageIndex === -1000) { - // all the data - start = 0; - end = Number.MAX_VALUE; - } else if (newPageIndex === -2000) { - // just the last page - start = -2000; - direction = -1; - } else if (myPageIndex <= newPageIndex) { - // proceed forward - start = (newPageIndex - 1) * 100; - end = newPageIndex * 100; - } else { - // move back - direction = -1; - start = newPageIndex * 100; - end = (newPageIndex + 1) * 100; + if (Meteor.isServer) { + matsCollections.CurveSettings.remove({ + name: params.name, + }); } + }, +}); - let dsiStart; - let dsiEnd; - for (let csi = 0; csi < ret.data.length; csi++) { - if (!ret.data[csi].x || ret.data[csi].x.length <= 100) { - continue; // don't bother pagenating datasets less than or equal to a page - ret is rawReturn - } - dsiStart = start; - dsiEnd = end; - if (dsiStart > ret.data[csi].x.length || dsiStart === -2000) { - // show the last page if we either requested it specifically or are trying to navigate past it - dsiStart = Math.floor(rawReturn.data[csi].x.length / 100) * 100; - dsiEnd = rawReturn.data[csi].x.length; - if (dsiEnd === dsiStart) { - // make sure the last page isn't empty--if rawReturn.data[csi].data.length/100 produces a whole number, - // dsiStart and dsiEnd would be the same. This makes sure that the last full page is indeed the last page, without a phantom empty page afterwards - dsiStart = dsiEnd - 100; - } - } - if (dsiStart < 0) { - // show the first page if we are trying to navigate before it - dsiStart = 0; - dsiEnd = 100; - } - if (dsiEnd < dsiStart) { - // make sure that the end is after the start - dsiEnd = dsiStart + 100; - } - if (dsiEnd > ret.data[csi].x.length) { - // make sure we don't request past the end -- if results are one page, this should convert the - // start and end from 0 and 100 to 0 and whatever the end is. - dsiEnd = ret.data[csi].x.length; - } - ret.data[csi].x = rawReturn.data[csi].x.slice(dsiStart, dsiEnd); - ret.data[csi].y = rawReturn.data[csi].y.slice(dsiStart, dsiEnd); - ret.data[csi].stats = rawReturn.data[csi].stats.slice(dsiStart, dsiEnd); - ret.data[csi].glob_stats = rawReturn.data[csi].glob_stats; +// drop a single instance of a scorecard +const dropScorecardInstance = new ValidatedMethod({ + name: "matsMethods.dropScorecardInstance", + validate: new SimpleSchema({ + userName: { + type: String, + }, + name: { + type: String, + }, + submittedTime: { + type: String, + }, + processedAt: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + return dropThisScorecardInstance( + params.userName, + params.name, + params.submittedTime, + params.processedAt + ); } + return null; + }, +}); - if (direction === 1) { - ret.dsiRealPageIndex = Math.floor(dsiEnd / 100); - } else { - ret.dsiRealPageIndex = Math.floor(dsiStart / 100); +// administration tools +const emailImage = new ValidatedMethod({ + name: "matsMethods.emailImage", + validate: new SimpleSchema({ + imageStr: { + type: String, + }, + toAddress: { + type: String, + }, + subject: { + type: String, + }, + }).validator(), + run(params) { + const { imageStr } = params; + const { toAddress } = params; + const { subject } = params; + if (!Meteor.userId()) { + throw new Meteor.Error(401, "not-logged-in"); } - ret.dsiTextDirection = direction; - return ret; - } -}; + const fromAddress = Meteor.user().services.google.email; + // these come from google - see + // http://masashi-k.blogspot.fr/2013/06/sending-mail-with-gmail-using-xoauth2.html + // http://stackoverflow.com/questions/24098461/nodemailer-gmail-what-exactly-is-a-refresh-token-and-how-do-i-get-one/24123550 -// private define a middleware for refreshing the metadata -const _refreshMetadataMWltData = function (params, req, res, next) { - if (Meteor.isServer) { - console.log("Server route asked to refresh metadata"); + // the gmail account for the credentials is mats.mail.daemon@gmail.com - pwd mats2015! + // var clientId = "339389735380-382sf11aicmgdgn7e72p4end5gnm9sad.apps.googleusercontent.com"; + // var clientSecret = "7CfNN-tRl5QAL595JTW2TkRl"; + // var refresh_token = "1/PDql7FR01N2gmq5NiTfnrT-OlCYC3U67KJYYDNPeGnA"; + const credentials = matsCollections.Credentials.findOne( + { + name: "oauth_google", + }, + { + clientId: 1, + clientSecret: 1, + refresh_token: 1, + } + ); + const { clientId } = credentials; + const { clientSecret } = credentials; + const refreshToken = credentials.refresh_token; + let smtpTransporter; try { - console.log("GUI asked to refresh metadata"); - _checkMetaDataRefresh(); + const Nodemailer = require("nodemailer"); + smtpTransporter = Nodemailer.createTransport("SMTP", { + service: "Gmail", + auth: { + XOAuth2: { + user: "mats.gsl@noaa.gov", + clientId, + clientSecret, + refreshToken, + }, + }, + }); } catch (e) { - console.log(e); - res.end( - `` + - `

refreshMetadata Failed!

` + - `

${e.message}

` + - `` - ); + throw new Meteor.Error(401, `Transport error ${e.message()}`); } - res.end("

refreshMetadata Done!

"); - } -}; + try { + const mailOptions = { + sender: fromAddress, + replyTo: fromAddress, + from: fromAddress, + to: toAddress, + subject, + attachments: [ + { + filename: "graph.png", + contents: Buffer.from(imageStr.split("base64,")[1], "base64"), + }, + ], + }; -// private middleware for causing the scorecard to refresh its mongo collection for a given document -const _refreshScorecard = function (params, req, res, next) { - if (Meteor.isServer) { - const docId = decodeURIComponent(params.docId); - // get userName, name, submitted, processedAt from id - // SC:anonymous--submitted:20230322230435--1block:0:02/19/2023_20_00_-_03/21/2023_20_00 - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardData: No cbScorecardPool defined"); - } - const statement = `SELECT sc.* - From - vxdata._default.SCORECARD sc - WHERE - sc.id='${docId}';`; - cbScorecardPool - .queryCB(statement) - .then((result) => { - // insert this result into the mongo Scorecard collection - createdAt is used for TTL - // created at gets updated each display even if it already existed. - // TTL is 24 hours - if (typeof result === "string") { - throw new Error(`Error from couchbase query - ${result}`); - } else if (result[0] === undefined) { - throw new Error("Error from couchbase query - document not found"); - } else { - matsCollections.Scorecard.upsert( - { - "scorecard.userName": result[0].userName, - "scorecard.name": result[0].name, - "scorecard.submitted": result[0].submitted, - "scorecard.processedAt": result[0].processedAt, - }, - { - $set: { - createdAt: new Date(), - scorecard: result[0], - }, - } + smtpTransporter.sendMail(mailOptions, function (error, response) { + if (error) { + console.log( + `smtpTransporter error ${error} from:${fromAddress} to:${toAddress}` ); + } else { + console.log(`${response} from:${fromAddress} to:${toAddress}`); } - res.end("

refreshScorecard Done!

"); - }) - .catch((err) => { - res.end( - `` + - `

refreshScorecard Failed!

` + - `

${err.message}

` + - `` - ); + smtpTransporter.close(); }); - } -}; + } catch (e) { + throw new Meteor.Error(401, `Send error ${e.message()}`); + } + return false; + }, +}); + +// administation tool +const getAuthorizations = new ValidatedMethod({ + name: "matsMethods.getAuthorizations", + validate: new SimpleSchema({}).validator(), + run() { + let roles = []; + if (Meteor.isServer) { + const userEmail = Meteor.user().services.google.email.toLowerCase(); + roles = matsCollections.Authorization.findOne({ + email: userEmail, + }).roles; + } + return roles; + }, +}); -const _setStatusScorecard = function (params, req, res, next) { - if (Meteor.isServer) { - const docId = decodeURIComponent(params.docId); - let body = ""; - req.on( - "data", - Meteor.bindEnvironment(function (data) { - body += data; - }) - ); +// administration tool - req.on( - "end", - Meteor.bindEnvironment(function () { - // console.log(body); - try { - const doc = JSON.parse(body); - const { status } = doc; - const { error } = doc; - const found = matsCollections.Scorecard.find({ id: docId }).fetch(); - if (found.length === 0) { - throw new Error("Error from scorecard lookup - document not found"); +const getRunEnvironment = new ValidatedMethod({ + name: "matsMethods.getRunEnvironment", + validate: new SimpleSchema({}).validator(), + run() { + return Meteor.settings.public.run_environment; + }, +}); + +const getDefaultGroupList = new ValidatedMethod({ + name: "matsMethods.getDefaultGroupList", + validate: new SimpleSchema({}).validator(), + run() { + return matsTypes.DEFAULT_GROUP_LIST; + }, +}); + +// retrieves the saved query results (or downsampled results) +const getGraphData = new ValidatedMethod({ + name: "matsMethods.getGraphData", + validate: new SimpleSchema({ + plotParams: { + type: Object, + blackbox: true, + }, + plotType: { + type: String, + }, + expireKey: { + type: Boolean, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + const plotGraphFunction = matsCollections.PlotGraphFunctions.findOne({ + plotType: params.plotType, + }); + const { dataFunction } = plotGraphFunction; + let ret; + try { + const hash = require("object-hash"); + const key = hash(params.plotParams); + if (process.env.NODE_ENV === "development" || params.expireKey) { + matsCache.expireKey(key); + } + const results = matsCache.getResult(key); + if (results === undefined) { + // results aren't in the cache - need to process data routine + const Future = require("fibers/future"); + const future = new Future(); + global[dataFunction](params.plotParams, function (result) { + ret = saveResultData(result); + future.return(ret); + }); + return future.wait(); + } + // results were already in the matsCache (same params and not yet expired) + // are results in the downsampled collection? + const dsResults = DownSampleResults.findOne( + { + key, + }, + {}, + { + disableOplog: true, } - matsCollections.Scorecard.upsert( + ); + if (dsResults !== undefined) { + // results are in the mongo cache downsampled collection - returned the downsampled graph data + ret = dsResults; + // update the expire time in the downsampled collection - this requires a new Date + DownSampleResults.rawCollection().update( { - id: docId, + key, }, { $set: { - status, + createdAt: new Date(), }, } ); - // set error if there is one somehow. (use the session?) - res.end("

setScorecardStatus Done!

"); - } catch (err) { - res.statusCode = 400; - res.end( - `` + - `

setScorecardStatus Failed!

` + - `

${err.message}

` + - `` + } else { + ret = results; // {key:someKey, result:resultObject} + // refresh expire time. The only way to perform a refresh on matsCache is to re-save the result. + matsCache.storeResult(results.key, results); + } + const sizeof = require("object-sizeof"); + console.log("result.data size is ", sizeof(results)); + return ret; + } catch (dataFunctionError) { + if (dataFunctionError.toLocaleString().indexOf("INFO:") !== -1) { + throw new Meteor.Error(dataFunctionError.message); + } else { + throw new Meteor.Error( + `Error in getGraphData function:${dataFunction} : ${dataFunctionError.message}` ); } - }) - ); - } -}; + } + } + return null; + }, +}); -// private save the result from the query into mongo and downsample if that result's size is greater than 1.2Mb -const _saveResultData = function (result) { - if (Meteor.isServer) { - const sizeof = require("object-sizeof"); - const hash = require("object-hash"); - const key = hash(result.basis.plotParams); - const threshold = 1200000; - let ret = {}; - try { - const dSize = sizeof(result.data); - // console.log("result.basis.data size is ", dSize); - // TimeSeries and DailyModelCycle are the only plot types that require downSampling - if ( - dSize > threshold && - (result.basis.plotParams.plotTypes.TimeSeries || - result.basis.plotParams.plotTypes.DailyModelCycle) - ) { - // greater than threshold need to downsample - // downsample and save it in DownSampleResult - console.log("DownSampling"); - const downsampler = require("downsample-lttb"); - let totalPoints = 0; - for (let di = 0; di < result.data.length; di++) { - totalPoints += result.data[di].x_epoch.length; - } - const allowedNumberOfPoints = (threshold / dSize) * totalPoints; - const downSampleResult = - result === undefined ? undefined : JSON.parse(JSON.stringify(result)); - for (var ci = 0; ci < result.data.length; ci++) { - const dsData = {}; - const xyDataset = result.data[ci].x_epoch.map(function (d, index) { - return [result.data[ci].x_epoch[index], result.data[ci].y[index]]; - }); - const ratioTotalPoints = xyDataset.length / totalPoints; - const myAllowedPoints = Math.round(ratioTotalPoints * allowedNumberOfPoints); - // downsample the array - var downsampledSeries; - if (myAllowedPoints < xyDataset.length && xyDataset.length > 2) { - downsampledSeries = downsampler.processData(xyDataset, myAllowedPoints); - // replace the y attributes (tooltips etc.) with the y attributes from the nearest x - let originalIndex = 0; - // skip through the original dataset capturing each downSampled data point - const arrayKeys = []; - const nonArrayKeys = []; - const keys = Object.keys(result.data[ci]); - for (var ki = 0; ki < keys.length; ki++) { - if (keys[ki] !== "x_epoch") { - if (Array.isArray(result.data[ci][keys[ki]])) { - arrayKeys.push(keys[ki]); - dsData[keys[ki]] = []; - } else { - nonArrayKeys.push(keys[ki]); - } - } - } - // We only ever downsample series plots - never profiles and series plots only ever have error_y arrays. - // This is a little hacky but what is happening is we putting error_y.array on the arrayKeys list so that it gets its - // downsampled equivalent values. - for (ki = 0; ki < nonArrayKeys.length; ki++) { - dsData[nonArrayKeys[ki]] = JSON.parse( - JSON.stringify(result.data[ci][nonArrayKeys[ki]]) - ); - } - // remove the original error_y array data. - dsData.error_y.array = []; - for (let dsi = 0; dsi < downsampledSeries.length; dsi++) { - while ( - originalIndex < result.data[ci].x_epoch.length && - result.data[ci].x_epoch[originalIndex] < downsampledSeries[dsi][0] - ) { - originalIndex++; - } - // capture the stuff related to this downSampled data point (downSampled data points are always a subset of original data points) - for (ki = 0; ki < arrayKeys.length; ki++) { - dsData[arrayKeys[ki]][dsi] = - result.data[ci][arrayKeys[ki]][originalIndex]; - } - dsData.error_y.array[dsi] = result.data[ci].error_y.array[originalIndex]; - } - // add downsampled annotation to curve options - downSampleResult[ci] = dsData; - downSampleResult[ci].annotation += " **DOWNSAMPLED**"; - } else { - downSampleResult[ci] = result.data[ci]; +// retrieves the saved query results (or downsampled results) for a specific key +const getGraphDataByKey = new ValidatedMethod({ + name: "matsMethods.getGraphDataByKey", + validate: new SimpleSchema({ + resultKey: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + let ret; + const key = params.resultKey; + try { + const dsResults = DownSampleResults.findOne( + { + key, + }, + {}, + { + disableOplog: true, } - downSampleResult.data[ci] = downSampleResult[ci]; + ); + if (dsResults !== undefined) { + ret = dsResults; + } else { + ret = matsCache.getResult(key); // {key:someKey, result:resultObject} } - DownSampleResults.rawCollection().insert({ - createdAt: new Date(), - key, - result: downSampleResult, - }); // createdAt ensures expiration set in mats-collections - ret = { - key, - result: downSampleResult, - }; - } else { - ret = { + const sizeof = require("object-sizeof"); + console.log("getGraphDataByKey results size is ", sizeof(dsResults)); + return ret; + } catch (error) { + throw new Meteor.Error( + `Error in getGraphDataByKey function:${key} : ${error.message}` + ); + } + } + return null; + }, +}); + +const getLayout = new ValidatedMethod({ + name: "matsMethods.getLayout", + validate: new SimpleSchema({ + resultKey: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + let ret; + const key = params.resultKey; + try { + ret = LayoutStoreCollection.rawCollection().findOne({ key, - result, - }; + }); + return ret; + } catch (error) { + throw new Meteor.Error(`Error in getLayout function:${key} : ${error.message}`); } - // save original dataset in the matsCache - if ( - result.basis.plotParams.plotTypes.TimeSeries || - result.basis.plotParams.plotTypes.DailyModelCycle - ) { - for (var ci = 0; ci < result.data.length; ci++) { - delete result.data[ci].x_epoch; // we only needed this as an index for downsampling - } + } + return null; + }, +}); + +const getScorecardSettings = new ValidatedMethod({ + name: "matsMethods.getScorecardSettings", + validate: new SimpleSchema({ + settingsKey: { + type: String, + }, + }).validator(), + async run(params) { + if (Meteor.isServer) { + const key = params.settingsKey; + try { + // global cbScorecardSettingsPool + const rv = await cbScorecardSettingsPool.getCB(key); + return { scorecardSettings: rv.content }; + } catch (error) { + throw new Meteor.Error( + `Error in getScorecardSettings function:${key} : ${error.message}` + ); + } + } + return null; + }, +}); + +const getPlotParamsFromScorecardInstance = new ValidatedMethod({ + name: "matsMethods.getPlotParamsFromScorecardInstance", + validate: new SimpleSchema({ + userName: { + type: String, + }, + name: { + type: String, + }, + submitted: { + type: String, + }, + processedAt: { + type: String, + }, + }).validator(), + run(params) { + try { + if (Meteor.isServer) { + return getThesePlotParamsFromScorecardInstance( + params.userName, + params.name, + params.submitted, + params.processedAt + ); } - matsCache.storeResult(key, { - key, - result, - }); // lifespan is handled by lowDb (internally) in matscache } catch (error) { - if (error.toLocaleString().indexOf("larger than the maximum size") !== -1) { - throw new Meteor.Error(": Requesting too much data... try averaging"); + throw new Meteor.Error( + `Error in getPlotParamsFromScorecardInstance function:${error.message}` + ); + } + return null; + }, +}); + +/* +getPlotResult is used by the graph/text_*_output templates which are used to display textual results. +Because the data isn't being rendered graphically this data is always full size, i.e. NOT downsampled. +That is why it only finds it in the Result file cache, never the DownSampleResult collection. + +Because the dataset can be so large ... e.g. megabytes the data retrieval is pagenated. The index is +applied to the underlying datasets.The data gets stripped down and flattened to only contain the data neccesary for text presentation. +A new page index of -1000 means get all the data i.e. no pagenation. + */ +const getPlotResult = new ValidatedMethod({ + name: "matsMethods.getPlotResult", + validate: new SimpleSchema({ + resultKey: { + type: String, + }, + pageIndex: { + type: Number, + }, + newPageIndex: { + type: Number, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + const rKey = params.resultKey; + const pi = params.pageIndex; + const npi = params.newPageIndex; + let ret = {}; + try { + ret = getFlattenedResultData(rKey, pi, npi); + } catch (e) { + console.log(e); } + return ret; } - return ret; - } -}; + return null; + }, +}); -// Utility method for writing out the meteor.settings file -const _write_settings = function (settings, appName) { - const fs = require("fs"); - let settingsPath = process.env.METEOR_SETTINGS_DIR; - if (!settingsPath) { - console.log( - "environment var METEOR_SETTINGS_DIR is undefined: setting it to /usr/app/settings" - ); - settingsPath = "/usr/app/settings"; - } - if (!fs.existsSync(settingsPath)) { - fs.mkdirSync(settingsPath, { - recursive: true, - }); - } - let appSettings = {}; - let newSettings = {}; - try { - const appSettingsData = fs.readFileSync(`${settingsPath}/${appName}/settings.json`); - appSettings = JSON.parse(appSettingsData); - } catch (e) { - appSettings = { - private: {}, - public: {}, - }; - } - newSettings = settings; - // Merge settings into appSettings - newSettings.private = { - ...appSettings.private, - ...settings.private, - }; - newSettings.public = { - ...appSettings.public, - ...settings.public, - }; - // write the settings file - const jsonSettings = JSON.stringify(newSettings, null, 2); - // console.log (jsonSettings); - fs.writeFileSync(`${settingsPath}/${appName}/settings.json`, jsonSettings, { - encoding: "utf8", - flag: "w", - }); -}; -// return the scorecard for the provided selectors -const _getScorecardData = async function (userName, name, submitted, processedAt) { - try { - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardData: No cbScorecardPool defined"); +const getReleaseNotes = new ValidatedMethod({ + name: "matsMethods.getReleaseNotes", + validate: new SimpleSchema({}).validator(), + run() { + // return Assets.getText('public/MATSReleaseNotes.html'); + // } + if (Meteor.isServer) { + const Future = require("fibers/future"); + const fse = require("fs-extra"); + const dFuture = new Future(); + let fData; + let file; + if (process.env.NODE_ENV === "development") { + file = `${process.env.PWD}/.meteor/local/build/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; + } else { + file = `${process.env.PWD}/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; + } + try { + fse.readFile(file, "utf8", function (err, data) { + if (err) { + fData = err.message; + dFuture.return(); + } else { + fData = data; + dFuture.return(); + } + }); + } catch (e) { + fData = e.message; + dFuture.return(); + } + dFuture.wait(); + return fData; } - const statement = `SELECT sc.* - From - vxdata._default.SCORECARD sc - WHERE - sc.type='SC' - AND sc.userName='${userName}' - AND sc.name='${name}' - AND sc.processedAt=${processedAt} - AND sc.submitted=${submitted};`; - const result = await cbScorecardPool.queryCBWithConsistency(statement); - if (typeof result === "string" && result.indexOf("ERROR")) { - throw new Meteor.Error(result); + return null; + }, +}); + +const setCurveParamDisplayText = new ValidatedMethod({ + name: "matsMethods.setCurveParamDisplayText", + validate: new SimpleSchema({ + paramName: { + type: String, + }, + newText: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + return matsCollections[params.paramName].update( + { name: params.paramName }, + { $set: { controlButtonText: params.newText } } + ); + } + return null; + }, +}); + +const getScorecardData = new ValidatedMethod({ + name: "matsMethods.getScorecardData", + validate: new SimpleSchema({ + userName: { + type: String, + }, + name: { + type: String, + }, + submitted: { + type: String, + }, + processedAt: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + return getThisScorecardData( + params.userName, + params.name, + params.submitted, + params.processedAt + ); + } + return null; + }, +}); + +const getScorecardInfo = new ValidatedMethod({ + name: "matsMethods.getScorecardInfo", + validate: new SimpleSchema({}).validator(), + run() { + if (Meteor.isServer) { + return getThisScorecardInfo(); + } + return null; + }, +}); + +// administration tool +const getUserAddress = new ValidatedMethod({ + name: "matsMethods.getUserAddress", + validate: new SimpleSchema({}).validator(), + run() { + if (Meteor.isServer) { + return Meteor.user().services.google.email.toLowerCase(); } - // insert this result into the mongo Scorecard collection - createdAt is used for TTL - // created at gets updated each display even if it already existed. - // TTL is 24 hours - matsCollections.Scorecard.upsert( - { - "scorecard.userName": result[0].userName, - "scorecard.name": result[0].name, - "scorecard.submitted": result[0].submitted, - "scorecard.processedAt": result[0].processedAt, - }, - { - $set: { - createdAt: new Date(), - scorecard: result[0], - }, - } - ); - const docID = matsCollections.Scorecard.findOne( - { - "scorecard.userName": result[0].userName, - "scorecard.name": result[0].name, - "scorecard.submitted": result[0].submitted, - "scorecard.processedAt": result[0].processedAt, - }, - { _id: 1 } - )._id; - // no need to return the whole thing, just the identifying fields - // and the ID. The app will find the whole thing in the mongo collection. - return { scorecard: result[0], docID }; - } catch (err) { - console.log(`_getScorecardData error : ${err.message}`); - return { - error: err.message, - }; - } -}; + return null; + }, +}); -// return the scorecard status information from the couchbase database -const _getScorecardInfo = async function () { - try { - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardInfo: No cbScorecardPool defined"); +// app utility +const insertColor = new ValidatedMethod({ + name: "matsMethods.insertColor", + validate: new SimpleSchema({ + newColor: { + type: String, + }, + insertAfterIndex: { + type: Number, + }, + }).validator(), + run(params) { + if (params.newColor === "rgb(255,255,255)") { + return false; } + const colorScheme = matsCollections.ColorScheme.findOne({}); + colorScheme.colors.splice(params.insertAfterIndex, 0, params.newColor); + matsCollections.update({}, colorScheme); + return false; + }, +}); - const statement = `SELECT - sc.id, - sc.userName, - sc.name, - sc.status, - sc.processedAt as processedAt, - sc.submitted, - sc.dateRange - From - vxdata._default.SCORECARD sc - WHERE - sc.type='SC';`; - const result = await cbScorecardPool.queryCBWithConsistency(statement); - scMap = {}; - result.forEach(function (elem) { - if (!Object.keys(scMap).includes(elem.userName)) { - scMap[elem.userName] = {}; +// administration tool +const readFunctionFile = new ValidatedMethod({ + name: "matsMethods.readFunctionFile", + validate: new SimpleSchema({ + file: { + type: String, + }, + type: { + type: String, + }, + }).validator(), + run(params) { + if (Meteor.isServer) { + const future = require("fibers/future"); + const fse = require("fs-extra"); + let path = ""; + let fData; + if (params.type === "data") { + path = `/web/static/dataFunctions/${params.file}`; + console.log(`exporting data file: ${path}`); + } else if (params.type === "graph") { + path = `/web/static/displayFunctions/${params.file}`; + console.log(`exporting graph file: ${path}`); + } else { + return "error - wrong type"; } - userElem = scMap[elem.userName]; - if (!Object.keys(userElem).includes(elem.name)) { - userElem[elem.name] = {}; + fse.readFile(path, function (err, data) { + if (err) throw err; + fData = data.toString(); + future.return(fData); + }); + return future.wait(); + } + return null; + }, +}); + +// refreshes the metadata for the app that's running +const refreshMetaData = new ValidatedMethod({ + name: "matsMethods.refreshMetaData", + validate: new SimpleSchema({}).validator(), + run() { + if (Meteor.isServer) { + try { + // console.log("GUI asked to refresh metadata"); + checkMetaDataRefresh(); + } catch (e) { + console.log(e); + throw new Meteor.Error("Server error: ", e.message); } - nameElem = userElem[elem.name]; - if (!Object.keys(nameElem).includes(elem.submited)) { - nameElem[elem.submitted] = {}; + } + return metaDataTableUpdates.find({}).fetch(); + }, +}); + +// administation tool +const removeAuthorization = new ValidatedMethod({ + name: "matsMethods.removeAuthorization", + validate: new SimpleSchema({ + settings: { + type: Object, + blackbox: true, + }, + }).validator(), + run(settings) { + if (Meteor.isServer) { + let email; + let roleName; + const { userRoleName } = settings; + const { authorizationRole } = settings; + const { newUserEmail } = settings; + const { existingUserEmail } = settings; + if (authorizationRole) { + // existing role - the role roleName - no need to verify as the selection list came from the database + roleName = authorizationRole; + } else if (userRoleName) { + roleName = userRoleName; + } + if (existingUserEmail) { + email = existingUserEmail; + } else { + email = newUserEmail; } - submittedElem = nameElem[elem.submitted]; - submittedElem[elem.processedAt] = { - id: elem.id, - status: elem.status, - submitted: elem.submitted, - }; - }); - return scMap; - } catch (err) { - console.log(`_getScorecardInfo error : ${err.message}`); - return { - error: err.message, - }; - } -}; -const _getPlotParamsFromScorecardInstance = async function ( - userName, - name, - submitted, - processedAt -) { - try { - if (cbScorecardPool === undefined) { - throw new Meteor.Error("_getScorecardInfo: No cbScorecardPool defined"); - } - const statement = `SELECT sc.plotParams - From - vxdata._default.SCORECARD sc - WHERE - sc.type='SC' - AND sc.userName='${userName}' - AND sc.name='${name}' - AND sc.processedAt=${processedAt} - AND sc.submitted=${submitted};`; - const result = await cbScorecardPool.queryCBWithConsistency(statement); - if (typeof result === "string" && result.indexOf("ERROR")) { - throw new Meteor.Error(result); + // if user and role remove the role from the user + if (email && roleName) { + matsCollections.Authorization.update( + { + email, + }, + { + $pull: { + roles: roleName, + }, + } + ); + } + // if user and no role remove the user + if (email && !roleName) { + matsCollections.Authorization.remove({ + email, + }); + } + // if role and no user remove role and remove role from all users + if (roleName && !email) { + // remove the role + matsCollections.Roles.remove({ + name: roleName, + }); + // remove the roleName role from all the authorizations + matsCollections.Authorization.update( + { + roles: roleName, + }, + { + $pull: { + roles: roleName, + }, + }, + { + multi: true, + } + ); + } } - return result[0]; - } catch (err) { - console.log(`_getPlotParamsFromScorecardInstance error : ${err.message}`); - return { - error: err.message, - }; - } -}; + return false; + }, +}); + +// app utility +const removeColor = new ValidatedMethod({ + name: "matsMethods.removeColor", + validate: new SimpleSchema({ + removeColor: { + type: String, + }, + }).validator(), + run(params) { + const colorScheme = matsCollections.ColorScheme.findOne({}); + const removeIndex = colorScheme.colors.indexOf(params.removeColor); + colorScheme.colors.splice(removeIndex, 1); + matsCollections.ColorScheme.update({}, colorScheme); + return false; + }, +}); -// PUBLIC METHODS -// administration tools -const addSentAddress = new ValidatedMethod({ - name: "matsMethods.addSentAddress", +// database controls +const removeDatabase = new ValidatedMethod({ + name: "matsMethods.removeDatabase", validate: new SimpleSchema({ - toAddress: { + dbName: { type: String, }, }).validator(), - run(toAddress) { - if (!Meteor.userId()) { - throw new Meteor.Error(401, "not-logged-in"); + run(dbName) { + if (Meteor.isServer) { + matsCollections.Databases.remove({ + name: dbName, + }); } - matsCollections.SentAddresses.upsert( - { - address: toAddress, - }, - { - address: toAddress, - userId: Meteor.userId(), - } - ); - return false; }, }); -// administation tool -const applyAuthorization = new ValidatedMethod({ - name: "matsMethods.applyAuthorization", +const applySettingsData = new ValidatedMethod({ + name: "matsMethods.applySettingsData", validate: new SimpleSchema({ settings: { type: Object, blackbox: true, }, }).validator(), - run(settings) { + // this method forces a restart on purpose. We do not want retries + applyOptions: { + noRetry: true, + }, + run(settingsParam) { if (Meteor.isServer) { - let roles; - let roleName; - let authorization; - - const { userRoleName } = settings; - const { userRoleDescription } = settings; - const { authorizationRole } = settings; - const { newUserEmail } = settings; - const { existingUserEmail } = settings; + // Read the existing settings file + const { settings } = settingsParam; + console.log( + "applySettingsData - matsCollections.appName.findOne({}) is ", + matsCollections.appName.findOne({}) + ); + const { appName } = matsCollections.Settings.findOne({}); + writeSettings(settings, appName); + // in development - when being run by meteor, this should force a restart of the app. + // in case I am in a container - exit and force a reload + console.log( + `applySettingsData - process.env.NODE_ENV is: ${process.env.NODE_ENV}` + ); + if (process.env.NODE_ENV !== "development") { + console.log("applySettingsData - exiting after writing new Settings"); + process.exit(0); + } + } + }, +}); - if (authorizationRole) { - // existing role - the role roleName - no need to verify as the selection list came from the database - roleName = authorizationRole; - } else if (userRoleName && userRoleDescription) { - // possible new role - see if it happens to already exist - const role = matsCollections.Roles.findOne({ - name: userRoleName, - }); - if (role === undefined) { - // need to add new role using description - matsCollections.Roles.upsert( - { - name: userRoleName, - }, - { - $set: { - description: userRoleDescription, - }, - } - ); - roleName = userRoleName; +// makes sure all of the parameters display appropriate selections in relation to one another +// for default settings ... +const resetApp = async function (appRef) { + if (Meteor.isServer) { + const metaDataTableRecords = appRef.appMdr; + const { appPools } = appRef; + const type = appRef.appType; + const scorecard = Meteor.settings.public.scorecard + ? Meteor.settings.public.scorecard + : false; + const dbType = appRef.dbType ? appRef.dbType : matsTypes.DbTypes.mysql; + const appName = Meteor.settings.public.app ? Meteor.settings.public.app : "unnamed"; + const appTitle = Meteor.settings.public.title + ? Meteor.settings.public.title + : "Unnamed App"; + const appGroup = Meteor.settings.public.group + ? Meteor.settings.public.group + : "Misc. Apps"; + const thresholdUnits = Meteor.settings.public.threshold_units + ? Meteor.settings.public.threshold_units + : {}; + let appDefaultGroup = ""; + let appDefaultDB = ""; + let appDefaultModel = ""; + let appColor; + switch (type) { + case matsTypes.AppTypes.metexpress: + appColor = Meteor.settings.public.color + ? Meteor.settings.public.color + : "darkorchid"; + appDefaultGroup = Meteor.settings.public.default_group + ? Meteor.settings.public.default_group + : "NO GROUP"; + appDefaultDB = Meteor.settings.public.default_db + ? Meteor.settings.public.default_db + : "mv_default"; + appDefaultModel = Meteor.settings.public.default_model + ? Meteor.settings.public.default_model + : "Default"; + break; + case matsTypes.AppTypes.mats: + default: + if (dbType === matsTypes.DbTypes.couchbase) { + appColor = "#33abbb"; } else { - // see if the description matches... - roleName = role.name; - const { description } = role; - if (description !== userRoleDescription) { - // have to update the description - matsCollections.Roles.upsert( - { - name: userRoleName, - }, - { - $set: { - description: userRoleDescription, - }, - } - ); - } + appColor = Meteor.settings.public.color + ? Meteor.settings.public.color + : "#3366bb"; } + break; + } + const appTimeOut = Meteor.settings.public.mysql_wait_timeout + ? Meteor.settings.public.mysql_wait_timeout + : 300; + let depEnv = process.env.NODE_ENV; + const curveParams = curveParamsByApp[Meteor.settings.public.app]; + let appsToScore; + if (Meteor.settings.public.scorecard) { + appsToScore = Meteor.settings.public.apps_to_score + ? Meteor.settings.public.apps_to_score + : []; + } + let mapboxKey = "undefined"; + + // see if there's any messages to display to the users + const appMessage = Meteor.settings.public.alert_message + ? Meteor.settings.public.alert_message + : undefined; + + // set meteor settings defaults if they do not exist + if (isEmpty(Meteor.settings.private) || isEmpty(Meteor.settings.public)) { + // create some default meteor settings and write them out + let homeUrl = ""; + if (!process.env.ROOT_URL) { + homeUrl = "https://localhost/home"; + } else { + const homeUrlArr = process.env.ROOT_URL.split("/"); + homeUrlArr.pop(); + homeUrl = `${homeUrlArr.join("/")}/home`; } - // now we have a role roleName - now we need an email - if (existingUserEmail) { - // existing user - no need to verify as the selection list came from the database - // see if it already has the role - authorization = matsCollections.Authorization.findOne({ - email: existingUserEmail, - }); - roles = authorization.roles; - if (roles.indexOf(roleName) === -1) { - // have to add the role - if (roleName) { - roles.push(roleName); + const settings = { + private: { + databases: [], + PYTHON_PATH: "/usr/bin/python3", + MAPBOX_KEY: mapboxKey, + }, + public: { + run_environment: depEnv, + apps_to_score: appsToScore, + default_group: appDefaultGroup, + default_db: appDefaultDB, + default_model: appDefaultModel, + proxy_prefix_path: "", + home: homeUrl, + appName, + mysql_wait_timeout: appTimeOut, + group: appGroup, + app_order: 1, + title: appTitle, + color: appColor, + threshold_units: thresholdUnits, + }, + }; + writeSettings(settings, appName); // this is going to cause the app to restart in the meteor development environment!!! + // exit for production - probably won't ever get here in development mode (running with meteor) + // This depends on whatever system is running the node process to restart it. + console.log("resetApp - exiting after creating default settings"); + process.exit(1); + } + + // mostly for running locally for debugging. We have to be able to choose the app from the app list in deployment.json + // normally (on a server) it will be an environment variable. + // to debug an integration or production deployment, set the environment variable deployment_environment to one of + // development, integration, production, metexpress + if (Meteor.settings.public && Meteor.settings.public.run_environment) { + depEnv = Meteor.settings.public.run_environment; + } else { + depEnv = process.env.NODE_ENV; + } + // get the mapbox key out of the settings file, if it exists + if (Meteor.settings.private && Meteor.settings.private.MAPBOX_KEY) { + mapboxKey = Meteor.settings.private.MAPBOX_KEY; + } + delete Meteor.settings.public.undefinedRoles; + for (let pi = 0; pi < appPools.length; pi += 1) { + const record = appPools[pi]; + const poolName = record.pool; + // if the database credentials have been set in the meteor.private.settings file then the global[poolName] + // will have been defined in the app main.js. Otherwise it will not have been defined. + // If it is undefined (requiring configuration) we will skip it but add + // the corresponding role to Meteor.settings.public.undefinedRoles - + // which will cause the app to route to the configuration page. + if (!global[poolName]) { + console.log(`resetApp adding ${global[poolName]}to undefined roles`); + // There was no pool defined for this poolName - probably needs to be configured so stash the role in the public settings + if (!Meteor.settings.public.undefinedRoles) { + Meteor.settings.public.undefinedRoles = []; + } + Meteor.settings.public.undefinedRoles.push(record.role); + } else { + try { + if (dbType !== matsTypes.DbTypes.couchbase) { + // default to mysql so that old apps won't break + global[poolName].on("connection", function (connection) { + connection.query("set group_concat_max_len = 4294967295"); + connection.query(`set session wait_timeout = ${appTimeOut}`); + // ("opening new " + poolName + " connection"); + }); } - matsCollections.Authorization.upsert( + // connections all work so make sure that Meteor.settings.public.undefinedRoles is undefined + delete Meteor.settings.public.undefinedRoles; + } catch (e) { + console.log( + `${poolName}: not initialized -= 1 could not open connection: Error:${e.message}` + ); + Meteor.settings.public.undefinedRoles = + Meteor.settings.public.undefinedRoles === undefined + ? [] + : Meteor.settings.public.undefinedRoles === undefined; + Meteor.settings.public.undefinedRoles.push(record.role); + } + } + } + // just in case - should never happen. + if ( + Meteor.settings.public.undefinedRoles && + Meteor.settings.public.undefinedRoles.length > 1 + ) { + throw new Meteor.Error( + `dbpools not initialized ${Meteor.settings.public.undefinedRoles}` + ); + } + + // Try getting version from env + let { version: appVersion, commit, branch } = versionInfo.getVersionsFromEnv(); + if (appVersion === "Unknown") { + // Try getting versionInfo from the appProduction database + console.log("VERSION not set in the environment - using localhost"); + appVersion = "localhost"; + commit = "HEAD"; + branch = "feature"; + } + const appType = type || matsTypes.AppTypes.mats; + + // remember that we updated the metadata tables just now - create metaDataTableUpdates + /* + metaDataTableUpdates: { - email: existingUserEmail, + name: dataBaseName, + tables: [tableName1, tableName2 ..], + lastRefreshed : timestamp + } + */ + // only create metadata tables if the resetApp was called with a real metaDataTables object + if (metaDataTableRecords instanceof matsTypes.MetaDataDBRecord) { + const metaDataTables = metaDataTableRecords.getRecords(); + for (let mdti = 0; mdti < metaDataTables.length; mdti += 1) { + const metaDataRef = metaDataTables[mdti]; + metaDataRef.lastRefreshed = moment().format(); + if (metaDataTableUpdates.find({ name: metaDataRef.name }).count() === 0) { + metaDataTableUpdates.update( + { + name: metaDataRef.name, }, + metaDataRef, { - $set: { - roles, - }, + upsert: true, } ); } - } else if (newUserEmail) { - // possible new authorization - see if it happens to exist - authorization = matsCollections.Authorization.findOne({ - email: newUserEmail, + } + } else { + throw new Meteor.Error("Server error: ", "resetApp: bad pool-database entry"); + } + // invoke the standard common routines + matsCollections.Roles.remove({}); + matsDataUtils.doRoles(); + matsCollections.Authorization.remove({}); + matsDataUtils.doAuthorization(); + matsCollections.Credentials.remove({}); + matsDataUtils.doCredentials(); + matsCollections.PlotGraphFunctions.remove({}); + matsCollections.ColorScheme.remove({}); + matsDataUtils.doColorScheme(); + matsCollections.Settings.remove({}); + matsDataUtils.doSettings( + appTitle, + dbType, + appVersion, + commit, + branch, + appName, + appType, + mapboxKey, + appDefaultGroup, + appDefaultDB, + appDefaultModel, + thresholdUnits, + appMessage, + scorecard + ); + matsCollections.PlotParams.remove({}); + matsCollections.CurveTextPatterns.remove({}); + // get the curve params for this app into their collections + matsCollections.CurveParamsInfo.remove({}); + matsCollections.CurveParamsInfo.insert({ + curve_params: curveParams, + }); + for (let cp = 0; cp < curveParams.length; cp += 1) { + if (matsCollections[curveParams[cp]] !== undefined) { + matsCollections[curveParams[cp]].remove({}); + } + } + // if this is a scorecard also get the apps to score out of the settings file + if (Meteor.settings.public && Meteor.settings.public.scorecard) { + if (Meteor.settings.public.apps_to_score) { + appsToScore = Meteor.settings.public.apps_to_score; + matsCollections.AppsToScore.remove({}); + matsCollections.AppsToScore.insert({ + apps_to_score: appsToScore, }); - if (authorization !== undefined) { - // authorization exists - add role to roles if necessary - roles = authorization.roles; - if (roles.indexOf(roleName) === -1) { - // have to add the role - if (roleName) { - roles.push(roleName); - } - matsCollections.Authorization.upsert( - { - email: existingUserEmail, - }, - { - $set: { - roles, - }, - } - ); - } - } else { - // need a new authorization - roles = []; - if (roleName) { - roles.push(roleName); - } - if (newUserEmail) { - matsCollections.Authorization.upsert( - { - email: newUserEmail, - }, - { - $set: { - roles, - }, - } - ); - } - } + } else { + throw new Meteor.Error( + "apps_to_score are not initialized in app settings -= 1cannot build selectors" + ); } - return false; } - }, -}); + // invoke the app specific routines + for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { + await global.appSpecificResetRoutines[ai](); + } + matsCache.clear(); + } +}; -// database controls -const applyDatabaseSettings = new ValidatedMethod({ - name: "matsMethods.applyDatabaseSettings", +const saveLayout = new ValidatedMethod({ + name: "matsMethods.saveLayout", validate: new SimpleSchema({ - settings: { + resultKey: { + type: String, + }, + layout: { + type: Object, + blackbox: true, + }, + curveOpsUpdate: { type: Object, blackbox: true, }, + annotation: { + type: String, + }, }).validator(), - - run(settings) { + run(params) { if (Meteor.isServer) { - if (settings.name) { - matsCollections.Databases.upsert( + const key = params.resultKey; + const { layout } = params; + const { curveOpsUpdate } = params; + const { annotation } = params; + try { + LayoutStoreCollection.upsert( { - name: settings.name, + key, }, { $set: { - name: settings.name, - role: settings.role, - status: settings.status, - host: settings.host, - database: settings.database, - user: settings.user, - password: settings.password, + createdAt: new Date(), + layout, + curveOpsUpdate, + annotation, }, } ); + } catch (error) { + throw new Meteor.Error( + `Error in saveLayout function:${key} : ${error.message}` + ); } - return false; } }, }); -// administration tools -const deleteSettings = new ValidatedMethod({ - name: "matsMethods.deleteSettings", +const saveScorecardSettings = new ValidatedMethod({ + name: "matsMethods.saveScorecardSettings", validate: new SimpleSchema({ - name: { + settingsKey: { + type: String, + }, + scorecardSettings: { type: String, }, }).validator(), run(params) { - if (!Meteor.userId()) { - throw new Meteor.Error("not-logged-in"); - } if (Meteor.isServer) { - matsCollections.CurveSettings.remove({ - name: params.name, - }); + const key = params.settingsKey; + const { scorecardSettings } = params; + try { + // TODO - remove after tests + console.log( + `saveScorecardSettings(${key}):\n${JSON.stringify( + scorecardSettings, + null, + 2 + )}` + ); + // global cbScorecardSettingsPool + (async function (id, doc) { + cbScorecardSettingsPool.upsertCB(id, doc); + })(key, scorecardSettings).then(() => { + console.log("upserted doc with id", key); + }); + // await cbScorecardSettingsPool.upsertCB(settingsKey, scorecardSettings); + } catch (err) { + console.log(`error writing scorecard to database: ${err.message}`); + } } }, }); -// drop a single instance of a scorecard -const dropScorecardInstance = new ValidatedMethod({ - name: "matsMethods.dropScorecardInstance", +// administration tools +const saveSettings = new ValidatedMethod({ + name: "matsMethods.saveSettings", + validate: new SimpleSchema({ + saveAs: { + type: String, + }, + p: { + type: Object, + blackbox: true, + }, + permission: { + type: String, + }, + }).validator(), + run(params) { + const user = "anonymous"; + matsCollections.CurveSettings.upsert( + { + name: params.saveAs, + }, + { + created: moment().format("MM/DD/YYYY HH:mm:ss"), + name: params.saveAs, + data: params.p, + owner: !Meteor.userId() ? "anonymous" : Meteor.userId(), + permission: params.permission, + savedAt: new Date(), + savedBy: !Meteor.user() ? "anonymous" : user, + } + ); + }, +}); + +/* test methods */ + +const testGetMetaDataTableUpdates = new ValidatedMethod({ + name: "matsMethods.testGetMetaDataTableUpdates", + validate: new SimpleSchema({}).validator(), + run() { + return metaDataTableUpdates.find({}).fetch(); + }, +}); + +const testGetTables = new ValidatedMethod({ + name: "matsMethods.testGetTables", validate: new SimpleSchema({ - userName: { + host: { type: String, }, - name: { + port: { type: String, }, - submittedTime: { + user: { type: String, }, - processedAt: { + password: { + type: String, + }, + database: { type: String, }, }).validator(), - run(params) { + async run(params) { if (Meteor.isServer) { - return _dropScorecardInstance( - params.userName, - params.name, - params.submittedTime, - params.processedAt - ); + if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { + const cbUtilities = new matsCouchbaseUtils.CBUtilities( + params.host, + params.bucket, + params.user, + params.password + ); + try { + const result = await cbUtilities.queryCB("select NOW_MILLIS() as time"); + console.log(`Couchbase get tables suceeded. result: ${result}`); + } catch (err) { + throw new Meteor.Error(err.message); + } + } else { + // default to mysql so that old apps won't break + const Future = require("fibers/future"); + const queryWrap = Future.wrap(function (callback) { + const connection = mysql.createConnection({ + host: params.host, + port: params.port, + user: params.user, + password: params.password, + database: params.database, + }); + connection.query("show tables;", function (err, result) { + if (err || result === undefined) { + // return callback(err,null); + return callback(err, null); + } + const tables = result.map(function (a) { + return a; + }); + + return callback(err, tables); + }); + connection.end(function (err) { + if (err) { + console.log("testGetTables cannot end connection"); + } + }); + }); + try { + return queryWrap().wait(); + } catch (e) { + throw new Meteor.Error(e.message); + } + } } + return null; }, }); -// administration tools -const emailImage = new ValidatedMethod({ - name: "matsMethods.emailImage", - validate: new SimpleSchema({ - imageStr: { - type: String, - }, - toAddress: { - type: String, - }, - subject: { - type: String, - }, - }).validator(), - run(params) { - const { imageStr } = params; - const { toAddress } = params; - const { subject } = params; - if (!Meteor.userId()) { - throw new Meteor.Error(401, "not-logged-in"); - } - const fromAddress = Meteor.user().services.google.email; - // these come from google - see - // http://masashi-k.blogspot.fr/2013/06/sending-mail-with-gmail-using-xoauth2.html - // http://stackoverflow.com/questions/24098461/nodemailer-gmail-what-exactly-is-a-refresh-token-and-how-do-i-get-one/24123550 - - // the gmail account for the credentials is mats.mail.daemon@gmail.com - pwd mats2015! - // var clientId = "339389735380-382sf11aicmgdgn7e72p4end5gnm9sad.apps.googleusercontent.com"; - // var clientSecret = "7CfNN-tRl5QAL595JTW2TkRl"; - // var refresh_token = "1/PDql7FR01N2gmq5NiTfnrT-OlCYC3U67KJYYDNPeGnA"; - const credentials = matsCollections.Credentials.findOne( +const testSetMetaDataTableUpdatesLastRefreshedBack = new ValidatedMethod({ + name: "matsMethods.testSetMetaDataTableUpdatesLastRefreshedBack", + validate: new SimpleSchema({}).validator(), + run() { + const mtu = metaDataTableUpdates.find({}).fetch(); + const id = mtu[0]._id; + metaDataTableUpdates.update( { - name: "oauth_google", + _id: id, }, { - clientId: 1, - clientSecret: 1, - refresh_token: 1, + $set: { + lastRefreshed: 0, + }, } ); - const { clientId } = credentials; - const { clientSecret } = credentials; - const { refresh_token } = credentials; + return metaDataTableUpdates.find({}).fetch(); + }, +}); + +// Define routes for server +if (Meteor.isServer) { + // add indexes to result and axes collections + DownSampleResults.rawCollection().createIndex( + { + createdAt: 1, + }, + { + expireAfterSeconds: 3600 * 8, + } + ); // 8 hour expiration + LayoutStoreCollection.rawCollection().createIndex( + { + createdAt: 1, + }, + { + expireAfterSeconds: 900, + } + ); // 15 min expiration + + // set the default proxy prefix path to "" + // If the settings are not complete, they will be set by the configuration and written out, which will cause the app to reset + if (Meteor.settings.public && !Meteor.settings.public.proxy_prefix_path) { + Meteor.settings.public.proxy_prefix_path = ""; + } + + // eslint-disable-next-line no-unused-vars + Picker.route("/status", function (params, req, res, next) { + Picker.middleware(status(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/status`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(status(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/status`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(status(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getCSV/:key", function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getCSV/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/app:/getCSV/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/CSV/:f/:key/:m/:a", function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/CSV/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/CSV/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getCSV(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getJSON/:key", function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getJSON/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/app:/getJSON/:key`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/JSON/:f/:key/:m/:a", function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/JSON/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/JSON/:f/:key/:m/:a`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getJSON(params, res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/clearCache", function (params, req, res, next) { + Picker.middleware(clearCache(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/clearCache`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(clearCache(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/clearCache`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(clearCache(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getApps", function (params, req, res, next) { + Picker.middleware(getApps(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getApps`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getApps(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getApps`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getApps(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getAppSumsDBs", function (params, req, res, next) { + Picker.middleware(getAppSumsDBs(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getAppSumsDBs`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getAppSumsDBs(res)); + } + ); - let smtpTransporter; - try { - smtpTransporter = Nodemailer.createTransport("SMTP", { - service: "Gmail", - auth: { - XOAuth2: { - user: "mats.gsl@noaa.gov", - clientId, - clientSecret, - refreshToken: refresh_token, - }, - }, - }); - } catch (e) { - throw new Meteor.Error(401, `Transport error ${e.message()}`); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getAppSumsDBs`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getAppSumsDBs(res)); } - try { - const mailOptions = { - sender: fromAddress, - replyTo: fromAddress, - from: fromAddress, - to: toAddress, - subject, - attachments: [ - { - filename: "graph.png", - contents: new Buffer(imageStr.split("base64,")[1], "base64"), - }, - ], - }; + ); - smtpTransporter.sendMail(mailOptions, function (error, response) { - if (error) { - console.log( - `smtpTransporter error ${error} from:${fromAddress} to:${toAddress}` - ); - } else { - console.log(`${response} from:${fromAddress} to:${toAddress}`); - } - smtpTransporter.close(); - }); - } catch (e) { - throw new Meteor.Error(401, `Send error ${e.message()}`); + // eslint-disable-next-line no-unused-vars + Picker.route("/getModels", function (params, req, res, next) { + Picker.middleware(getModels(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getModels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getModels(res)); } - return false; - }, -}); + ); -// administation tool -const getAuthorizations = new ValidatedMethod({ - name: "matsMethods.getAuthorizations", - validate: new SimpleSchema({}).validator(), - run() { - let roles = []; - if (Meteor.isServer) { - const userEmail = Meteor.user().services.google.email.toLowerCase(); - roles = matsCollections.Authorization.findOne({ - email: userEmail, - }).roles; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getModels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getModels(res)); } - return roles; - }, -}); + ); -// administration tool + // eslint-disable-next-line no-unused-vars + Picker.route("/getRegions", function (params, req, res, next) { + Picker.middleware(getRegions(res)); + }); -const getRunEnvironment = new ValidatedMethod({ - name: "matsMethods.getRunEnvironment", - validate: new SimpleSchema({}).validator(), - run() { - return Meteor.settings.public.run_environment; - }, -}); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getRegions`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegions(res)); + } + ); -const getDefaultGroupList = new ValidatedMethod({ - name: "matsMethods.getDefaultGroupList", - validate: new SimpleSchema({}).validator(), - run() { - return matsTypes.DEFAULT_GROUP_LIST; - }, -}); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getRegions`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegions(res)); + } + ); -// retrieves the saved query results (or downsampled results) -const getGraphData = new ValidatedMethod({ - name: "matsMethods.getGraphData", - validate: new SimpleSchema({ - plotParams: { - type: Object, - blackbox: true, - }, - plotType: { - type: String, - }, - expireKey: { - type: Boolean, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const plotGraphFunction = matsCollections.PlotGraphFunctions.findOne({ - plotType: params.plotType, - }); - const { dataFunction } = plotGraphFunction; - let ret; - try { - const hash = require("object-hash"); - const key = hash(params.plotParams); - if (process.env.NODE_ENV === "development" || params.expireKey) { - matsCache.expireKey(key); - } - const results = matsCache.getResult(key); - if (results === undefined) { - // results aren't in the cache - need to process data routine - const Future = require("fibers/future"); - const future = new Future(); - global[dataFunction](params.plotParams, function (results) { - ret = _saveResultData(results); - future.return(ret); - }); - return future.wait(); - } - // results were already in the matsCache (same params and not yet expired) - // are results in the downsampled collection? - const dsResults = DownSampleResults.findOne( - { - key, - }, - {}, - { - disableOplog: true, - } - ); - if (dsResults !== undefined) { - // results are in the mongo cache downsampled collection - returned the downsampled graph data - ret = dsResults; - // update the expire time in the downsampled collection - this requires a new Date - DownSampleResults.rawCollection().update( - { - key, - }, - { - $set: { - createdAt: new Date(), - }, - } - ); - } else { - ret = results; // {key:someKey, result:resultObject} - // refresh expire time. The only way to perform a refresh on matsCache is to re-save the result. - matsCache.storeResult(results.key, results); - } - const sizeof = require("object-sizeof"); - console.log("result.data size is ", sizeof(results)); - return ret; - } catch (dataFunctionError) { - if (dataFunctionError.toLocaleString().indexOf("INFO:") !== -1) { - throw new Meteor.Error(dataFunctionError.message); - } else { - throw new Meteor.Error( - `Error in getGraphData function:${dataFunction} : ${dataFunctionError.message}` - ); - } - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getRegionsValuesMap", function (params, req, res, next) { + Picker.middleware(getRegionsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getRegionsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegionsValuesMap(res)); } - }, -}); + ); -// retrieves the saved query results (or downsampled results) for a specific key -const getGraphDataByKey = new ValidatedMethod({ - name: "matsMethods.getGraphDataByKey", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - let ret; - const key = params.resultKey; - try { - const dsResults = DownSampleResults.findOne( - { - key, - }, - {}, - { - disableOplog: true, - } - ); - if (dsResults !== undefined) { - ret = dsResults; - } else { - ret = matsCache.getResult(key); // {key:someKey, result:resultObject} - } - const sizeof = require("object-sizeof"); - console.log("getGraphDataByKey results size is ", sizeof(dsResults)); - return ret; - } catch (error) { - throw new Meteor.Error( - `Error in getGraphDataByKey function:${key} : ${error.message}` - ); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getRegionsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getRegionsValuesMap(res)); } - }, -}); + ); -const getLayout = new ValidatedMethod({ - name: "matsMethods.getLayout", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - let ret; - const key = params.resultKey; - try { - ret = LayoutStoreCollection.rawCollection().findOne({ - key, - }); - return ret; - } catch (error) { - throw new Meteor.Error(`Error in getLayout function:${key} : ${error.message}`); - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getStatistics", function (params, req, res, next) { + Picker.middleware(getStatistics(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getStatistics`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatistics(res)); } - }, -}); + ); -const getScorecardSettings = new ValidatedMethod({ - name: "matsMethods.getScorecardSettings", - validate: new SimpleSchema({ - settingsKey: { - type: String, - }, - }).validator(), - async run(params) { - if (Meteor.isServer) { - const key = params.settingsKey; - try { - // global cbScorecardSettingsPool - const rv = await cbScorecardSettingsPool.getCB(key); - return { scorecardSettings: rv.content }; - } catch (error) { - throw new Meteor.Error( - `Error in getScorecardSettings function:${key} : ${error.message}` - ); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getStatistics`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatistics(res)); } - }, -}); + ); -const getPlotParamsFromScorecardInstance = new ValidatedMethod({ - name: "matsMethods.getPlotParamsFromScorecardInstance", - validate: new SimpleSchema({ - userName: { - type: String, - }, - name: { - type: String, - }, - submitted: { - type: String, - }, - processedAt: { - type: String, - }, - }).validator(), - run(params) { - try { - if (Meteor.isServer) { - return _getPlotParamsFromScorecardInstance( - params.userName, - params.name, - params.submitted, - params.processedAt - ); - } - } catch (error) { - throw new Meteor.Error( - `Error in getPlotParamsFromScorecardInstance function:${error.message}` - ); + // eslint-disable-next-line no-unused-vars + Picker.route("/getStatisticsValuesMap", function (params, req, res, next) { + Picker.middleware(getStatisticsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getStatisticsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatisticsValuesMap(res)); } - }, -}); + ); -/* -getPlotResult is used by the graph/text_*_output templates which are used to display textual results. -Because the data isn't being rendered graphically this data is always full size, i.e. NOT downsampled. -That is why it only finds it in the Result file cache, never the DownSampleResult collection. + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getStatisticsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getStatisticsValuesMap(res)); + } + ); -Because the dataset can be so large ... e.g. megabytes the data retrieval is pagenated. The index is -applied to the underlying datasets.The data gets stripped down and flattened to only contain the data neccesary for text presentation. -A new page index of -1000 means get all the data i.e. no pagenation. - */ -const getPlotResult = new ValidatedMethod({ - name: "matsMethods.getPlotResult", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - pageIndex: { - type: Number, - }, - newPageIndex: { - type: Number, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const rKey = params.resultKey; - const pi = params.pageIndex; - const npi = params.newPageIndex; - let ret = {}; - try { - ret = _getFlattenedResultData(rKey, pi, npi); - } catch (e) { - console.log(e); - } - return ret; + // eslint-disable-next-line no-unused-vars + Picker.route("/getVariables", function (params, req, res, next) { + Picker.middleware(getVariables(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getVariables`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariables(res)); } - }, -}); + ); -const getReleaseNotes = new ValidatedMethod({ - name: "matsMethods.getReleaseNotes", - validate: new SimpleSchema({}).validator(), - run() { - // return Assets.getText('public/MATSReleaseNotes.html'); - // } - if (Meteor.isServer) { - const future = require("fibers/future"); - const fse = require("fs-extra"); - const dFuture = new future(); - let fData; - let file; - if (process.env.NODE_ENV === "development") { - file = `${process.env.PWD}/.meteor/local/build/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; - } else { - file = `${process.env.PWD}/programs/server/assets/packages/randyp_mats-common/public/MATSReleaseNotes.html`; - } - try { - fse.readFile(file, "utf8", function (err, data) { - if (err) { - fData = err.message; - dFuture.return(); - } else { - fData = data; - dFuture.return(); - } - }); - } catch (e) { - fData = e.message; - dFuture.return(); - } - dFuture.wait(); - return fData; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getVariables`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariables(res)); } - }, -}); + ); -const setCurveParamDisplayText = new ValidatedMethod({ - name: "matsMethods.setCurveParamDisplayText", - validate: new SimpleSchema({ - paramName: { - type: String, - }, - newText: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - return matsCollections[params.paramName].update( - { name: params.paramName }, - { $set: { controlButtonText: params.newText } } - ); + // eslint-disable-next-line no-unused-vars + Picker.route("/getVariablesValuesMap", function (params, req, res, next) { + Picker.middleware(getVariablesValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getVariablesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariablesValuesMap(res)); } - }, -}); + ); -const getScorecardData = new ValidatedMethod({ - name: "matsMethods.getScorecardData", - validate: new SimpleSchema({ - userName: { - type: String, - }, - name: { - type: String, - }, - submitted: { - type: String, - }, - processedAt: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - return _getScorecardData( - params.userName, - params.name, - params.submitted, - params.processedAt - ); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getVariablesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getVariablesValuesMap(res)); } - }, -}); + ); -const getScorecardInfo = new ValidatedMethod({ - name: "matsMethods.getScorecardInfo", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - return _getScorecardInfo(); + // eslint-disable-next-line no-unused-vars + Picker.route("/getThresholds", function (params, req, res, next) { + Picker.middleware(getThresholds(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getThresholds`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholds(res)); } - }, -}); + ); -// administration tool -const getUserAddress = new ValidatedMethod({ - name: "matsMethods.getUserAddress", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - return Meteor.user().services.google.email.toLowerCase(); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholds`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholds(res)); } - }, -}); + ); -// app utility -const insertColor = new ValidatedMethod({ - name: "matsMethods.insertColor", - validate: new SimpleSchema({ - newColor: { - type: String, - }, - insertAfterIndex: { - type: Number, - }, - }).validator(), - run(params) { - if (params.newColor === "rgb(255,255,255)") { - return false; + // eslint-disable-next-line no-unused-vars + Picker.route("/getThresholdsValuesMap", function (params, req, res, next) { + Picker.middleware(getThresholdsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getThresholdsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholdsValuesMap(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getThresholdsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getThresholdsValuesMap(res)); } - const colorScheme = matsCollections.ColorScheme.findOne({}); - colorScheme.colors.splice(params.insertAfterIndex, 0, newColor); - matsCollections.update({}, colorScheme); - return false; - }, -}); + ); -// administration tool -const readFunctionFile = new ValidatedMethod({ - name: "matsMethods.readFunctionFile", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - const future = require("fibers/future"); - const fse = require("fs-extra"); - let path = ""; - let fData; - if (type === "data") { - path = `/web/static/dataFunctions/${file}`; - console.log(`exporting data file: ${path}`); - } else if (type === "graph") { - path = `/web/static/displayFunctions/${file}`; - console.log(`exporting graph file: ${path}`); - } else { - return "error - wrong type"; - } - fse.readFile(path, function (err, data) { - if (err) throw err; - fData = data.toString(); - future.return(fData); - }); - return future.wait(); + // eslint-disable-next-line no-unused-vars + Picker.route("/getScales", function (params, req, res, next) { + Picker.middleware(getScales(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getScales`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScales(res)); } - }, -}); + ); -// refreshes the metadata for the app that's running -const refreshMetaData = new ValidatedMethod({ - name: "matsMethods.refreshMetaData", - validate: new SimpleSchema({}).validator(), - run() { - if (Meteor.isServer) { - try { - // console.log("GUI asked to refresh metadata"); - _checkMetaDataRefresh(); - } catch (e) { - console.log(e); - throw new Meteor.Error("Server error: ", e.message); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getScales`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScales(res)); } - return metaDataTableUpdates.find({}).fetch(); - }, -}); + ); -// administation tool -const removeAuthorization = new ValidatedMethod({ - name: "matsMethods.removeAuthorization", - validate: new SimpleSchema({ - settings: { - type: Object, - blackbox: true, - }, - }).validator(), - run(settings) { - if (Meteor.isServer) { - let email; - let roleName; - const { userRoleName } = settings; - const { authorizationRole } = settings; - const { newUserEmail } = settings; - const { existingUserEmail } = settings; - if (authorizationRole) { - // existing role - the role roleName - no need to verify as the selection list came from the database - roleName = authorizationRole; - } else if (userRoleName) { - roleName = userRoleName; - } - if (existingUserEmail) { - email = existingUserEmail; - } else { - email = newUserEmail; - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getScalesValuesMap", function (params, req, res, next) { + Picker.middleware(getScalesValuesMap(res)); + }); - // if user and role remove the role from the user - if (email && roleName) { - matsCollections.Authorization.update( - { - email, - }, - { - $pull: { - roles: roleName, - }, - } - ); - } - // if user and no role remove the user - if (email && !roleName) { - matsCollections.Authorization.remove({ - email, - }); - } - // if role and no user remove role and remove role from all users - if (roleName && !email) { - // remove the role - matsCollections.Roles.remove({ - name: roleName, - }); - // remove the roleName role from all the authorizations - matsCollections.Authorization.update( - { - roles: roleName, - }, - { - $pull: { - roles: roleName, - }, - }, - { - multi: true, - } - ); - } - return false; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getScalesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScalesValuesMap(res)); } - }, -}); - -// app utility -const removeColor = new ValidatedMethod({ - name: "matsMethods.removeColor", - validate: new SimpleSchema({ - removeColor: { - type: String, - }, - }).validator(), - run(removeColor) { - const colorScheme = matsCollections.ColorScheme.findOne({}); - const removeIndex = colorScheme.colors.indexOf(removeColor); - colorScheme.colors.splice(removeIndex, 1); - matsCollections.ColorScheme.update({}, colorScheme); - return false; - }, -}); + ); -// database controls -const removeDatabase = new ValidatedMethod({ - name: "matsMethods.removeDatabase", - validate: new SimpleSchema({ - dbName: { - type: String, - }, - }).validator(), - run(dbName) { - if (Meteor.isServer) { - matsCollections.Databases.remove({ - name: dbName, - }); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getScalesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getScalesValuesMap(res)); } - }, -}); + ); -const applySettingsData = new ValidatedMethod({ - name: "matsMethods.applySettingsData", - validate: new SimpleSchema({ - settings: { - type: Object, - blackbox: true, - }, - }).validator(), - // this method forces a restart on purpose. We do not want retries - applyOptions: { - noRetry: true, - }, - run(settingsParam) { - if (Meteor.isServer) { - // Read the existing settings file - const { settings } = settingsParam; - console.log( - "applySettingsData - matsCollections.appName.findOne({}) is ", - matsCollections.appName.findOne({}) - ); - const { appName } = matsCollections.Settings.findOne({}); - _write_settings(settings, appName); - // in development - when being run by meteor, this should force a restart of the app. - // in case I am in a container - exit and force a reload - console.log( - `applySettingsData - process.env.NODE_ENV is: ${process.env.NODE_ENV}` - ); - if (process.env.NODE_ENV !== "development") { - console.log("applySettingsData - exiting after writing new Settings"); - process.exit(0); - } + // eslint-disable-next-line no-unused-vars + Picker.route("/getTruths", function (params, req, res, next) { + Picker.middleware(getTruths(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getTruths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruths(res)); } - }, -}); + ); -// makes sure all of the parameters display appropriate selections in relation to one another -// for default settings ... -const resetApp = async function (appRef) { - if (Meteor.isServer) { - const fse = require("fs-extra"); - const metaDataTableRecords = appRef.appMdr; - const { appPools } = appRef; - const type = appRef.appType; - const scorecard = Meteor.settings.public.scorecard - ? Meteor.settings.public.scorecard - : false; - const dbType = appRef.dbType ? appRef.dbType : matsTypes.DbTypes.mysql; - const appName = Meteor.settings.public.app ? Meteor.settings.public.app : "unnamed"; - const appTitle = Meteor.settings.public.title - ? Meteor.settings.public.title - : "Unnamed App"; - const appGroup = Meteor.settings.public.group - ? Meteor.settings.public.group - : "Misc. Apps"; - const thresholdUnits = Meteor.settings.public.threshold_units - ? Meteor.settings.public.threshold_units - : {}; - let appDefaultGroup = ""; - let appDefaultDB = ""; - let appDefaultModel = ""; - let appColor; - switch (type) { - case matsTypes.AppTypes.mats: - if (dbType === matsTypes.DbTypes.couchbase) { - appColor = "#33abbb"; - } else { - appColor = Meteor.settings.public.color - ? Meteor.settings.public.color - : "#3366bb"; - } - break; - case matsTypes.AppTypes.metexpress: - appColor = Meteor.settings.public.color - ? Meteor.settings.public.color - : "darkorchid"; - appDefaultGroup = Meteor.settings.public.default_group - ? Meteor.settings.public.default_group - : "NO GROUP"; - appDefaultDB = Meteor.settings.public.default_db - ? Meteor.settings.public.default_db - : "mv_default"; - appDefaultModel = Meteor.settings.public.default_model - ? Meteor.settings.public.default_model - : "Default"; - break; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getTruths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruths(res)); } - const appTimeOut = Meteor.settings.public.mysql_wait_timeout - ? Meteor.settings.public.mysql_wait_timeout - : 300; - let dep_env = process.env.NODE_ENV; - const curve_params = curveParamsByApp[Meteor.settings.public.app]; - let apps_to_score; - if (Meteor.settings.public.scorecard) { - apps_to_score = Meteor.settings.public.apps_to_score - ? Meteor.settings.public.apps_to_score - : []; + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getTruthsValuesMap", function (params, req, res, next) { + Picker.middleware(getTruthsValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getTruthsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruthsValuesMap(res)); } - let mapboxKey = "undefined"; + ); - // see if there's any messages to display to the users - const appMessage = Meteor.settings.public.alert_message - ? Meteor.settings.public.alert_message - : undefined; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getTruthsValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getTruthsValuesMap(res)); + } + ); - // set meteor settings defaults if they do not exist - if (isEmpty(Meteor.settings.private) || isEmpty(Meteor.settings.public)) { - // create some default meteor settings and write them out - let homeUrl = ""; - if (!process.env.ROOT_URL) { - homeUrl = "https://localhost/home"; - } else { - const homeUrlArr = process.env.ROOT_URL.split("/"); - homeUrlArr.pop(); - homeUrl = `${homeUrlArr.join("/")}/home`; - } - const settings = { - private: { - databases: [], - PYTHON_PATH: "/usr/bin/python3", - MAPBOX_KEY: mapboxKey, - }, - public: { - run_environment: dep_env, - apps_to_score, - default_group: appDefaultGroup, - default_db: appDefaultDB, - default_model: appDefaultModel, - proxy_prefix_path: "", - home: homeUrl, - appName, - mysql_wait_timeout: appTimeOut, - group: appGroup, - app_order: 1, - title: appTitle, - color: appColor, - threshold_units: thresholdUnits, - }, - }; - _write_settings(settings, appName); // this is going to cause the app to restart in the meteor development environment!!! - // exit for production - probably won't ever get here in development mode (running with meteor) - // This depends on whatever system is running the node process to restart it. - console.log("resetApp - exiting after creating default settings"); - process.exit(1); + // eslint-disable-next-line no-unused-vars + Picker.route("/getFcstLengths", function (params, req, res, next) { + Picker.middleware(getFcstLengths(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getFcstLengths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstLengths(res)); } + ); - // mostly for running locally for debugging. We have to be able to choose the app from the app list in deployment.json - // normally (on a server) it will be an environment variable. - // to debug an integration or production deployment, set the environment variable deployment_environment to one of - // development, integration, production, metexpress - if (Meteor.settings.public && Meteor.settings.public.run_environment) { - dep_env = Meteor.settings.public.run_environment; - } else { - dep_env = process.env.NODE_ENV; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstLengths`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstLengths(res)); } - // get the mapbox key out of the settings file, if it exists - if (Meteor.settings.private && Meteor.settings.private.MAPBOX_KEY) { - mapboxKey = Meteor.settings.private.MAPBOX_KEY; + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getFcstTypes", function (params, req, res, next) { + Picker.middleware(getFcstTypes(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getFcstTypes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypes(res)); } - delete Meteor.settings.public.undefinedRoles; - for (let pi = 0; pi < appPools.length; pi++) { - const record = appPools[pi]; - const poolName = record.pool; - // if the database credentials have been set in the meteor.private.settings file then the global[poolName] - // will have been defined in the app main.js. Otherwise it will not have been defined. - // If it is undefined (requiring configuration) we will skip it but add - // the corresponding role to Meteor.settings.public.undefinedRoles - - // which will cause the app to route to the configuration page. - if (!global[poolName]) { - console.log(`resetApp adding ${global[poolName]}to undefined roles`); - // There was no pool defined for this poolName - probably needs to be configured so stash the role in the public settings - if (!Meteor.settings.public.undefinedRoles) { - Meteor.settings.public.undefinedRoles = []; - } - Meteor.settings.public.undefinedRoles.push(record.role); - continue; - } - try { - if (dbType === matsTypes.DbTypes.couchbase) { - // simple couchbase test - const time = await cbPool.queryCB("select NOW_MILLIS() as time;"); - break; - } else { - // default to mysql so that old apps won't break - global[poolName].on("connection", function (connection) { - connection.query("set group_concat_max_len = 4294967295"); - connection.query(`set session wait_timeout = ${appTimeOut}`); - // ("opening new " + poolName + " connection"); - }); - } - } catch (e) { - console.log( - `${poolName}: not initialized-- could not open connection: Error:${e.message}` - ); - Meteor.settings.public.undefinedRoles = - Meteor.settings.public.undefinedRoles === undefined - ? [] - : Meteor.settings.public.undefinedRoles === undefined; - Meteor.settings.public.undefinedRoles.push(record.role); - continue; - } - // connections all work so make sure that Meteor.settings.public.undefinedRoles is undefined - delete Meteor.settings.public.undefinedRoles; + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypes(res)); } - // just in case - should never happen. - if ( - Meteor.settings.public.undefinedRoles && - Meteor.settings.public.undefinedRoles.length > 1 - ) { - throw new Meteor.Error( - `dbpools not initialized ${Meteor.settings.public.undefinedRoles}` - ); + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getFcstTypesValuesMap", function (params, req, res, next) { + Picker.middleware(getFcstTypesValuesMap(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getFcstTypesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypesValuesMap(res)); } + ); - // Try getting version from env - let { version: appVersion, commit, branch } = versionInfo.getVersionsFromEnv(); - if (appVersion === "Unknown") { - // Try getting versionInfo from the appProduction database - console.log("VERSION not set in the environment - using localhost"); - appVersion = "localhost"; - commit = "HEAD"; + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getFcstTypesValuesMap`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getFcstTypesValuesMap(res)); } - const appType = type || matsTypes.AppTypes.mats; + ); - // remember that we updated the metadata tables just now - create metaDataTableUpdates - /* - metaDataTableUpdates: - { - name: dataBaseName, - tables: [tableName1, tableName2 ..], - lastRefreshed : timestamp - } - */ - // only create metadata tables if the resetApp was called with a real metaDataTables object - if (metaDataTableRecords instanceof matsTypes.MetaDataDBRecord) { - const metaDataTables = metaDataTableRecords.getRecords(); - for (let mdti = 0; mdti < metaDataTables.length; mdti++) { - const metaDataRef = metaDataTables[mdti]; - metaDataRef.lastRefreshed = moment().format(); - if (metaDataTableUpdates.find({ name: metaDataRef.name }).count() === 0) { - metaDataTableUpdates.update( - { - name: metaDataRef.name, - }, - metaDataRef, - { - upsert: true, - } - ); - } - } - } else { - throw new Meteor.Error("Server error: ", "resetApp: bad pool-database entry"); + // eslint-disable-next-line no-unused-vars + Picker.route("/getValidTimes", function (params, req, res, next) { + Picker.middleware(getValidTimes(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getValidTimes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getValidTimes(res)); } - // invoke the standard common routines - matsCollections.Roles.remove({}); - matsDataUtils.doRoles(); - matsCollections.Authorization.remove({}); - matsDataUtils.doAuthorization(); - matsCollections.Credentials.remove({}); - matsDataUtils.doCredentials(); - matsCollections.PlotGraphFunctions.remove({}); - matsCollections.ColorScheme.remove({}); - matsDataUtils.doColorScheme(); - matsCollections.Settings.remove({}); - matsDataUtils.doSettings( - appTitle, - dbType, - appVersion, - commit, - appName, - appType, - mapboxKey, - appDefaultGroup, - appDefaultDB, - appDefaultModel, - thresholdUnits, - appMessage, - scorecard - ); - matsCollections.PlotParams.remove({}); - matsCollections.CurveTextPatterns.remove({}); - // get the curve params for this app into their collections - matsCollections.CurveParamsInfo.remove({}); - matsCollections.CurveParamsInfo.insert({ - curve_params, - }); - for (let cp = 0; cp < curve_params.length; cp++) { - if (matsCollections[curve_params[cp]] !== undefined) { - matsCollections[curve_params[cp]].remove({}); - } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getValidTimes`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getValidTimes(res)); } - // if this is a scorecard also get the apps to score out of the settings file - if (Meteor.settings.public && Meteor.settings.public.scorecard) { - if (Meteor.settings.public.apps_to_score) { - apps_to_score = Meteor.settings.public.apps_to_score; - matsCollections.AppsToScore.remove({}); - matsCollections.AppsToScore.insert({ - apps_to_score, - }); - } else { - throw new Meteor.Error( - "apps_to_score are not initialized in app settings--cannot build selectors" - ); - } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getLevels", function (params, req, res, next) { + Picker.middleware(getLevels(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getLevels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getLevels(res)); + } + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getLevels`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getLevels(res)); + } + ); + + // eslint-disable-next-line no-unused-vars + Picker.route("/getDates", function (params, req, res, next) { + Picker.middleware(getDates(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/getDates`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getDates(res)); } - // invoke the app specific routines - for (let ai = 0; ai < appSpecificResetRoutines.length; ai += 1) { - await global.appSpecificResetRoutines[ai](); + ); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/getDates`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(getDates(res)); } - matsCache.clear(); - } -}; + ); -const saveLayout = new ValidatedMethod({ - name: "matsMethods.saveLayout", - validate: new SimpleSchema({ - resultKey: { - type: String, - }, - layout: { - type: Object, - blackbox: true, - }, - curveOpsUpdate: { - type: Object, - blackbox: true, - }, - annotation: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const key = params.resultKey; - const { layout } = params; - const { curveOpsUpdate } = params; - const { annotation } = params; - try { - LayoutStoreCollection.upsert( - { - key, - }, - { - $set: { - createdAt: new Date(), - layout, - curveOpsUpdate, - annotation, - }, - } - ); - } catch (error) { - throw new Meteor.Error( - `Error in saveLayout function:${key} : ${error.message}` - ); - } + // create picker routes for refreshMetaData + // eslint-disable-next-line no-unused-vars + Picker.route("/refreshMetadata", function (params, req, res, next) { + Picker.middleware(refreshMetadataMWltData(res)); + }); + + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/refreshMetadata`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshMetadataMWltData(res)); } - }, -}); + ); -const saveScorecardSettings = new ValidatedMethod({ - name: "matsMethods.saveScorecardSettings", - validate: new SimpleSchema({ - settingsKey: { - type: String, - }, - scorecardSettings: { - type: String, - }, - }).validator(), - run(params) { - if (Meteor.isServer) { - const key = params.settingsKey; - const { scorecardSettings } = params; - try { - // TODO - remove after tests - console.log( - `saveScorecardSettings(${key}):\n${JSON.stringify( - scorecardSettings, - null, - 2 - )}` - ); - // global cbScorecardSettingsPool - (async function (id, doc) { - cbScorecardSettingsPool.upsertCB(id, doc); - })(key, scorecardSettings).then(() => { - console.log("upserted doc with id", key); - }); - // await cbScorecardSettingsPool.upsertCB(settingsKey, scorecardSettings); - } catch (err) { - console.log(`error writing scorecard to database: ${err.message}`); - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/refreshMetadata`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshMetadataMWltData(res)); } - }, -}); + ); + // eslint-disable-next-line no-unused-vars + Picker.route("/refreshScorecard/:docId", function (params, req, res, next) { + Picker.middleware(refreshScorecard(params, res)); + }); -// administration tools -const saveSettings = new ValidatedMethod({ - name: "matsMethods.saveSettings", - validate: new SimpleSchema({ - saveAs: { - type: String, - }, - p: { - type: Object, - blackbox: true, - }, - permission: { - type: String, - }, - }).validator(), - run(params) { - const user = "anonymous"; - matsCollections.CurveSettings.upsert( - { - name: params.saveAs, - }, - { - created: moment().format("MM/DD/YYYY HH:mm:ss"), - name: params.saveAs, - data: params.p, - owner: !Meteor.userId() ? "anonymous" : Meteor.userId(), - permission: params.permission, - savedAt: new Date(), - savedBy: !Meteor.user() ? "anonymous" : user, - } - ); - }, -}); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/refreshScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshScorecard(params, res)); + } + ); -/* test methods */ + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/refreshScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(refreshScorecard(params, res)); + } + ); -const testGetMetaDataTableUpdates = new ValidatedMethod({ - name: "matsMethods.testGetMetaDataTableUpdates", - validate: new SimpleSchema({}).validator(), - run() { - return metaDataTableUpdates.find({}).fetch(); - }, -}); + // eslint-disable-next-line no-unused-vars + Picker.route("/setStatusScorecard/:docId", function (params, req, res, next) { + Picker.middleware(setStatusScorecard(params, req, res)); + }); -const testGetTables = new ValidatedMethod({ - name: "matsMethods.testGetTables", - validate: new SimpleSchema({ - host: { - type: String, - }, - port: { - type: String, - }, - user: { - type: String, - }, - password: { - type: String, - }, - database: { - type: String, - }, - }).validator(), - async run(params) { - if (Meteor.isServer) { - if (matsCollections.Settings.findOne().dbType === matsTypes.DbTypes.couchbase) { - const cbUtilities = new matsCouchbaseUtils.CBUtilities( - params.host, - params.bucket, - params.user, - params.password - ); - try { - const result = await cbUtilities.queryCB("select NOW_MILLIS() as time"); - } catch (err) { - throw new Meteor.Error(e.message); - } - } else { - // default to mysql so that old apps won't break - const Future = require("fibers/future"); - const queryWrap = Future.wrap(function (callback) { - const connection = mysql.createConnection({ - host: params.host, - port: params.port, - user: params.user, - password: params.password, - database: params.database, - }); - connection.query("show tables;", function (err, result) { - if (err || result === undefined) { - // return callback(err,null); - return callback(err, null); - } - const tables = result.map(function (a) { - return a; - }); + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/setStatusScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(setStatusScorecard(params, req, res)); + } + ); - return callback(err, tables); - }); - connection.end(function (err) { - if (err) { - console.log("testGetTables cannot end connection"); - } - }); - }); - try { - return queryWrap().wait(); - } catch (e) { - throw new Meteor.Error(e.message); - } - } + Picker.route( + `${Meteor.settings.public.proxy_prefix_path}/:app/setStatusScorecard/:docId`, + // eslint-disable-next-line no-unused-vars + function (params, req, res, next) { + Picker.middleware(setStatusScorecard(params, req, res)); } - }, -}); + ); +} -const testSetMetaDataTableUpdatesLastRefreshedBack = new ValidatedMethod({ - name: "matsMethods.testSetMetaDataTableUpdatesLastRefreshedBack", - validate: new SimpleSchema({}).validator(), - run() { - const mtu = metaDataTableUpdates.find({}).fetch(); - const id = mtu[0]._id; - metaDataTableUpdates.update( - { - _id: id, - }, - { - $set: { - lastRefreshed: 0, - }, - } - ); - return metaDataTableUpdates.find({}).fetch(); - }, -}); +// eslint-disable-next-line no-undef export default matsMethods = { addSentAddress, applyAuthorization, diff --git a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js index fbbf1917d..4ef5dde68 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js @@ -2,7 +2,7 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsCollections, matsTypes } from "meteor/randyp:mats-common"; +import { matsCollections, matsTypes, matsDataUtils } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; import { moment } from "meteor/momentjs:moment"; import { _ } from "meteor/underscore"; @@ -1497,7 +1497,7 @@ const generateGridScaleProbPlotOptions = function (axisMap) { const yPad = (ymax - ymin) * 0.025 !== 0 ? (ymax - ymin) * 0.025 : 0.025; const newYmax = Math.log10(ymax + yPad * 100); const newYmin = - Number.isNaN(Math.log10(ymin - yPad)) || Math.log10(ymin - yPad) < 1 + matsDataUtils.isThisANaN(Math.log10(ymin - yPad)) || Math.log10(ymin - yPad) < 1 ? 0 : Math.log10(ymin - yPad); layout.yaxis.range = [newYmin, newYmax]; diff --git a/meteor_packages/mats-common/imports/startup/server/data_process_util.js b/meteor_packages/mats-common/imports/startup/server/data_process_util.js index 13906a9f0..7382394f7 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_process_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_process_util.js @@ -125,11 +125,11 @@ const processDataXYCurve = function ( diffFrom === null || !( Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[0]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[1]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_y.array[di] = null; @@ -206,46 +206,46 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.y[di] - errorLength).toPrecision(4)} to ${Number( @@ -255,27 +255,27 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.y[di], nGood: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; @@ -656,11 +656,11 @@ const processDataProfile = function ( diffFrom === null || !( Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[0]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || - !Number.isNaN(returnDataset[diffFrom[1]].subHit[di]) + !matsDataUtils.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_x.array[di] = null; @@ -696,23 +696,23 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; @@ -721,23 +721,23 @@ const processDataProfile = function ( `
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !Number.isNaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !Number.isNaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !Number.isNaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !Number.isNaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.x[di] - errorLength).toPrecision( @@ -747,27 +747,27 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.x[di], nGood: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !Number.isNaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; diff --git a/meteor_packages/mats-common/imports/startup/server/data_query_util.js b/meteor_packages/mats-common/imports/startup/server/data_query_util.js index 29a955763..061270e15 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_query_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_query_util.js @@ -391,7 +391,7 @@ const parseQueryDataXYCurve = function ( const n = rows[rowIndex].sub_data.toString().split(",").length; if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -417,7 +417,7 @@ const parseQueryDataXYCurve = function ( absSum, statisticStr ); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -470,7 +470,7 @@ const parseQueryDataXYCurve = function ( if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -508,7 +508,7 @@ const parseQueryDataXYCurve = function ( } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -552,7 +552,7 @@ const parseQueryDataXYCurve = function ( } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1024,7 +1024,7 @@ const parseQueryDataReliability = function (rows, d, appParams, kernel) { currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1194,7 +1194,7 @@ const parseQueryDataPerformanceDiagram = function (rows, d, appParams) { currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1379,7 +1379,7 @@ const parseQueryDataSimpleScatter = function ( absSumX, statisticXStr ); - xStat = Number.isNaN(Number(xStat)) ? null : xStat; + xStat = matsDataUtils.isThisANaN(Number(xStat)) ? null : xStat; yStat = matsDataUtils.calculateStatScalar( squareDiffSumY, NSumY, @@ -1389,7 +1389,7 @@ const parseQueryDataSimpleScatter = function ( absSumY, statisticYStr ); - yStat = Number.isNaN(Number(yStat)) ? null : yStat; + yStat = matsDataUtils.isThisANaN(Number(yStat)) ? null : yStat; } else { xStat = null; yStat = null; @@ -1429,7 +1429,7 @@ const parseQueryDataSimpleScatter = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1722,7 +1722,7 @@ const parseQueryDataMapScalar = function ( absSum, `${statistic}_${variable}` ); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; } else { queryVal = null; } @@ -1752,7 +1752,7 @@ const parseQueryDataMapScalar = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1830,7 +1830,7 @@ const parseQueryDataMapScalar = function ( absSumSum, `${statistic}_${variable}` ); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database e.message = `Error in parseQueryDataMapScalar. The expected fields don't seem to be present in the results cache: ${e.message}`; @@ -2007,7 +2007,7 @@ const parseQueryDataMapCTC = function ( const n = rows[rowIndex].nTimes; if (hit + fa + miss + cn > 0) { queryVal = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statistic); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; switch (statistic) { case "PODy (POD of value < threshold)": case "PODy (POD of value > threshold)": @@ -2068,7 +2068,7 @@ const parseQueryDataMapCTC = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2135,7 +2135,7 @@ const parseQueryDataMapCTC = function ( thisSubHit.length, statistic ); - queryVal = Number.isNaN(Number(queryVal)) ? null : queryVal; + queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database e.message = `Error in parseQueryDataMapCTC. The expected fields don't seem to be present in the results cache: ${e.message}`; @@ -2325,7 +2325,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { const n = rows[rowIndex].sub_data.toString().split(",").length; if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -2351,7 +2351,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { absSum, statisticStr ); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -2376,7 +2376,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2406,7 +2406,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2438,7 +2438,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2590,7 +2590,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { cn = Number(rows[rowIndex].cn); if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; } } else if ( rows[rowIndex].stat === undefined && @@ -2614,7 +2614,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { absSum, statisticStr ); - stat = Number.isNaN(Number(stat)) ? null : stat; + stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; const variable = statisticStr.split("_")[1]; stdev = matsDataUtils.calculateStatScalar( squareDiffSum, @@ -2667,7 +2667,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2685,7 +2685,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2707,7 +2707,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!Number.isNaN(Number(currSubData[1]))) { + if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); diff --git a/meteor_packages/mats-common/imports/startup/server/data_util.js b/meteor_packages/mats-common/imports/startup/server/data_util.js index 9500a50a9..31a63e32f 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_util.js @@ -9,6 +9,12 @@ import { HTTP } from "meteor/jkuester:http"; /* eslint-disable global-require */ /* eslint-disable no-console */ +// wrapper for NaN check +const isThisANaN = function (val) { + // eslint-disable-next-line no-restricted-globals + return !val || isNaN(val); +}; + // this function checks if two JSON objects are identical const areObjectsEqual = function (o, p) { if ((o && !p) || (p && !o)) { @@ -348,6 +354,7 @@ const doSettings = function ( dbType, version, commit, + branch, appName, appType, mapboxKey, @@ -458,7 +465,7 @@ const callMetadataAPI = function ( // calculates the statistic for ctc plots const calculateStatCTC = function (hit, fa, miss, cn, n, statistic) { - if (Number.isNaN(hit) || Number.isNaN(fa) || Number.isNaN(miss) || Number.isNaN(cn)) + if (isThisANaN(hit) || isThisANaN(fa) || isThisANaN(miss) || isThisANaN(cn)) return null; let queryVal; switch (statistic) { @@ -540,12 +547,12 @@ const calculateStatScalar = function ( statisticAndVariable ) { if ( - Number.isNaN(squareDiffSum) || - Number.isNaN(NSum) || - Number.isNaN(obsModelDiffSum) || - Number.isNaN(modelSum) || - Number.isNaN(obsSum) || - Number.isNaN(absSum) + isThisANaN(squareDiffSum) || + isThisANaN(NSum) || + isThisANaN(obsModelDiffSum) || + isThisANaN(modelSum) || + isThisANaN(obsSum) || + isThisANaN(absSum) ) return null; let queryVal; @@ -578,7 +585,7 @@ const calculateStatScalar = function ( default: queryVal = null; } - if (Number.isNaN(queryVal)) return null; + if (isThisANaN(queryVal)) return null; // need to convert to correct units for surface data but not upperair if (statistic !== "N") { if ( @@ -1191,7 +1198,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { let secs; let delta; for (i = 0; i < sSecs.length; i += 1) { - if (Number.isNaN(sVals[i])) { + if (isThisANaN(sVals[i])) { n -= 1; } else { secs = sSecs[i]; @@ -1214,7 +1221,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { console.log(`matsDataUtil.getErr: ${error}`); } for (i = 0; i < sVals.length; i += 1) { - if (!Number.isNaN(sVals[i])) { + if (!isThisANaN(sVals[i])) { minVal = minVal < sVals[i] ? minVal : sVals[i]; maxVal = maxVal > sVals[i] ? maxVal : sVals[i]; dataSum += sVals[i]; @@ -1247,7 +1254,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { let nDeltas = 0; for (i = 0; i < sSecs.length; i += 1) { - if (!Number.isNaN(sVals[i])) { + if (!isThisANaN(sVals[i])) { let sec = sSecs[i]; if (typeof sec === "string" || sec instanceof String) sec = Number(sec); let lev; @@ -1544,7 +1551,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins": // get the user's chosen number of bins binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1559,7 +1566,7 @@ const setHistogramParameters = function (plotParams) { case "Choose a bin bound": // let the histogram routine know that we want the bins shifted over to whatever was input pivotVal = Number(plotParams["bin-pivot"]); - if (Number.isNaN(pivotVal)) { + if (isThisANaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1567,7 +1574,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and make zero a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to zero binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1578,13 +1585,13 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and choose a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to whatever was input binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } pivotVal = Number(plotParams["bin-pivot"]); - if (Number.isNaN(pivotVal)) { + if (isThisANaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1595,7 +1602,7 @@ const setHistogramParameters = function (plotParams) { binBounds = plotParams["bin-bounds"].split(",").map(function (item) { let thisItem = item.trim(); thisItem = Number(thisItem); - if (!Number.isNaN(thisItem)) { + if (!isThisANaN(thisItem)) { return thisItem; } throw new Error( @@ -1619,17 +1626,17 @@ const setHistogramParameters = function (plotParams) { case "Manual bin start, number, and stride": // get the bin start, number, and stride. binNum = Number(plotParams["bin-number"]); - if (Number.isNaN(binNum)) { + if (isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } binStart = Number(plotParams["bin-start"]); - if (Number.isNaN(binStart)) { + if (isThisANaN(binStart)) { throw new Error("Error parsing bin start: please enter the desired bin start."); } binStride = Number(plotParams["bin-stride"]); - if (Number.isNaN(binStride)) { + if (isThisANaN(binStride)) { throw new Error( "Error parsing bin stride: please enter the desired bin stride." ); @@ -1708,7 +1715,7 @@ const calculateHistogramBins = function ( binLowBounds[binParams.binNum - 1] = fullUpBound; binMeans[binParams.binNum - 1] = fullUpBound + binInterval / 2; - if (binParams.pivotVal !== undefined && !Number.isNaN(binParams.pivotVal)) { + if (binParams.pivotVal !== undefined && !isThisANaN(binParams.pivotVal)) { // need to shift the bounds and means over so that one of the bounds is on the chosen pivot const closestBoundToPivot = binLowBounds.reduce(function (prev, curr) { return Math.abs(curr - binParams.pivotVal) < Math.abs(prev - binParams.pivotVal) @@ -2180,6 +2187,7 @@ export default matsDataUtils = { average, median, stdev, + isThisANaN, dateConvert, getDateRange, secsConvert, diff --git a/meteor_packages/mats-common/package.js b/meteor_packages/mats-common/package.js index 15ebd7bb5..b872f5da8 100644 --- a/meteor_packages/mats-common/package.js +++ b/meteor_packages/mats-common/package.js @@ -30,10 +30,10 @@ Package.onUse(function (api) { "node-file-cache": "1.0.2", "python-shell": "5.0.0", couchbase: "4.3.1", - "simpl-schema": "3.4.6", "vanillajs-datepicker": "1.3.4", daterangepicker: "3.1.0", "lighten-darken-color": "1.0.0", + nodemailer: "6.9.14", }); api.mainModule("server/main.js", "server"); api.mainModule("client/main.js", "client"); @@ -67,6 +67,8 @@ Package.onUse(function (api) { api.use("momentjs:moment"); api.use("pcel:mysql"); api.use("reactive-var"); + api.use("jkuester:http"); + api.use("aldeed:simple-schema"); // modules api.export("matsCollections", ["client", "server"]); @@ -346,51 +348,3 @@ Package.onUse(function (api) { "server" ); }); - -Package.onTest(function (api) { - api.use("ecmascript"); - api.use("meteortesting:mocha"); - api.use("randyp:mats-common"); - api.addFiles("imports/startup/api/version-info-tests.js"); - - // try duplicating the runtime deps - Npm.depends({ - "fs-extra": "7.0.0", - "@babel/runtime": "7.10.4", - "meteor-node-stubs": "0.4.1", - url: "0.11.0", - "jquery-ui": "1.12.1", - "csv-stringify": "4.3.1", - "node-file-cache": "1.0.2", - "python-shell": "3.0.1", - couchbase: "3.2.3", - "simpl-schema": "1.12.0", - }); - api.use("natestrauser:select2", "client"); - api.use("mdg:validated-method"); - api.use("ecmascript"); - api.use("modules"); - api.imply("ecmascript"); - api.use(["templating"], "client"); - api.use("accounts-ui", "client"); - api.use("accounts-password", "client"); - api.use("service-configuration", "server"); - api.use("yasinuslu:json-view", "client"); - api.use("mdg:validated-method"); - api.use("session"); - api.imply("session"); - api.use("twbs:bootstrap"); - api.use("msavin:mongol"); - api.use("differential:event-hooks"); - api.use("risul:bootstrap-colorpicker"); - api.use("logging"); - api.use("reload"); - api.use("random"); - api.use("ejson"); - api.use("spacebars"); - api.use("check"); - api.use("ostrio:flow-router-extra"); - api.use("meteorhacks:picker"); - api.use("momentjs:moment"); - api.use("pcel:mysql"); -}); From 23fc6a761c34f5df76b2ad0023448e4fdd35b4e3 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Thu, 15 Aug 2024 16:37:05 -0600 Subject: [PATCH 36/55] Further MATScommon linting --- .eslintrc.json | 2 +- meteor_packages/mats-common/client/error.js | 5 +- meteor_packages/mats-common/client/info.js | 5 +- meteor_packages/mats-common/client/main.js | 2 + meteor_packages/mats-common/client/status.js | 5 +- .../imports/startup/api/matsMethods.js | 7 + .../imports/startup/api/version-info-tests.js | 14 +- .../imports/startup/api/version-info.js | 1 + .../imports/startup/both/mats-collections.js | 5 +- .../imports/startup/both/mats-curve-params.js | 1 + .../imports/startup/both/mats-types.js | 21 +- .../imports/startup/client/curve_util.js | 217 ++++++++--------- .../imports/startup/client/graph_util.js | 104 +++++---- .../imports/startup/client/init.js | 9 +- .../imports/startup/client/routes.js | 15 +- .../startup/server/data_plot_ops_util.js | 4 +- .../startup/server/data_process_util.js | 79 ++++--- .../imports/startup/server/data_query_util.js | 59 ++--- .../imports/startup/server/data_util.js | 74 +++--- tests/src/steps/given.js | 95 -------- tests/src/steps/then.js | 220 ------------------ tests/src/steps/when.js | 110 --------- tests/src/support/action/clearInputField.js | 7 - tests/src/support/action/clickElement.js | 25 -- tests/src/support/action/clickMatsButton.js | 89 ------- .../action/clickMatsButtonReturnImmediate.js | 85 ------- .../src/support/action/closeAllButFirstTab.js | 22 -- .../support/action/closeLastOpenedWindow.js | 16 -- tests/src/support/action/deleteCookies.js | 7 - tests/src/support/action/dragElement.js | 8 - .../support/action/focusLastOpenedWindow.js | 15 -- tests/src/support/action/handleModal.js | 21 -- .../src/support/action/matsClearParameter.js | 12 - tests/src/support/action/matsDebug.js | 6 - .../src/support/action/matsRefreshBrowser.js | 10 - tests/src/support/action/matsRefreshPage.js | 6 - tests/src/support/action/moveTo.js | 21 -- tests/src/support/action/openWebsite.js | 13 -- .../action/openWebsiteAndWaitForPlotType.js | 20 -- tests/src/support/action/pause.js | 13 -- tests/src/support/action/pressButton.js | 7 - .../src/support/action/saveMatsParameters.js | 10 - tests/src/support/action/scroll.js | 7 - tests/src/support/action/selectOption.js | 46 ---- .../src/support/action/selectOptionByIndex.js | 17 -- tests/src/support/action/setCookie.js | 12 - tests/src/support/action/setInputField.js | 26 --- .../support/action/setMatsCurveDateRange.js | 38 --- tests/src/support/action/setMatsDateRange.js | 49 ---- tests/src/support/action/setMatsPlotFormat.js | 22 -- tests/src/support/action/setMatsPlotType.js | 10 - .../support/action/setMatsSelectedOption.js | 73 ------ .../support/action/setPredefinedDateRange.js | 46 ---- tests/src/support/action/setPromptText.js | 11 - .../src/support/action/setScatterDateRange.js | 29 --- tests/src/support/action/setWindowSize.js | 8 - tests/src/support/action/waitFor.js | 48 ---- tests/src/support/action/waitForDisplayed.js | 17 -- tests/src/support/check/checkClass.js | 26 --- .../src/support/check/checkContainsAnyText.js | 42 ---- tests/src/support/check/checkContainsText.js | 55 ----- tests/src/support/check/checkCookieContent.js | 28 --- tests/src/support/check/checkCookieExists.js | 25 -- tests/src/support/check/checkDimension.js | 51 ---- tests/src/support/check/checkElementExists.js | 21 -- tests/src/support/check/checkEqualsText.js | 51 ---- tests/src/support/check/checkFocus.js | 19 -- tests/src/support/check/checkFontProperty.js | 51 ---- tests/src/support/check/checkInURLPath.js | 25 -- tests/src/support/check/checkIsEmpty.js | 13 -- .../support/check/checkIsOpenedInNewWindow.js | 34 --- tests/src/support/check/checkMatsAppTitle.js | 23 -- .../support/check/checkMatsCurveDatesValue.js | 15 -- .../support/check/checkMatsCurveIsAdded.js | 23 -- .../check/checkMatsCurveListContains.js | 25 -- .../src/support/check/checkMatsCurveNumber.js | 39 ---- .../src/support/check/checkMatsDatesValue.js | 15 -- .../support/check/checkMatsGraphPlotType.js | 19 -- .../src/support/check/checkMatsInfoMessage.js | 18 -- .../check/checkMatsLegendListContains.js | 21 -- .../src/support/check/checkMatsParameters.js | 16 -- .../src/support/check/checkMatsPlotNumber.js | 16 -- tests/src/support/check/checkModal.js | 31 --- tests/src/support/check/checkModalText.js | 31 --- tests/src/support/check/checkNewWindow.js | 19 -- tests/src/support/check/checkOffset.js | 36 --- .../src/support/check/checkParameterValue.js | 18 -- tests/src/support/check/checkProperty.js | 53 ----- tests/src/support/check/checkSelected.js | 19 -- tests/src/support/check/checkTitle.js | 25 -- tests/src/support/check/checkTitleContains.js | 25 -- tests/src/support/check/checkURL.js | 25 -- tests/src/support/check/checkURLPath.js | 33 --- .../src/support/check/checkWithinViewport.js | 25 -- tests/src/support/check/compareText.js | 26 --- tests/src/support/check/isDisplayed.js | 21 -- tests/src/support/check/isEnabled.js | 22 -- tests/src/support/check/isExisting.js | 21 -- tests/src/support/check/isGroupValue.js | 14 -- tests/src/support/check/isMainDisplayed.js | 11 - .../src/support/check/isMatsButtonEnabled.js | 42 ---- .../src/support/check/isMatsButtonVisible.js | 54 ----- tests/src/support/check/isMatsCurveColor.js | 14 -- .../src/support/check/isMatsGraphDisplayed.js | 25 -- tests/src/support/check/isMatsInfoVisible.js | 23 -- tests/src/support/check/isMatsPlotFormat.js | 17 -- tests/src/support/check/isMatsPlotType.js | 16 -- .../src/support/check/isMatsSelectedOption.js | 29 --- tests/src/support/hooks.js | 151 ------------ tests/src/support/lib/checkIfElementExists.js | 31 --- 110 files changed, 334 insertions(+), 3120 deletions(-) delete mode 100644 tests/src/steps/given.js delete mode 100644 tests/src/steps/then.js delete mode 100644 tests/src/steps/when.js delete mode 100644 tests/src/support/action/clearInputField.js delete mode 100644 tests/src/support/action/clickElement.js delete mode 100644 tests/src/support/action/clickMatsButton.js delete mode 100644 tests/src/support/action/clickMatsButtonReturnImmediate.js delete mode 100644 tests/src/support/action/closeAllButFirstTab.js delete mode 100644 tests/src/support/action/closeLastOpenedWindow.js delete mode 100644 tests/src/support/action/deleteCookies.js delete mode 100644 tests/src/support/action/dragElement.js delete mode 100644 tests/src/support/action/focusLastOpenedWindow.js delete mode 100644 tests/src/support/action/handleModal.js delete mode 100644 tests/src/support/action/matsClearParameter.js delete mode 100644 tests/src/support/action/matsDebug.js delete mode 100644 tests/src/support/action/matsRefreshBrowser.js delete mode 100644 tests/src/support/action/matsRefreshPage.js delete mode 100644 tests/src/support/action/moveTo.js delete mode 100644 tests/src/support/action/openWebsite.js delete mode 100644 tests/src/support/action/openWebsiteAndWaitForPlotType.js delete mode 100644 tests/src/support/action/pause.js delete mode 100644 tests/src/support/action/pressButton.js delete mode 100644 tests/src/support/action/saveMatsParameters.js delete mode 100644 tests/src/support/action/scroll.js delete mode 100644 tests/src/support/action/selectOption.js delete mode 100644 tests/src/support/action/selectOptionByIndex.js delete mode 100644 tests/src/support/action/setCookie.js delete mode 100644 tests/src/support/action/setInputField.js delete mode 100644 tests/src/support/action/setMatsCurveDateRange.js delete mode 100644 tests/src/support/action/setMatsDateRange.js delete mode 100644 tests/src/support/action/setMatsPlotFormat.js delete mode 100644 tests/src/support/action/setMatsPlotType.js delete mode 100644 tests/src/support/action/setMatsSelectedOption.js delete mode 100644 tests/src/support/action/setPredefinedDateRange.js delete mode 100644 tests/src/support/action/setPromptText.js delete mode 100644 tests/src/support/action/setScatterDateRange.js delete mode 100644 tests/src/support/action/setWindowSize.js delete mode 100644 tests/src/support/action/waitFor.js delete mode 100644 tests/src/support/action/waitForDisplayed.js delete mode 100644 tests/src/support/check/checkClass.js delete mode 100644 tests/src/support/check/checkContainsAnyText.js delete mode 100644 tests/src/support/check/checkContainsText.js delete mode 100644 tests/src/support/check/checkCookieContent.js delete mode 100644 tests/src/support/check/checkCookieExists.js delete mode 100644 tests/src/support/check/checkDimension.js delete mode 100644 tests/src/support/check/checkElementExists.js delete mode 100644 tests/src/support/check/checkEqualsText.js delete mode 100644 tests/src/support/check/checkFocus.js delete mode 100644 tests/src/support/check/checkFontProperty.js delete mode 100644 tests/src/support/check/checkInURLPath.js delete mode 100644 tests/src/support/check/checkIsEmpty.js delete mode 100644 tests/src/support/check/checkIsOpenedInNewWindow.js delete mode 100644 tests/src/support/check/checkMatsAppTitle.js delete mode 100644 tests/src/support/check/checkMatsCurveDatesValue.js delete mode 100644 tests/src/support/check/checkMatsCurveIsAdded.js delete mode 100644 tests/src/support/check/checkMatsCurveListContains.js delete mode 100644 tests/src/support/check/checkMatsCurveNumber.js delete mode 100644 tests/src/support/check/checkMatsDatesValue.js delete mode 100644 tests/src/support/check/checkMatsGraphPlotType.js delete mode 100644 tests/src/support/check/checkMatsInfoMessage.js delete mode 100644 tests/src/support/check/checkMatsLegendListContains.js delete mode 100644 tests/src/support/check/checkMatsParameters.js delete mode 100644 tests/src/support/check/checkMatsPlotNumber.js delete mode 100644 tests/src/support/check/checkModal.js delete mode 100644 tests/src/support/check/checkModalText.js delete mode 100644 tests/src/support/check/checkNewWindow.js delete mode 100644 tests/src/support/check/checkOffset.js delete mode 100644 tests/src/support/check/checkParameterValue.js delete mode 100644 tests/src/support/check/checkProperty.js delete mode 100644 tests/src/support/check/checkSelected.js delete mode 100644 tests/src/support/check/checkTitle.js delete mode 100644 tests/src/support/check/checkTitleContains.js delete mode 100644 tests/src/support/check/checkURL.js delete mode 100644 tests/src/support/check/checkURLPath.js delete mode 100644 tests/src/support/check/checkWithinViewport.js delete mode 100644 tests/src/support/check/compareText.js delete mode 100644 tests/src/support/check/isDisplayed.js delete mode 100644 tests/src/support/check/isEnabled.js delete mode 100644 tests/src/support/check/isExisting.js delete mode 100644 tests/src/support/check/isGroupValue.js delete mode 100644 tests/src/support/check/isMainDisplayed.js delete mode 100644 tests/src/support/check/isMatsButtonEnabled.js delete mode 100644 tests/src/support/check/isMatsButtonVisible.js delete mode 100644 tests/src/support/check/isMatsCurveColor.js delete mode 100644 tests/src/support/check/isMatsGraphDisplayed.js delete mode 100644 tests/src/support/check/isMatsInfoVisible.js delete mode 100644 tests/src/support/check/isMatsPlotFormat.js delete mode 100644 tests/src/support/check/isMatsPlotType.js delete mode 100644 tests/src/support/check/isMatsSelectedOption.js delete mode 100644 tests/src/support/hooks.js delete mode 100644 tests/src/support/lib/checkIfElementExists.js diff --git a/.eslintrc.json b/.eslintrc.json index 3c5c1d052..df1dde61e 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -82,7 +82,7 @@ "no-prototype-builtins": "warn", "no-await-in-loop": "warn", "no-dupe-else-if": "warn", - "meteor/no-session": "warn", + "meteor/no-session": "off", "meteor/template-names": "warn", "meteor/eventmap-params": "warn", "meteor/no-template-lifecycle-assignments": "warn", diff --git a/meteor_packages/mats-common/client/error.js b/meteor_packages/mats-common/client/error.js index 27b76e506..af9b9db8b 100644 --- a/meteor_packages/mats-common/client/error.js +++ b/meteor_packages/mats-common/client/error.js @@ -2,6 +2,9 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ +/* global setInfo, Session, $ */ +/* eslint-disable no-undef */ + setError = function (error) { let myError = ""; let myStackTrace = ""; @@ -27,7 +30,7 @@ setError = function (error) { $("#error").modal("show"); }; -clearError = function (message) { +clearError = function () { Session.set("errorMessage", ""); Session.set("stackTrace", ""); $("#error").modal("hide"); diff --git a/meteor_packages/mats-common/client/info.js b/meteor_packages/mats-common/client/info.js index 736f65615..5664c22ba 100644 --- a/meteor_packages/mats-common/client/info.js +++ b/meteor_packages/mats-common/client/info.js @@ -2,12 +2,15 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ +/* global Session, $ */ +/* eslint-disable no-undef */ + setInfo = function (info) { Session.set("infoMessage", info); $("#info").modal("show"); }; -clearInfo = function (info) { +clearInfo = function () { Session.set("infoMessage", ""); $("#info").modal("hide"); }; diff --git a/meteor_packages/mats-common/client/main.js b/meteor_packages/mats-common/client/main.js index 4a1f74293..9c8cdd8d5 100644 --- a/meteor_packages/mats-common/client/main.js +++ b/meteor_packages/mats-common/client/main.js @@ -6,6 +6,8 @@ * Created by pierce on 8/31/16. */ +/* eslint-disable import/no-unresolved */ + import "../imports/startup/client"; import "../imports/startup/both"; import "@fortawesome/fontawesome-free"; diff --git a/meteor_packages/mats-common/client/status.js b/meteor_packages/mats-common/client/status.js index 8635918dc..a9902617d 100644 --- a/meteor_packages/mats-common/client/status.js +++ b/meteor_packages/mats-common/client/status.js @@ -2,11 +2,14 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ +/* global Session */ +/* eslint-disable no-undef */ + setStatus = function (status) { Session.set("statusMessage", status); }; -clearStatus = function (status) { +clearStatus = function () { Session.set("statusMessage", ""); }; diff --git a/meteor_packages/mats-common/imports/startup/api/matsMethods.js b/meteor_packages/mats-common/imports/startup/api/matsMethods.js index 38a70de68..87b4a4c5d 100644 --- a/meteor_packages/mats-common/imports/startup/api/matsMethods.js +++ b/meteor_packages/mats-common/imports/startup/api/matsMethods.js @@ -52,6 +52,12 @@ const status = function (res) { } }; +// wrapper for NaN check +const isThisANaN = function (val) { + // eslint-disable-next-line no-restricted-globals + return !val || isNaN(val); +}; + // private - used to see if the main page needs to update its selectors const checkMetaDataRefresh = async function () { // This routine compares the current last modified time of the tables (MYSQL) or documents (Couchbase) @@ -3946,6 +3952,7 @@ if (Meteor.isServer) { // eslint-disable-next-line no-undef export default matsMethods = { + isThisANaN, addSentAddress, applyAuthorization, applyDatabaseSettings, diff --git a/meteor_packages/mats-common/imports/startup/api/version-info-tests.js b/meteor_packages/mats-common/imports/startup/api/version-info-tests.js index c7b24bf6c..b1d688170 100644 --- a/meteor_packages/mats-common/imports/startup/api/version-info-tests.js +++ b/meteor_packages/mats-common/imports/startup/api/version-info-tests.js @@ -1,5 +1,7 @@ import { versionInfo } from "meteor/randyp:mats-common"; +/* eslint-disable no-undef */ + const assert = require("assert"); describe("getVersionsFromEnv", function () { @@ -25,18 +27,18 @@ describe("getVersionsFromEnv", function () { // Test it("Correctly reads version from env", function () { process.env.VERSION = "4.2.0"; - const { version, commit, branch } = versionInfo.getVersionsFromEnv(); - assert.equal(version, "4.2.0"); + const versionStats = versionInfo.getVersionsFromEnv(); + assert.equal(versionStats.version, "4.2.0"); }); it("Correctly reads commit from env", function () { process.env.COMMIT = "ae214rfda"; - const { version, commit, branch } = versionInfo.getVersionsFromEnv(); - assert.equal(commit, "ae214rfda"); + const versionStats = versionInfo.getVersionsFromEnv(); + assert.equal(versionStats.commit, "ae214rfda"); }); it("Correctly reads version from env", function () { process.env.BRANCH = "test"; - const { version, commit, branch } = versionInfo.getVersionsFromEnv(); - assert.equal(branch, "test"); + const versionStats = versionInfo.getVersionsFromEnv(); + assert.equal(versionStats.branch, "test"); }); it("Correctly handles no env", function () { const { version, commit, branch } = versionInfo.getVersionsFromEnv(); diff --git a/meteor_packages/mats-common/imports/startup/api/version-info.js b/meteor_packages/mats-common/imports/startup/api/version-info.js index d4f985e9e..ba06ef818 100644 --- a/meteor_packages/mats-common/imports/startup/api/version-info.js +++ b/meteor_packages/mats-common/imports/startup/api/version-info.js @@ -20,6 +20,7 @@ function getVersionsFromEnv() { }; } +// eslint-disable-next-line no-undef export default versionInfo = { getVersionsFromEnv, }; diff --git a/meteor_packages/mats-common/imports/startup/both/mats-collections.js b/meteor_packages/mats-common/imports/startup/both/mats-collections.js index dc897386f..7abdea832 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-collections.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-collections.js @@ -9,6 +9,8 @@ import { Mongo } from "meteor/mongo"; import { Meteor } from "meteor/meteor"; import { curveParamsByApp } from "./mats-curve-params"; +/* eslint-disable no-console */ + const params = curveParamsByApp[Meteor.settings.public.app]; if (!params) { console.log( @@ -20,7 +22,7 @@ if (!params) { } const paramCollections = {}; let currParam; -for (let i = 0; i < params.length; i++) { +for (let i = 0; i < params.length; i += 1) { currParam = params[i]; paramCollections[currParam] = new Mongo.Collection(currParam); } @@ -86,6 +88,7 @@ const explicitCollections = { Scorecard, }; +// eslint-disable-next-line no-undef export default matsCollections = { ...paramCollections, ...explicitCollections, diff --git a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js index 5b165afde..32e9f2407 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-curve-params.js @@ -1,3 +1,4 @@ +// eslint-disable-next-line no-undef, import/prefer-default-export export const curveParamsByApp = { "cb-metar": [ "label", diff --git a/meteor_packages/mats-common/imports/startup/both/mats-types.js b/meteor_packages/mats-common/imports/startup/both/mats-types.js index 8548d5c7e..0914b4da2 100644 --- a/meteor_packages/mats-common/imports/startup/both/mats-types.js +++ b/meteor_packages/mats-common/imports/startup/both/mats-types.js @@ -185,39 +185,40 @@ of table names. The internal list can be appended. The getRecords returns the in */ class MetaDataDBRecord { constructor(poolName, dbName, tables) { - if (!typeof poolName === "string") { + if (!(typeof poolName === "string")) { throw new Error("MetaDataDBRecord.constructor : poolName is not a string"); } - if (!typeof dbName === "string") { + if (!(typeof dbName === "string")) { throw new Error("MetaDataDBRecord.constructor : dbName is not a string"); } - if (!tables instanceof Array) { + if (!(tables instanceof Array)) { throw new Error("MetaDataDBRecord.constructor : tables is not an array"); } - this._records = []; + this.records = []; const record = { pool: poolName, name: dbName, tables }; - this._records.push(record); + this.records.push(record); } addRecord(poolName, dbName, tables) { - if (!typeof poolName === "string") { + if (!(typeof poolName === "string")) { throw new Error("MetaDataDBRecord.constructor : poolName is not a string"); } - if (!typeof dbName === "string") { + if (!(typeof dbName === "string")) { throw new Error("MetaDataDBRecord.constructor : dbName is not a string"); } - if (!tables instanceof Array) { + if (!(tables instanceof Array)) { throw new Error("MetaDataDBRecord.constructor : tables is not an array"); } const record = { pool: poolName, name: dbName, tables }; - this._records.push(record); + this.records.push(record); } getRecords() { - return this._records; + return this.records; } } +// eslint-disable-next-line no-undef export default matsTypes = { InputTypes, ScorecardStatus, diff --git a/meteor_packages/mats-common/imports/startup/client/curve_util.js b/meteor_packages/mats-common/imports/startup/client/curve_util.js index ca24b486a..1e6cfd1e7 100644 --- a/meteor_packages/mats-common/imports/startup/client/curve_util.js +++ b/meteor_packages/mats-common/imports/startup/client/curve_util.js @@ -5,74 +5,70 @@ import { matsTypes, matsCollections, + matsMethods, matsPlotUtils, matsParamUtils, - Info, - matsMethods, } from "meteor/randyp:mats-common"; -/* - global dataset variable - container for graph dataset. - This (plotResult) is very important. It isn't "var" because it needs to be a meteor global scope. - The page is rendered whe the graph page comes up, but the data from the data processing callback - in plotList.js or curveList.js may not have set the global variable - PlotResult. - */ +/* global $, _, Session, setError, setInfo */ +/* eslint-disable no-console */ // var plotResultData = null; -- this was the global variable for the text output data, but now it is set elsewhere let graphResult = null; // this is the global variable for the data on the graph -let plot; -const sizeof = function (_1) { - const _2 = [_1]; - let _3 = 0; - for (let _4 = 0; _4 < _2.length; _4++) { - switch (typeof _2[_4]) { +const sizeof = function (val1) { + const val2 = [val1]; + let val24Keys = []; + let val3 = 0; + for (let val4 = 0; val4 < val2.length; val4 += 1) { + switch (typeof val2[val4]) { case "boolean": - _3 += 4; + val3 += 4; break; case "number": - _3 += 8; + val3 += 8; break; case "string": - _3 += 2 * _2[_4].length; + val3 += 2 * val2[val4].length; break; case "object": - if (Object.prototype.toString.call(_2[_4]) !== "[object Array]") { - for (var _5 in _2[_4]) { - _3 += 2 * _5.length; + val24Keys = Object.keys(val2[val4]); + if (Object.prototype.toString.call(val2[val4]) !== "[object Array]") { + for (let v24idx = 0; v24idx < val24Keys.length; v24idx += 1) { + const val5 = val2[val4][val24Keys[v24idx]]; + val3 += 2 * val5.length; } } - for (var _5 in _2[_4]) { - let _6 = false; - for (let _7 = 0; _7 < _2.length; _7++) { - if (_2[_7] === _2[_4][_5]) { - _6 = true; + for (let v24idx = 0; v24idx < val24Keys.length; v24idx += 1) { + const val5 = val2[val4][val24Keys[v24idx]]; + let val6 = false; + for (let val7 = 0; val7 < val2.length; val7 += 1) { + if (val2[val7] === val2[val4][val5]) { + val6 = true; break; } } - if (!_6) { - _2.push(_2[_4][_5]); + if (!val6) { + val2.push(val2[val4][val5]); } } + break; + default: + val3 = 0; } } - return _3; + return val3; }; -// Retrieves the globally stored plotResultData for the text output and other things. -// Re-sets the plotResultData if the requested page range has changed, or if it has not been previously set. -const getPlotResultData = function () { - const pageIndex = Session.get("pageIndex"); - const newPageIndex = Session.get("newPageIndex"); - if ( - plotResultData === undefined || - plotResultData === null || - Session.get("textRefreshNeeded") === true - ) { - setPlotResultData(); +const showSpinner = function () { + if (document.getElementById("spinner")) { + document.getElementById("spinner").style.display = "block"; + } +}; +const hideSpinner = function () { + if (document.getElementById("spinner")) { + document.getElementById("spinner").style.display = "none"; } - return plotResultData; }; // Sets the global plotResultData variable for the text output to the requested range from the Results data stored in mongo, via a MatsMethod. @@ -94,15 +90,18 @@ const setPlotResultData = function () { Session.set("textRefreshNeeded", false); } if (!result) { + // eslint-disable-next-line no-undef plotResultData = undefined; Session.set("textRefreshNeeded", false); hideSpinner(); return; } + // eslint-disable-next-line no-undef plotResultData = result; Session.set("pageIndex", result.dsiRealPageIndex); Session.set("pageTextDirection", result.dsiTextDirection); Session.set("textLoaded", new Date()); + // eslint-disable-next-line no-undef console.log("size of plotResultData is ", sizeof(plotResultData)); Session.set("textRefreshNeeded", false); hideSpinner(); @@ -111,8 +110,25 @@ const setPlotResultData = function () { } }; +// Retrieves the globally stored plotResultData for the text output and other things. +// Re-sets the plotResultData if the requested page range has changed, or if it has not been previously set. +const getPlotResultData = function () { + if ( + // eslint-disable-next-line no-undef + plotResultData === undefined || + // eslint-disable-next-line no-undef + plotResultData === null || + Session.get("textRefreshNeeded") === true + ) { + setPlotResultData(); + } + // eslint-disable-next-line no-undef + return plotResultData; +}; + // resets the global plotResultData variable for the text output to null const resetPlotResultData = function () { + // eslint-disable-next-line no-undef plotResultData = null; Session.set("textLoaded", new Date()); }; @@ -145,6 +161,7 @@ const setCurveParamDisplayText = function (paramName, newText) { paramName, newText, }, + // eslint-disable-next-line no-unused-vars function (error, res) { if (error !== undefined) { setError(error); @@ -164,13 +181,6 @@ const getUsedLabels = function () { return Session.get("UsedLabels"); }; -const getNextCurveLabel = function () { - if (Session.get("NextCurveLabel") === undefined) { - setNextCurveLabel(); - } - return Session.get("NextCurveLabel"); -}; - // determine the next curve Label and set it in the session // private, not exported const setNextCurveLabel = function () { @@ -194,7 +204,7 @@ const setNextCurveLabel = function () { if (lastUsedLabel !== undefined) { const minusPrefix = lastUsedLabel.replace(labelPrefix, ""); const tryNum = parseInt(minusPrefix, 10); - if (!isNaN(tryNum)) { + if (!matsMethods.isThisANaN(tryNum)) { lastLabelNumber = tryNum; } } @@ -202,10 +212,23 @@ const setNextCurveLabel = function () { let nextCurveLabel = labelPrefix + newLabelNumber; // the label might be one from a removed curve so the next ones might be used while (_.indexOf(usedLabels, nextCurveLabel) !== -1) { - newLabelNumber++; + newLabelNumber += 1; nextCurveLabel = labelPrefix + newLabelNumber; } Session.set("NextCurveLabel", nextCurveLabel); + return null; +}; + +const getNextCurveLabel = function () { + if (Session.get("NextCurveLabel") === undefined) { + setNextCurveLabel(); + } + return Session.get("NextCurveLabel"); +}; + +// function for random color +const randomRGB = function () { + return Math.floor(Math.random() * 226); }; // determine the next curve color and set it in the session @@ -213,27 +236,17 @@ const setNextCurveLabel = function () { const setNextCurveColor = function () { const usedColors = Session.get("UsedColors"); const { colors } = matsCollections.ColorScheme.findOne({}, { fields: { colors: 1 } }); - let lastUsedIndex = -1; - if (usedColors !== undefined) { - lastUsedIndex = _.indexOf(colors, _.last(usedColors)); - } + const lastUsedIndex = usedColors ? usedColors.length - 1 : -1; let nextCurveColor; if (lastUsedIndex !== undefined && lastUsedIndex !== -1) { if (lastUsedIndex < colors.length - 1) { - let newIndex = lastUsedIndex + 1; - nextCurveColor = colors[newIndex]; - // the color might be one from a removed curve so the next ones might be used - while (_.indexOf(usedColors, nextCurveColor) !== -1) { - newIndex++; - nextCurveColor = colors[newIndex]; - } + nextCurveColor = colors[lastUsedIndex + 1]; } else { // out of defaults - const rint = Math.round(0xffffff * Math.random()); - nextCurveColor = `rgb(${rint >> 16},${(rint >> 8) & 255},${rint & 255})`; + nextCurveColor = `rgb(${randomRGB()},${randomRGB()},${randomRGB()})`; } } else { - nextCurveColor = colors[0]; + [nextCurveColor] = colors; } Session.set("NextCurveColor", nextCurveColor); }; @@ -291,7 +304,7 @@ const clearAllUsed = function () { const setUsedColors = function () { const curves = Session.get("Curves"); const usedColors = []; - for (let i = 0; i < curves.length; i++) { + for (let i = 0; i < curves.length; i += 1) { const { color } = curves[i]; usedColors.push(color); } @@ -303,7 +316,7 @@ const setUsedColors = function () { const setUsedLabels = function () { const curves = Session.get("Curves"); const usedLabels = []; - for (let i = 0; i < curves.length; i++) { + for (let i = 0; i < curves.length; i += 1) { const { label } = curves[i]; usedLabels.push(label); } @@ -341,29 +354,14 @@ const addDiffs = function () { return false; } + let baseIndex = 0; switch (matsPlotUtils.getPlotFormat()) { - case matsTypes.PlotFormats.matching: - var baseIndex = 0; // This will probably not default to curve 0 in the future - for (var ci = 1; ci < curves.length; ci++) { - var newCurve = $.extend(true, {}, curves[ci]); - newCurve.label = `${curves[ci].label}-${curves[0].label}`; - newCurve.color = getNextCurveColor(); - newCurve.diffFrom = [ci, baseIndex]; - // do not create extra diff if it already exists - if (_.findWhere(curves, { label: newCurve.label }) === undefined) { - newCurves.push(newCurve); - Session.set("Curves", newCurves); - setUsedColorsAndLabels(); - } - } - break; case matsTypes.PlotFormats.pairwise: - var baseIndex = 0; // This will probably not default to curve 0 in the future - for (var ci = 1; ci < curves.length; ci++) { + for (let ci = 1; ci < curves.length; ci += 1) { if (ci % 2 !== 0) { // only diff on odd curves against previous curve baseIndex = ci - 1; - var newCurve = $.extend(true, {}, curves[ci]); + const newCurve = $.extend(true, {}, curves[ci]); newCurve.label = `${curves[ci].label}-${curves[baseIndex].label}`; newCurve.color = getNextCurveColor(); newCurve.diffFrom = [ci, baseIndex]; @@ -377,9 +375,23 @@ const addDiffs = function () { } break; case matsTypes.PlotFormats.absolute: - var baseIndex = 0; // This will probably not default to curve 0 in the future - for (var ci = 1; ci < curves.length; ci++) { - var newCurve = $.extend(true, {}, curves[ci]); + for (let ci = 1; ci < curves.length; ci += 1) { + const newCurve = $.extend(true, {}, curves[ci]); + newCurve.label = `${curves[ci].label}-${curves[0].label}`; + newCurve.color = getNextCurveColor(); + newCurve.diffFrom = [ci, baseIndex]; + // do not create extra diff if it already exists + if (_.findWhere(curves, { label: newCurve.label }) === undefined) { + newCurves.push(newCurve); + Session.set("Curves", newCurves); + setUsedColorsAndLabels(); + } + } + break; + case matsTypes.PlotFormats.matching: + default: + for (let ci = 1; ci < curves.length; ci += 1) { + const newCurve = $.extend(true, {}, curves[ci]); newCurve.label = `${curves[ci].label}-${curves[0].label}`; newCurve.color = getNextCurveColor(); newCurve.diffFrom = [ci, baseIndex]; @@ -392,6 +404,7 @@ const addDiffs = function () { } break; } + return null; }; // remove difference curves @@ -427,6 +440,7 @@ const checkDiffs = function () { const checkIfDisplayAllQCParams = function (faceOptions) { // we only want to allow people to filter sub-values for apps with scalar or precalculated stats. // the stats in the list below are representative of these apps. + const theseFaceOptions = faceOptions; const subValueFilterableStats = [ "RMSE", // scalar stats "ACC", // anomalycor stats @@ -447,17 +461,17 @@ const checkIfDisplayAllQCParams = function (faceOptions) { 0 || _.intersection(thisAppsStatistics.options, doNotFilterStats).length > 0) ) { - if (faceOptions.QCParamGroup === "block") { + if (theseFaceOptions.QCParamGroup === "block") { // not a map plot, display only the gaps selector - faceOptions.QCParamGroup = "none"; - faceOptions["QCParamGroup-gaps"] = "block"; - } else if (faceOptions["QCParamGroup-lite"] === "block") { + theseFaceOptions.QCParamGroup = "none"; + theseFaceOptions["QCParamGroup-gaps"] = "block"; + } else if (theseFaceOptions["QCParamGroup-lite"] === "block") { // map plot, display nothing - faceOptions["QCParamGroup-lite"] = "none"; + theseFaceOptions["QCParamGroup-lite"] = "none"; } } } - return faceOptions; + return theseFaceOptions; }; const setSelectorVisibility = function (plotType, faceOptions, selectorsToReset) { @@ -468,7 +482,7 @@ const setSelectorVisibility = function (plotType, faceOptions, selectorsToReset) ) { // reset selectors that may have been set to something invalid for the new plot type const resetSelectors = Object.keys(selectorsToReset); - for (let ridx = 0; ridx < resetSelectors.length; ridx++) { + for (let ridx = 0; ridx < resetSelectors.length; ridx += 1) { if (matsParamUtils.getParameterForName(resetSelectors[ridx]) !== undefined) { if ( matsParamUtils.getParameterForName(resetSelectors[ridx]).type === @@ -489,7 +503,7 @@ const setSelectorVisibility = function (plotType, faceOptions, selectorsToReset) // show/hide selectors appropriate to this plot type let elem; const faceSelectors = Object.keys(faceOptions); - for (let fidx = 0; fidx < faceSelectors.length; fidx++) { + for (let fidx = 0; fidx < faceSelectors.length; fidx += 1) { elem = document.getElementById(`${faceSelectors[fidx]}-item`); if ( elem && @@ -1495,17 +1509,7 @@ const showScatterFace = function () { return selectorsToReset; }; -const showSpinner = function () { - if (document.getElementById("spinner")) { - document.getElementById("spinner").style.display = "block"; - } -}; -const hideSpinner = function () { - if (document.getElementById("spinner")) { - document.getElementById("spinner").style.display = "none"; - } -}; - +// eslint-disable-next-line no-undef export default matsCurveUtils = { addDiffs, checkDiffs, @@ -1525,6 +1529,7 @@ export default matsCurveUtils = { setGraphResult, setUsedColorsAndLabels, setUsedLabels, + setCurveParamDisplayText, showSpinner, showTimeseriesFace, showProfileFace, diff --git a/meteor_packages/mats-common/imports/startup/client/graph_util.js b/meteor_packages/mats-common/imports/startup/client/graph_util.js index b6ab15025..f5a1d5fee 100644 --- a/meteor_packages/mats-common/imports/startup/client/graph_util.js +++ b/meteor_packages/mats-common/imports/startup/client/graph_util.js @@ -2,11 +2,14 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsTypes, matsCollections } from "meteor/randyp:mats-common"; +import { matsTypes } from "meteor/randyp:mats-common"; + +/* global $, Session */ +/* eslint-disable no-console */ // set the label for the hide show buttons (NO DATA) for the initial time here const setNoDataLabels = function (dataset) { - for (let c = 0; c < dataset.length; c++) { + for (let c = 0; c < dataset.length; c += 1) { if (dataset[c].x.length === 0) { Session.set(`${dataset[c].curveId}hideButtonText`, "NO DATA"); if (document.getElementById(`${dataset[c].curveId}-curve-show-hide`)) { @@ -186,7 +189,7 @@ const setNoDataLabels = function (dataset) { }; const setNoDataLabelsMap = function (dataset) { - for (let c = 0; c < dataset.length; c++) { + for (let c = 0; c < dataset.length; c += 1) { if (dataset[c].lat.length === 0) { Session.set(`${dataset[c].curveId}heatMapButtonText`, "NO DATA"); if (document.getElementById(`${dataset[c].curveId}-curve-show-hide-heatmap`)) { @@ -206,7 +209,7 @@ const setNoDataLabelsMap = function (dataset) { ).style.color = "white"; } } else { - var heatMapText; + let heatMapText; if (dataset[c].datatype === "ctc") { heatMapText = "hide heat map"; } else { @@ -233,6 +236,48 @@ const setNoDataLabelsMap = function (dataset) { } }; +const standAloneSquareWidthHeight = function () { + console.log("squareWidthHeight"); + const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); + const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); + const min = Math.min(vpw, vph); + return `${(0.9 * min).toString()}px`; +}; +const standAloneRectangleWidth = function () { + console.log("rectangleWidth"); + const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); + return `${(0.925 * vpw).toString()}px`; +}; +const standAloneRectangleHeight = function () { + console.log("rectangleHeight"); + const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); + return `${(0.825 * vph).toString()}px`; +}; + +const squareWidthHeight = function () { + const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); + const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); + const min = Math.min(vpw, vph); + if (min < 400) { + return `${(0.9 * min).toString()}px`; + } + return `${(0.7 * min).toString()}px`; +}; +const rectangleWidth = function () { + const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); + if (vpw < 400) { + return `${(0.9 * vpw).toString()}px`; + } + return `${(0.9 * vpw).toString()}px`; +}; +const rectangleHeight = function () { + const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); + if (vph < 400) { + return `${(0.8 * vph).toString()}px`; + } + return `${(0.7 * vph).toString()}px`; +}; + // plot width helper used in multiple places const width = function (plotType) { switch (plotType) { @@ -353,48 +398,6 @@ const standAloneHeight = function (plotType) { } }; -const standAloneSquareWidthHeight = function () { - console.log("squareWidthHeight"); - const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); - const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); - const min = Math.min(vpw, vph); - return `${(0.9 * min).toString()}px`; -}; -const standAloneRectangleWidth = function () { - console.log("rectangleWidth"); - const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); - return `${(0.925 * vpw).toString()}px`; -}; -const standAloneRectangleHeight = function () { - console.log("rectangleHeight"); - const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); - return `${(0.825 * vph).toString()}px`; -}; - -const squareWidthHeight = function () { - const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); - const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); - const min = Math.min(vpw, vph); - if (min < 400) { - return `${(0.9 * min).toString()}px`; - } - return `${(0.7 * min).toString()}px`; -}; -const rectangleWidth = function () { - const vpw = Math.min(document.documentElement.clientWidth, window.innerWidth || 0); - if (vpw < 400) { - return `${(0.9 * vpw).toString()}px`; - } - return `${(0.9 * vpw).toString()}px`; -}; -const rectangleHeight = function () { - const vph = Math.min(document.documentElement.clientHeight, window.innerHeight || 0); - if (vph < 400) { - return `${(0.8 * vph).toString()}px`; - } - return `${(0.7 * vph).toString()}px`; -}; - const resizeGraph = function (plotType) { document.getElementById("placeholder").style.width = width(plotType); document.getElementById("placeholder").style.height = height(plotType); @@ -586,13 +589,14 @@ const downloadFile = function (fileURL, fileName) { // for IE < 11 else if (!!window.ActiveXObject && document.execCommand) { - const _window = window.open(fileURL, "_blank"); - _window.document.close(); - _window.document.execCommand("SaveAs", true, fileName || fileURL); - _window.close(); + const thisWindow = window.open(fileURL, "_blank"); + thisWindow.document.close(); + thisWindow.document.execCommand("SaveAs", true, fileName || fileURL); + thisWindow.close(); } }; +// eslint-disable-next-line no-undef export default matsGraphUtils = { setNoDataLabels, setNoDataLabelsMap, diff --git a/meteor_packages/mats-common/imports/startup/client/init.js b/meteor_packages/mats-common/imports/startup/client/init.js index 37f432d60..f9d432ca4 100644 --- a/meteor_packages/mats-common/imports/startup/client/init.js +++ b/meteor_packages/mats-common/imports/startup/client/init.js @@ -6,6 +6,9 @@ import { Meteor } from "meteor/meteor"; import matsCollections from "meteor/randyp:mats-common"; import { curveParamsByApp } from "../both/mats-curve-params"; +/* global Accounts, Session */ +/* eslint-disable no-console */ + if (Meteor.isClient) { const params = curveParamsByApp[Meteor.settings.public.app]; if (!params) { @@ -16,7 +19,7 @@ if (Meteor.isClient) { "curveParams are not defined in imports/startup/both/mats-curve-params.js. Please define some curveParams for this app." ); } - for (let i = 0; i < params.length; i++) { + for (let i = 0; i < params.length; i += 1) { Meteor.subscribe(params[i]); } Meteor.subscribe("Scatter2dParams"); @@ -49,8 +52,8 @@ if (Meteor.isClient) { }, }); - const ref = location.href; - const pathArray = location.href.split("/"); + const ref = window.location.href; + const pathArray = window.location.href.split("/"); const protocol = pathArray[0]; const hostport = pathArray[2]; const hostName = hostport.split(":")[0]; diff --git a/meteor_packages/mats-common/imports/startup/client/routes.js b/meteor_packages/mats-common/imports/startup/client/routes.js index 0c320841b..aca60f44c 100644 --- a/meteor_packages/mats-common/imports/startup/client/routes.js +++ b/meteor_packages/mats-common/imports/startup/client/routes.js @@ -5,6 +5,8 @@ import { Meteor } from "meteor/meteor"; import { FlowRouter } from "meteor/ostrio:flow-router-extra"; +/* global Session */ + // localhost routes FlowRouter.route("/", { @@ -27,14 +29,14 @@ FlowRouter.route("/", { FlowRouter.route("/CSV/:graphFunction/:key/:matching/:appName", { name: "csv", - action(params) { + action() { window.location.href = FlowRouter.path; }, }); FlowRouter.route("/JSON/:graphFunction/:key/:matching/:appName", { name: "json", - action(params) { + action() { window.location.href = FlowRouter.path; }, }); @@ -88,7 +90,7 @@ FlowRouter.route( `${Meteor.settings.public.proxy_prefix_path}/CSV/:graphFunction/:key/:matching/:appName`, { name: "csv", - action(params) { + action() { window.location.href = FlowRouter.path; }, } @@ -98,7 +100,7 @@ FlowRouter.route( `${Meteor.settings.public.proxy_prefix_path}/JSON/:graphFunction/:key/:matching/:appName`, { name: "json", - action(params) { + action() { window.location.href = FlowRouter.path; }, } @@ -162,7 +164,7 @@ FlowRouter.route( `${Meteor.settings.public.proxy_prefix_path}/*/CSV/:graphFunction/:key/:matching/:appName`, { name: "csv", - action(params) { + action() { window.location.href = FlowRouter.path; }, } @@ -172,7 +174,7 @@ FlowRouter.route( `${Meteor.settings.public.proxy_prefix_path}/*/JSON/:graphFunction/:key/:matching/:appName`, { name: "json", - action(params) { + action() { window.location.href = FlowRouter.path; }, } @@ -223,7 +225,6 @@ FlowRouter.route(`${Meteor.settings.public.proxy_prefix_path}/*/`, { FlowRouter.route("/*", { action() { - console.log("route: " + " not found"); this.render("notFound"); }, }); diff --git a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js index 4ef5dde68..71952d4d1 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_plot_ops_util.js @@ -2,7 +2,7 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsCollections, matsTypes, matsDataUtils } from "meteor/randyp:mats-common"; +import { matsCollections, matsTypes, matsMethods } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; import { moment } from "meteor/momentjs:moment"; import { _ } from "meteor/underscore"; @@ -1497,7 +1497,7 @@ const generateGridScaleProbPlotOptions = function (axisMap) { const yPad = (ymax - ymin) * 0.025 !== 0 ? (ymax - ymin) * 0.025 : 0.025; const newYmax = Math.log10(ymax + yPad * 100); const newYmin = - matsDataUtils.isThisANaN(Math.log10(ymin - yPad)) || Math.log10(ymin - yPad) < 1 + matsMethods.isThisANaN(Math.log10(ymin - yPad)) || Math.log10(ymin - yPad) < 1 ? 0 : Math.log10(ymin - yPad); layout.yaxis.range = [newYmin, newYmax]; diff --git a/meteor_packages/mats-common/imports/startup/server/data_process_util.js b/meteor_packages/mats-common/imports/startup/server/data_process_util.js index 7382394f7..090aaa9c4 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_process_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_process_util.js @@ -5,6 +5,7 @@ import { matsTypes, matsCollections, + matsMethods, matsDataUtils, matsDataMatchUtils, matsDataDiffUtils, @@ -125,11 +126,11 @@ const processDataXYCurve = function ( diffFrom === null || !( Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || - !matsDataUtils.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) + !matsMethods.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || - !matsDataUtils.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) + !matsMethods.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_y.array[di] = null; @@ -206,46 +207,46 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsMethods.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsMethods.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsMethods.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsMethods.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsMethods.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsMethods.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.y[di] - errorLength).toPrecision(4)} to ${Number( @@ -255,27 +256,32 @@ const processDataXYCurve = function ( data.stats[di] = { stat: data.y[di], n: - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.y[di], nGood: - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.y[di] === null ? null : data.y[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; @@ -656,11 +662,11 @@ const processDataProfile = function ( diffFrom === null || !( Array.isArray(returnDataset[diffFrom[0]].subHit[di]) || - !matsDataUtils.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) + !matsMethods.isThisANaN(returnDataset[diffFrom[0]].subHit[di]) ) || !( Array.isArray(returnDataset[diffFrom[1]].subHit[di]) || - !matsDataUtils.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) + !matsMethods.isThisANaN(returnDataset[diffFrom[1]].subHit[di]) ) ) { data.error_x.array[di] = null; @@ -696,23 +702,23 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0, hit: - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null, fa: - Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsMethods.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null, miss: - Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsMethods.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null, cn: - Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsMethods.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null, }; @@ -721,23 +727,23 @@ const processDataProfile = function ( `
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? data.subHit[di].length : 0 }
Hits: ${ - Array.isArray(data.subHit[di]) || !matsDataUtils.isThisANaN(data.subHit[di]) + Array.isArray(data.subHit[di]) || !matsMethods.isThisANaN(data.subHit[di]) ? matsDataUtils.sum(data.subHit[di]) : null }
False alarms: ${ - Array.isArray(data.subFa[di]) || !matsDataUtils.isThisANaN(data.subFa[di]) + Array.isArray(data.subFa[di]) || !matsMethods.isThisANaN(data.subFa[di]) ? matsDataUtils.sum(data.subFa[di]) : null }
Misses: ${ - Array.isArray(data.subMiss[di]) || !matsDataUtils.isThisANaN(data.subMiss[di]) + Array.isArray(data.subMiss[di]) || !matsMethods.isThisANaN(data.subMiss[di]) ? matsDataUtils.sum(data.subMiss[di]) : null }
Correct Nulls: ${ - Array.isArray(data.subCn[di]) || !matsDataUtils.isThisANaN(data.subCn[di]) + Array.isArray(data.subCn[di]) || !matsMethods.isThisANaN(data.subCn[di]) ? matsDataUtils.sum(data.subCn[di]) : null }
Errorbars: ${Number(data.x[di] - errorLength).toPrecision( @@ -747,27 +753,32 @@ const processDataProfile = function ( data.stats[di] = { stat: data.x[di], n: - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, raw_stat: data.x[di], nGood: - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0, avgInterest: - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null, }; data.text[di] = `${data.text[di]}
${statisticSelect}: ${ data.x[di] === null ? null : data.x[di].toPrecision(4) }
n: ${ - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? data.subInterest[di].length : 0 }
Average Interest: ${ - Array.isArray(data.subInterest[di]) || !matsDataUtils.isThisANaN(data.subInterest[di]) + Array.isArray(data.subInterest[di]) || + !matsMethods.isThisANaN(data.subInterest[di]) ? matsDataUtils.average(data.subInterest[di]).toPrecision(4) : null }`; diff --git a/meteor_packages/mats-common/imports/startup/server/data_query_util.js b/meteor_packages/mats-common/imports/startup/server/data_query_util.js index 061270e15..17c5ae4ff 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_query_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_query_util.js @@ -2,7 +2,12 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsDataUtils, matsTypes, matsCollections } from "meteor/randyp:mats-common"; +import { + matsDataUtils, + matsTypes, + matsCollections, + matsMethods, +} from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; import { _ } from "meteor/underscore"; @@ -391,7 +396,7 @@ const parseQueryDataXYCurve = function ( const n = rows[rowIndex].sub_data.toString().split(",").length; if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; + stat = matsMethods.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -417,7 +422,7 @@ const parseQueryDataXYCurve = function ( absSum, statisticStr ); - stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; + stat = matsMethods.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -470,7 +475,7 @@ const parseQueryDataXYCurve = function ( if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -508,7 +513,7 @@ const parseQueryDataXYCurve = function ( } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -552,7 +557,7 @@ const parseQueryDataXYCurve = function ( } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1024,7 +1029,7 @@ const parseQueryDataReliability = function (rows, d, appParams, kernel) { currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1194,7 +1199,7 @@ const parseQueryDataPerformanceDiagram = function (rows, d, appParams) { currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1379,7 +1384,7 @@ const parseQueryDataSimpleScatter = function ( absSumX, statisticXStr ); - xStat = matsDataUtils.isThisANaN(Number(xStat)) ? null : xStat; + xStat = matsMethods.isThisANaN(Number(xStat)) ? null : xStat; yStat = matsDataUtils.calculateStatScalar( squareDiffSumY, NSumY, @@ -1389,7 +1394,7 @@ const parseQueryDataSimpleScatter = function ( absSumY, statisticYStr ); - yStat = matsDataUtils.isThisANaN(Number(yStat)) ? null : yStat; + yStat = matsMethods.isThisANaN(Number(yStat)) ? null : yStat; } else { xStat = null; yStat = null; @@ -1429,7 +1434,7 @@ const parseQueryDataSimpleScatter = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1722,7 +1727,7 @@ const parseQueryDataMapScalar = function ( absSum, `${statistic}_${variable}` ); - queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; + queryVal = matsMethods.isThisANaN(Number(queryVal)) ? null : queryVal; } else { queryVal = null; } @@ -1752,7 +1757,7 @@ const parseQueryDataMapScalar = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -1830,7 +1835,7 @@ const parseQueryDataMapScalar = function ( absSumSum, `${statistic}_${variable}` ); - queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; + queryVal = matsMethods.isThisANaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database e.message = `Error in parseQueryDataMapScalar. The expected fields don't seem to be present in the results cache: ${e.message}`; @@ -2007,7 +2012,7 @@ const parseQueryDataMapCTC = function ( const n = rows[rowIndex].nTimes; if (hit + fa + miss + cn > 0) { queryVal = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statistic); - queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; + queryVal = matsMethods.isThisANaN(Number(queryVal)) ? null : queryVal; switch (statistic) { case "PODy (POD of value < threshold)": case "PODy (POD of value > threshold)": @@ -2068,7 +2073,7 @@ const parseQueryDataMapCTC = function ( currSubData = thisSubData[sdIdx].split(";"); thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2135,7 +2140,7 @@ const parseQueryDataMapCTC = function ( thisSubHit.length, statistic ); - queryVal = matsDataUtils.isThisANaN(Number(queryVal)) ? null : queryVal; + queryVal = matsMethods.isThisANaN(Number(queryVal)) ? null : queryVal; } catch (e) { // this is an error produced by a bug in the query function, not an error returned by the mysql database e.message = `Error in parseQueryDataMapCTC. The expected fields don't seem to be present in the results cache: ${e.message}`; @@ -2325,7 +2330,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { const n = rows[rowIndex].sub_data.toString().split(",").length; if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; + stat = matsMethods.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -2351,7 +2356,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { absSum, statisticStr ); - stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; + stat = matsMethods.isThisANaN(Number(stat)) ? null : stat; } else { stat = null; } @@ -2376,7 +2381,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2406,7 +2411,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2438,7 +2443,7 @@ const parseQueryDataHistogram = function (rows, d, appParams, statisticStr) { } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2590,7 +2595,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { cn = Number(rows[rowIndex].cn); if (hit + fa + miss + cn > 0) { stat = matsDataUtils.calculateStatCTC(hit, fa, miss, cn, n, statisticStr); - stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; + stat = matsMethods.isThisANaN(Number(stat)) ? null : stat; } } else if ( rows[rowIndex].stat === undefined && @@ -2614,7 +2619,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { absSum, statisticStr ); - stat = matsDataUtils.isThisANaN(Number(stat)) ? null : stat; + stat = matsMethods.isThisANaN(Number(stat)) ? null : stat; const variable = statisticStr.split("_")[1]; stdev = matsDataUtils.calculateStatScalar( squareDiffSum, @@ -2667,7 +2672,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { if (isCTC) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2685,7 +2690,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { } else if (isScalar) { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); @@ -2707,7 +2712,7 @@ const parseQueryDataContour = function (rows, d, appParams, statisticStr) { } else { thisSubSecs.push(Number(currSubData[0])); if (hasLevels) { - if (!matsDataUtils.isThisANaN(Number(currSubData[1]))) { + if (!matsMethods.isThisANaN(Number(currSubData[1]))) { thisSubLevs.push(Number(currSubData[1])); } else { thisSubLevs.push(currSubData[1]); diff --git a/meteor_packages/mats-common/imports/startup/server/data_util.js b/meteor_packages/mats-common/imports/startup/server/data_util.js index 31a63e32f..677d14bb7 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_util.js @@ -2,19 +2,13 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsTypes, matsCollections } from "meteor/randyp:mats-common"; +import { matsTypes, matsCollections, matsMethods } from "meteor/randyp:mats-common"; import { Meteor } from "meteor/meteor"; import { HTTP } from "meteor/jkuester:http"; /* eslint-disable global-require */ /* eslint-disable no-console */ -// wrapper for NaN check -const isThisANaN = function (val) { - // eslint-disable-next-line no-restricted-globals - return !val || isNaN(val); -}; - // this function checks if two JSON objects are identical const areObjectsEqual = function (o, p) { if ((o && !p) || (p && !o)) { @@ -284,26 +278,12 @@ const doColorScheme = function () { "rgb(255,0,0)", "rgb(0,0,255)", "rgb(255,165,0)", - "rgb(128,128,128)", - "rgb(238,130,238)", - + "rgb(95,95,95)", "rgb(238,130,238)", "rgb(0,0,139)", "rgb(148,0,211)", - "rgb(105,105,105)", - "rgb(255,140,0)", - - "rgb(235,92,92)", - "rgb(82,92,245)", - "rgb(133,143,143)", - "rgb(235,143,92)", + "rgb(135,135,135)", "rgb(190,120,120)", - - "rgb(225,82,92)", - "rgb(72,82,245)", - "rgb(123,133,143)", - "rgb(225,133,92)", - "rgb(180,120,120)", ], }); } @@ -465,7 +445,12 @@ const callMetadataAPI = function ( // calculates the statistic for ctc plots const calculateStatCTC = function (hit, fa, miss, cn, n, statistic) { - if (isThisANaN(hit) || isThisANaN(fa) || isThisANaN(miss) || isThisANaN(cn)) + if ( + matsMethods.isThisANaN(hit) || + matsMethods.isThisANaN(fa) || + matsMethods.isThisANaN(miss) || + matsMethods.isThisANaN(cn) + ) return null; let queryVal; switch (statistic) { @@ -547,12 +532,12 @@ const calculateStatScalar = function ( statisticAndVariable ) { if ( - isThisANaN(squareDiffSum) || - isThisANaN(NSum) || - isThisANaN(obsModelDiffSum) || - isThisANaN(modelSum) || - isThisANaN(obsSum) || - isThisANaN(absSum) + matsMethods.isThisANaN(squareDiffSum) || + matsMethods.isThisANaN(NSum) || + matsMethods.isThisANaN(obsModelDiffSum) || + matsMethods.isThisANaN(modelSum) || + matsMethods.isThisANaN(obsSum) || + matsMethods.isThisANaN(absSum) ) return null; let queryVal; @@ -585,7 +570,7 @@ const calculateStatScalar = function ( default: queryVal = null; } - if (isThisANaN(queryVal)) return null; + if (matsMethods.isThisANaN(queryVal)) return null; // need to convert to correct units for surface data but not upperair if (statistic !== "N") { if ( @@ -1198,7 +1183,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { let secs; let delta; for (i = 0; i < sSecs.length; i += 1) { - if (isThisANaN(sVals[i])) { + if (matsMethods.isThisANaN(sVals[i])) { n -= 1; } else { secs = sSecs[i]; @@ -1221,7 +1206,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { console.log(`matsDataUtil.getErr: ${error}`); } for (i = 0; i < sVals.length; i += 1) { - if (!isThisANaN(sVals[i])) { + if (!matsMethods.isThisANaN(sVals[i])) { minVal = minVal < sVals[i] ? minVal : sVals[i]; maxVal = maxVal > sVals[i] ? maxVal : sVals[i]; dataSum += sVals[i]; @@ -1254,7 +1239,7 @@ const getErr = function (sVals, sSecs, sLevs, appParams) { let nDeltas = 0; for (i = 0; i < sSecs.length; i += 1) { - if (!isThisANaN(sVals[i])) { + if (!matsMethods.isThisANaN(sVals[i])) { let sec = sSecs[i]; if (typeof sec === "string" || sec instanceof String) sec = Number(sec); let lev; @@ -1551,7 +1536,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins": // get the user's chosen number of bins binNum = Number(plotParams["bin-number"]); - if (isThisANaN(binNum)) { + if (matsMethods.isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1566,7 +1551,7 @@ const setHistogramParameters = function (plotParams) { case "Choose a bin bound": // let the histogram routine know that we want the bins shifted over to whatever was input pivotVal = Number(plotParams["bin-pivot"]); - if (isThisANaN(pivotVal)) { + if (matsMethods.isThisANaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1574,7 +1559,7 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and make zero a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to zero binNum = Number(plotParams["bin-number"]); - if (isThisANaN(binNum)) { + if (matsMethods.isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); @@ -1585,13 +1570,13 @@ const setHistogramParameters = function (plotParams) { case "Set number of bins and choose a bin bound": // get the user's chosen number of bins and let the histogram routine know that we want the bins shifted over to whatever was input binNum = Number(plotParams["bin-number"]); - if (isThisANaN(binNum)) { + if (matsMethods.isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } pivotVal = Number(plotParams["bin-pivot"]); - if (isThisANaN(pivotVal)) { + if (matsMethods.isThisANaN(pivotVal)) { throw new Error("Error parsing bin pivot: please enter the desired bin pivot."); } break; @@ -1602,7 +1587,7 @@ const setHistogramParameters = function (plotParams) { binBounds = plotParams["bin-bounds"].split(",").map(function (item) { let thisItem = item.trim(); thisItem = Number(thisItem); - if (!isThisANaN(thisItem)) { + if (!matsMethods.isThisANaN(thisItem)) { return thisItem; } throw new Error( @@ -1626,17 +1611,17 @@ const setHistogramParameters = function (plotParams) { case "Manual bin start, number, and stride": // get the bin start, number, and stride. binNum = Number(plotParams["bin-number"]); - if (isThisANaN(binNum)) { + if (matsMethods.isThisANaN(binNum)) { throw new Error( "Error parsing bin number: please enter the desired number of bins." ); } binStart = Number(plotParams["bin-start"]); - if (isThisANaN(binStart)) { + if (matsMethods.isThisANaN(binStart)) { throw new Error("Error parsing bin start: please enter the desired bin start."); } binStride = Number(plotParams["bin-stride"]); - if (isThisANaN(binStride)) { + if (matsMethods.isThisANaN(binStride)) { throw new Error( "Error parsing bin stride: please enter the desired bin stride." ); @@ -1715,7 +1700,7 @@ const calculateHistogramBins = function ( binLowBounds[binParams.binNum - 1] = fullUpBound; binMeans[binParams.binNum - 1] = fullUpBound + binInterval / 2; - if (binParams.pivotVal !== undefined && !isThisANaN(binParams.pivotVal)) { + if (binParams.pivotVal !== undefined && !matsMethods.isThisANaN(binParams.pivotVal)) { // need to shift the bounds and means over so that one of the bounds is on the chosen pivot const closestBoundToPivot = binLowBounds.reduce(function (prev, curr) { return Math.abs(curr - binParams.pivotVal) < Math.abs(prev - binParams.pivotVal) @@ -2187,7 +2172,6 @@ export default matsDataUtils = { average, median, stdev, - isThisANaN, dateConvert, getDateRange, secsConvert, diff --git a/tests/src/steps/given.js b/tests/src/steps/given.js deleted file mode 100644 index 010114335..000000000 --- a/tests/src/steps/given.js +++ /dev/null @@ -1,95 +0,0 @@ -import { Given } from "cucumber"; - -import checkContainsAnyText from "../support/check/checkContainsAnyText"; -import checkIsEmpty from "../support/check/checkIsEmpty"; -import checkContainsText from "../support/check/checkContainsText"; -import checkCookieContent from "../support/check/checkCookieContent"; -import checkCookieExists from "../support/check/checkCookieExists"; -import checkDimension from "../support/check/checkDimension"; -import checkElementExists from "../support/check/checkElementExists"; -import checkEqualsText from "../support/check/checkEqualsText"; -import checkModal from "../support/check/checkModal"; -import checkOffset from "../support/check/checkOffset"; -import checkProperty from "../support/check/checkProperty"; -import checkSelected from "../support/check/checkSelected"; -import checkTitle from "../support/check/checkTitle"; -import checkUrl from "../support/check/checkURL"; -import closeAllButFirstTab from "../support/action/closeAllButFirstTab"; -import compareText from "../support/check/compareText"; -import isEnabled from "../support/check/isEnabled"; -import isDisplayed from "../support/check/isDisplayed"; -import openWebsite from "../support/action/openWebsite"; -import setWindowSize from "../support/action/setWindowSize"; -import openWebsiteAndWaitForPlotType from "../support/action/openWebsiteAndWaitForPlotType"; -import saveMatsParameters from "../support/action/saveMatsParameters"; - -Given(/^I remember the parameter values$/, saveMatsParameters); - -Given(/^I open the (url|site) "([^"]*)?"$/, openWebsite); - -Given( - /^I load the app "([^"]*)?"$/, - { wrapperOptions: { retry: 2 } }, - openWebsiteAndWaitForPlotType -); - -Given(/^the element "([^"]*)?" is( not)* displayed$/, isDisplayed); - -Given(/^the element "([^"]*)?" is( not)* enabled$/, isEnabled); - -Given(/^the element "([^"]*)?" is( not)* selected$/, checkSelected); - -Given(/^the checkbox "([^"]*)?" is( not)* checked$/, checkSelected); - -Given(/^there is (an|no) element "([^"]*)?" on the page$/, checkElementExists); - -Given(/^the title is( not)* "([^"]*)?"$/, checkTitle); - -Given( - /^the element "([^"]*)?" contains( not)* the same text as element "([^"]*)?"$/, - compareText -); - -Given( - /^the (button|element) "([^"]*)?"( not)* matches the text "([^"]*)?"$/, - checkEqualsText -); - -Given( - /^the (button|element|container) "([^"]*)?"( not)* contains the text "([^"]*)?"$/, - checkContainsText -); - -Given( - /^the (button|element) "([^"]*)?"( not)* contains any text$/, - checkContainsAnyText -); - -Given(/^the (button|element) "([^"]*)?" is( not)* empty$/, checkIsEmpty); - -Given(/^the page url is( not)* "([^"]*)?"$/, checkUrl); - -Given( - /^the( css)* attribute "([^"]*)?" from element "([^"]*)?" is( not)* "([^"]*)?"$/, - checkProperty -); - -Given( - /^the cookie "([^"]*)?" contains( not)* the value "([^"]*)?"$/, - checkCookieContent -); - -Given(/^the cookie "([^"]*)?" does( not)* exist$/, checkCookieExists); - -Given(/^the element "([^"]*)?" is( not)* ([\d]+)px (broad|tall)$/, checkDimension); - -Given( - /^the element "([^"]*)?" is( not)* positioned at ([\d]+)px on the (x|y) axis$/, - checkOffset -); - -Given(/^I have a screen that is ([\d]+) by ([\d]+) pixels$/, setWindowSize); - -Given(/^I have closed all but the first (window|tab)$/, closeAllButFirstTab); - -Given(/^a (alertbox|confirmbox|prompt) is( not)* opened$/, checkModal); diff --git a/tests/src/steps/then.js b/tests/src/steps/then.js deleted file mode 100644 index 513b33753..000000000 --- a/tests/src/steps/then.js +++ /dev/null @@ -1,220 +0,0 @@ -import checkClass from "../support/check/checkClass"; -import checkContainsAnyText from "../support/check/checkContainsAnyText"; -import checkIsEmpty from "../support/check/checkIsEmpty"; -import checkContainsText from "../support/check/checkContainsText"; -import checkCookieContent from "../support/check/checkCookieContent"; -import checkCookieExists from "../support/check/checkCookieExists"; -import checkDimension from "../support/check/checkDimension"; -import checkEqualsText from "../support/check/checkEqualsText"; -import checkFocus from "../support/check/checkFocus"; -import checkInURLPath from "../support/check/checkInURLPath"; -import checkIsOpenedInNewWindow from "../support/check/checkIsOpenedInNewWindow"; -import checkModal from "../support/check/checkModal"; -import checkModalText from "../support/check/checkModalText"; -import checkNewWindow from "../support/check/checkNewWindow"; -import checkOffset from "../support/check/checkOffset"; -import checkProperty from "../support/check/checkProperty"; -import checkFontProperty from "../support/check/checkFontProperty"; -import checkSelected from "../support/check/checkSelected"; -import checkMatsDatesValue from "../support/check/checkMatsDatesValue"; -import checkMatsCurveDatesValue from "../support/check/checkMatsCurveDatesValue"; -import checkTitle from "../support/check/checkTitle"; -import checkTitleContains from "../support/check/checkTitleContains"; -import checkURL from "../support/check/checkURL"; -import checkURLPath from "../support/check/checkURLPath"; -import checkWithinViewport from "../support/check/checkWithinViewport"; -import compareText from "../support/check/compareText"; -import isEnabled from "../support/check/isEnabled"; -import isExisting from "../support/check/isExisting"; -import isVisible from "../support/check/isDisplayed"; -import waitFor from "../support/action/waitFor"; -import waitForVisible from "../support/action/waitForDisplayed"; -import checkIfElementExists from "../support/lib/checkIfElementExists"; -import checkParameterValue from "../support/check/checkParameterValue"; -import checkMatsAppTitle from "../support/check/checkMatsAppTitle"; -import checkMatsCurveIsAdded from "../support/check/checkMatsCurveIsAdded"; -import checkMatsGraphPlotType from "../support/check/checkMatsGraphPlotType"; -import checkMatsCurveNumber from "../support/check/checkMatsCurveNumber"; -import checkMatsPlotNumber from "../support/check/checkMatsPlotNumber"; -import checkMatsCurveListContains from "../support/check/checkMatsCurveListContains"; -import checkMatsLegendListContains from "../support/check/checkMatsLegendListContains"; -import isGraphDisplayed from "../support/check/isMatsGraphDisplayed"; -import isMainDisplayed from "../support/check/isMainDisplayed"; -import isMatsButtonVisible from "../support/check/isMatsButtonVisible"; -import isMatsPlotType from "../support/check/isMatsPlotType"; -import isMatsPlotFormat from "../support/check/isMatsPlotFormat"; -import isMatsSelectedOption from "../support/check/isMatsSelectedOption"; -import isMatsCurveColor from "../support/check/isMatsCurveColor"; -import checkMatsParameters from "../support/check/checkMatsParameters"; -import isMatsInfoVisible from "../support/check/isMatsInfoVisible"; -import checkMatsInfoMessage from "../support/check/checkMatsInfoMessage"; -import matsDebug from "../support/action/matsDebug"; -import isMatsButtonEnabled from "../support/check/isMatsButtonEnabled"; - -const { Then } = require("cucumber"); - -Then(/^I debug$/, matsDebug); - -Then(/^the "info" dialog should( not)* be visible$/, isMatsInfoVisible); - -Then(/^the "([^"]*)" button should( not)* be enabled$/, isMatsButtonEnabled); - -Then(/^I should see "([^"]*)" in the "info" dialog$/, checkMatsInfoMessage); - -Then(/^the parameter values should remain unchanged$/, checkMatsParameters); - -Then(/^the "([^"]*)" color should be "([^"]*)"$/, isMatsCurveColor); - -Then(/^the "([^"]*)?" parameter value matches "([^"]*)?"$/, isMatsSelectedOption); - -Then(/^the "([^"]*)?" button should be visible$/, isMatsButtonVisible); - -Then(/^"([^"]*)?" is added$/, checkMatsCurveIsAdded); - -Then(/^I should be on the graph page$/, isGraphDisplayed); - -Then(/^I should be on the main page$/, isMainDisplayed); - -Then(/^I should have a "([^"]*)?" plot$/, checkMatsGraphPlotType); - -Then(/^the plot type should be "([^"]*)?"$/, isMatsPlotType); - -Then(/^the plot format should be "([^"]*)?"$/, isMatsPlotFormat); - -Then(/^I should have (\d+) curve.*$/, checkMatsCurveNumber); - -Then(/^I should have (\d+) trace.*$/, checkMatsPlotNumber); - -Then(/^I expect the app title to be "([^"]*)?"$/, checkMatsAppTitle); - -Then( - /^I should see a list of curves containing "([^"]*)?"$/, - checkMatsCurveListContains -); - -Then( - /^I should see a list of legends containing "([^"]*)?"$/, - checkMatsLegendListContains -); - -Then( - /^I expect that the "([^"]*)?" parameter value matches "([^"]*)?"$/, - checkParameterValue -); - -Then(/^the dates value is "([^"]*)?"$/, checkMatsDatesValue); - -Then(/^the curve-dates value is "([^"]*)?"$/, checkMatsCurveDatesValue); - -Then(/^the plot type should be "([^"]*)?"&/, isMatsPlotType); -Then(/^I expect that the title is( not)* "([^"]*)?"$/, checkTitle); - -Then(/^I expect that the title( not)* contains "([^"]*)?"$/, checkTitleContains); - -Then( - /^I expect that element "([^"]*)?" does( not)* appear exactly "([^"]*)?" times$/, - checkIfElementExists -); - -Then(/^I expect that element "([^"]*)?" is( not)* displayed$/, isVisible); - -Then(/^I expect that element "([^"]*)?" becomes( not)* displayed$/, waitForVisible); - -Then( - /^I expect that element "([^"]*)?" is( not)* within the viewport$/, - checkWithinViewport -); - -Then(/^I expect that element "([^"]*)?" does( not)* exist$/, isExisting); - -Then( - /^I expect that element "([^"]*)?"( not)* contains the same text as element "([^"]*)?"$/, - compareText -); - -Then( - /^I expect that (button|element) "([^"]*)?"( not)* matches the text "([^"]*)?"$/, - checkEqualsText -); - -Then( - /^I expect that (button|element|container) "([^"]*)?"( not)* contains the text "([^"]*)?"$/, - checkContainsText -); - -Then( - /^I expect that (button|element) "([^"]*)?"( not)* contains any text$/, - checkContainsAnyText -); - -Then(/^I expect that (button|element) "([^"]*)?" is( not)* empty$/, checkIsEmpty); - -Then(/^I expect that the url is( not)* "([^"]*)?"$/, checkURL); - -Then(/^I expect that the path is( not)* "([^"]*)?"$/, checkURLPath); - -Then(/^I expect the url to( not)* contain "([^"]*)?"$/, checkInURLPath); - -Then( - /^I expect that the( css)* attribute "([^"]*)?" from element "([^"]*)?" is( not)* "([^"]*)?"$/, - checkProperty -); - -Then( - /^I expect that the font( css)* attribute "([^"]*)?" from element "([^"]*)?" is( not)* "([^"]*)?"$/, - checkFontProperty -); - -Then(/^I expect that checkbox "([^"]*)?" is( not)* checked$/, checkSelected); - -Then(/^I expect that element "([^"]*)?" is( not)* selected$/, checkSelected); - -Then(/^I expect that element "([^"]*)?" is( not)* enabled$/, isEnabled); - -Then( - /^I expect that cookie "([^"]*)?"( not)* contains "([^"]*)?"$/, - checkCookieContent -); - -Then(/^I expect that cookie "([^"]*)?"( not)* exists$/, checkCookieExists); - -Then( - /^I expect that element "([^"]*)?" is( not)* ([\d]+)px (broad|tall)$/, - checkDimension -); - -Then( - /^I expect that element "([^"]*)?" is( not)* positioned at ([\d+.?\d*]+)px on the (x|y) axis$/, - checkOffset -); - -Then( - /^I expect that element "([^"]*)?" (has|does not have) the class "([^"]*)?"$/, - checkClass -); - -Then(/^I expect a new (window|tab) has( not)* been opened$/, checkNewWindow); - -Then( - /^I expect the url "([^"]*)?" is opened in a new (tab|window)$/, - checkIsOpenedInNewWindow -); - -Then(/^I expect that element "([^"]*)?" is( not)* focused$/, checkFocus); - -Then( - /^I wait on element "([^"]*)?"(?: for (\d+)ms)*(?: to( not)* (be checked|be enabled|be selected|be displayed|contain a text|contain a value|exist))*$/, - { - wrapperOptions: { - retry: 3, - }, - }, - waitFor -); - -Then(/^I expect that a (alertbox|confirmbox|prompt) is( not)* opened$/, checkModal); - -Then( - /^I expect that a (alertbox|confirmbox|prompt)( not)* contains the text "([^"]*)?"$/, - checkModalText -); diff --git a/tests/src/steps/when.js b/tests/src/steps/when.js deleted file mode 100644 index 770fe5094..000000000 --- a/tests/src/steps/when.js +++ /dev/null @@ -1,110 +0,0 @@ -import clearInputField from "../support/action/clearInputField"; -import clickElement from "../support/action/clickElement"; -import closeLastOpenedWindow from "../support/action/closeLastOpenedWindow"; -import deleteCookies from "../support/action/deleteCookies"; -import dragElement from "../support/action/dragElement"; -import focusLastOpenedWindow from "../support/action/focusLastOpenedWindow"; -import handleModal from "../support/action/handleModal"; -import moveTo from "../support/action/moveTo"; -import pause from "../support/action/pause"; -import pressButton from "../support/action/pressButton"; -import scroll from "../support/action/scroll"; -import selectOption from "../support/action/selectOption"; -import selectOptionByIndex from "../support/action/selectOptionByIndex"; -import setCookie from "../support/action/setCookie"; -import setInputField from "../support/action/setInputField"; -import setPromptText from "../support/action/setPromptText"; - -import setPredefinedDateRange from "../support/action/setPredefinedDateRange"; -import setMatsDateRange from "../support/action/setMatsDateRange"; -import setMatsCurveDateRange from "../support/action/setMatsCurveDateRange"; -import setScatterDateRange from "../support/action/setScatterDateRange"; -import setMatsPlotType from "../support/action/setMatsPlotType"; -import setMatsPlotFormat from "../support/action/setMatsPlotFormat"; -import clickMatsButton from "../support/action/clickMatsButton"; -import setMatsSelectedOption from "../support/action/setMatsSelectedOption"; -import matsRefreshBrowser from "../support/action/matsRefreshBrowser"; -import matsRefreshPage from "../support/action/matsRefreshPage"; -import matsClearParameter from "../support/action/matsClearParameter"; - -const { When } = require("cucumber"); - -When(/^I clear the "([^"]*)?" parameter$/, matsClearParameter); - -When(/^I refresh the browser$/, matsRefreshBrowser); - -When(/^I refresh the page$/, matsRefreshPage); - -When(/^I click the "([^"]*)?" radio button$/, setMatsPlotFormat); - -When(/^I click the "([^"]*)?" button$/, clickMatsButton); - -When( - /^I change the "([^"]*)" parameter to "([^"]*)"$/, - { wrapperOptions: { retry: 2 } }, - setMatsSelectedOption -); - -When(/^I set the plot type to "([^"]*)?"$/, setMatsPlotType); - -When( - /^I choose a predefined "([^"]*)?" range of "([^"]*)?"$/, - { wrapperOptions: { retry: 2 } }, - setPredefinedDateRange -); - -When( - /^I set the dates to "([^"]*)?"$/, - { wrapperOptions: { retry: 2 } }, - setMatsDateRange -); - -When( - /^I set the curve-dates to "([^"]*)?"$/, - { wrapperOptions: { retry: 2 } }, - setMatsCurveDateRange -); - -When( - /^I set the scatter dates to "([^"]*)?"$/, - { wrapperOptions: { retry: 2 } }, - setScatterDateRange -); - -When(/^I (click|doubleclick) on the (link|button|element) "([^"]*)?"$/, clickElement); - -When(/^I (add|set) "([^"]*)?" to the inputfield "([^"]*)?"$/, setInputField); - -When(/^I clear the inputfield "([^"]*)?"$/, clearInputField); - -When(/^I drag element "([^"]*)?" to element "([^"]*)?"$/, dragElement); - -When(/^I pause for (\d+)ms$/, pause); - -When(/^I set a cookie "([^"]*)?" with the content "([^"]*)?"$/, setCookie); - -When(/^I delete the cookie "([^"]*)?"$/, deleteCookies); - -When(/^I press "([^"]*)?"$/, pressButton); - -When(/^I (accept|dismiss) the (alertbox|confirmbox|prompt)$/, handleModal); - -When(/^I enter "([^"]*)?" into the prompt$/, setPromptText); - -When(/^I scroll to element "([^"]*)?"$/, scroll); - -When(/^I close the last opened (window|tab)$/, closeLastOpenedWindow); - -When(/^I focus the last opened (window|tab)$/, focusLastOpenedWindow); - -When( - /^I select the (\d+)(st|nd|rd|th) option for element "([^"]*)?"$/, - selectOptionByIndex -); - -When( - /^I select the option with the (name|value|text) "([^"]*)?" for element "([^"]*)?"$/, - selectOption -); - -When(/^I move to element "([^"]*)?"(?: with an offset of (\d+),(\d+))*$/, moveTo); diff --git a/tests/src/support/action/clearInputField.js b/tests/src/support/action/clearInputField.js deleted file mode 100644 index a5d5f81bf..000000000 --- a/tests/src/support/action/clearInputField.js +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Clear a given input field (placeholder for WDIO's clearElement) - * @param {String} selector Element selector - */ -export default (selector) => { - $(selector).clearValue(); -}; diff --git a/tests/src/support/action/clickElement.js b/tests/src/support/action/clickElement.js deleted file mode 100644 index 0ae44d390..000000000 --- a/tests/src/support/action/clickElement.js +++ /dev/null @@ -1,25 +0,0 @@ -import checkIfElementExists from "../lib/checkIfElementExists"; - -/** - * Perform an click action on the given element - * @param {String} action The action to perform (click or doubleClick) - * @param {String} type Type of the element (link or selector) - * @param {String} selector Element selector - */ -export default (action, type, selector) => { - /** - * Element to perform the action on - * @type {String} - */ - const selector2 = type === "link" ? `=${selector}` : selector; - - /** - * The method to call on the browser object - * @type {String} - */ - const method = action === "click" ? "click" : "doubleClick"; - - checkIfElementExists(selector2); - - $(selector2)[method](); -}; diff --git a/tests/src/support/action/clickMatsButton.js b/tests/src/support/action/clickMatsButton.js deleted file mode 100644 index d182c4d86..000000000 --- a/tests/src/support/action/clickMatsButton.js +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Perform an click action on the button with the given label - * @param {String} label button label - */ -export default (label) => { - /** - * LabelOf the Mats button to click - * @type {String} - */ - let selector; - let nrOfElements; - let cPart; - // special buttons first - if (label === "Remove All") { - nrOfElements = $$("#remove-all").length; - $("#remove-all").waitForClickable(); - $("#remove-all").click(); - } else if (label === "Remove all the curves") { - nrOfElements = $$("#confirm-remove-all").length; - // I don't know why there are two of the confirm buttons - // there has to be a better way to handle this - $$("#confirm-remove-all")[1].waitForClickable(); - $$("#confirm-remove-all")[1].click(); - } else if (label.match("Remove curve .*")) { - // this is the 'Remove curve curvelabel' confirm button - cPart = label.replace("Remove curve ", ""); - nrOfElements = $$(`#confirm-remove-curve*=${cPart}`).length; - expect(nrOfElements).toBeGreaterThan( - 0, - `Expected an "${selector}" button to exist` - ); - $$(`#confirm-remove-curve*=${cPart}`)[1].waitForDisplayed(); - $$(`#confirm-remove-curve*=${cPart}`)[1].waitForClickable(); - $$(`#confirm-remove-curve*=${cPart}`)[1].click(); - } else if (label.match("Remove .*")) { - // This is the 'Remove curvelabel' remove button - cPart = label.replace("Remove ", ""); - nrOfElements = $$(`#curve-list-remove*=${cPart}`).length; - expect(nrOfElements).toBeGreaterThan( - 0, - `Expected an "${selector}" button to exist` - ); - $(`#curve-list-remove*=${cPart}`).waitForClickable(); - $(`#curve-list-remove*=${cPart}`).click(); - } else { - // normal buttons - switch (label) { - case "Add Curve": - selector = $("#add"); - selector.waitForDisplayed(); - nrOfElements = $$("#add").length; - break; - case "Back": - selector = $("#backButton"); - selector.waitForDisplayed(); - nrOfElements = $$("#backButton").length; - break; - case "Plot Unmatched": - selector = $("#plotUnmatched"); - selector.waitForDisplayed(); - nrOfElements = $$("#plotUnmatched").length; - break; - case "Plot Matched": - selector = $("#plotMatched"); - selector.waitForDisplayed(); - nrOfElements = $$("#plotMatched").length; - break; - case "Reset to Defaults": - selector = $("#reset"); - selector.waitForDisplayed(); - nrOfElements = $$("#reset").length; - break; - case "Clear": - selector = $("#clear-info"); - selector.waitForDisplayed(); - nrOfElements = $$("#clear-info").length; - break; - default: - throw new Error("Unhandled button label???"); - } - // these are for the switch statement i.e. 'normal buttons' - expect(nrOfElements).toBeGreaterThan( - 0, - `Expected an "${selector}" button to exist` - ); - selector.waitForClickable(); - selector.click(); - } -}; diff --git a/tests/src/support/action/clickMatsButtonReturnImmediate.js b/tests/src/support/action/clickMatsButtonReturnImmediate.js deleted file mode 100644 index 78d223872..000000000 --- a/tests/src/support/action/clickMatsButtonReturnImmediate.js +++ /dev/null @@ -1,85 +0,0 @@ -/** - * Perform an click action on the button with the given label - * @param {String} label button label - */ -export default (label) => { - /** - * LabelOf the Mats button to click - * @type {String} - */ - const method = "click"; - let selector; - let nrOfElements; - let cPart; - // special buttons first - if (label === "Remove All") { - nrOfElements = $$("#remove-all").length; - $("#remove-all")[method](); - } else if (label === "Remove all the curves") { - nrOfElements = $$("#confirm-remove-all").length; - // I don't know why there are two of the confirm buttons - // there has to be a better way to handle this - $$("#confirm-remove-all")[1][method](); - } else if (label.match("Remove curve .*")) { - // this is the 'Remove curve curvelabel' confirm button - cPart = label.replace("Remove curve ", ""); - nrOfElements = $$(`#confirm-remove-curve*=${cPart}`).length; - expect(nrOfElements).toBeGreaterThan( - 0, - `Expected an "${selector}" button to exist` - ); - $$(`#confirm-remove-curve*=${cPart}`)[1].waitForDisplayed(); - $$(`#confirm-remove-curve*=${cPart}`)[1].click(); - } else if (label.match("Remove .*")) { - // This is the 'Remove curvelabel' remove button - cPart = label.replace("Remove ", ""); - nrOfElements = $$(`#curve-list-remove*=${cPart}`).length; - expect(nrOfElements).toBeGreaterThan( - 0, - `Expected an "${selector}" button to exist` - ); - $(`#curve-list-remove*=${cPart}`)[method](); - } else { - // normal buttons - switch (label) { - case "Add Curve": - selector = $("#add"); - selector.waitForDisplayed(); - nrOfElements = $$("#add").length; - break; - case "Back": - selector = $("#backButton"); - selector.waitForDisplayed(); - nrOfElements = $$("#backButton").length; - break; - case "Plot Unmatched": - selector = $("#plotUnmatched"); - selector.waitForDisplayed(); - nrOfElements = $$("#plotUnmatched").length; - break; - case "Plot Matched": - selector = $("#plotMatched"); - selector.waitForDisplayed(); - nrOfElements = $$("#plotMatched").length; - break; - case "Reset to Defaults": - selector = $("#reset"); - selector.waitForDisplayed(); - nrOfElements = $$("#reset").length; - break; - case "Clear": - selector = $("#clear-info"); - selector.waitForDisplayed(); - nrOfElements = $$("#clear-info").length; - break; - default: - throw new Error("Unhandled button label???"); - } - // these are for the switch statement i.e. 'normal buttons' - expect(nrOfElements).toBeGreaterThan( - 0, - `Expected an "${selector}" button to exist` - ); - selector[method](); - } -}; diff --git a/tests/src/support/action/closeAllButFirstTab.js b/tests/src/support/action/closeAllButFirstTab.js deleted file mode 100644 index 209445e44..000000000 --- a/tests/src/support/action/closeAllButFirstTab.js +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Close all but the first tab - * @param {String} obsolete Type of object to close (window or tab) - */ -/* eslint-disable no-unused-vars */ -export default (obsolete) => { - /* eslint-enable no-unused-vars */ - /** - * Get all the window handles - * @type {Object} - */ - const windowHandles = browser.getWindowHandles(); - - // Close all tabs but the first one - windowHandles.reverse(); - windowHandles.forEach((handle, index) => { - browser.switchToWindow(handle); - if (index < windowHandles.length - 1) { - browser.closeWindow(); - } - }); -}; diff --git a/tests/src/support/action/closeLastOpenedWindow.js b/tests/src/support/action/closeLastOpenedWindow.js deleted file mode 100644 index 85b53b0d3..000000000 --- a/tests/src/support/action/closeLastOpenedWindow.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Close the last opened window - * @param {String} obsolete Type of object to close (window or tab) - */ -/* eslint-disable no-unused-vars */ -export default (obsolete) => { - /* eslint-enable no-unused-vars */ - /** - * The last opened window handle - * @type {Object} - */ - const lastWindowHandle = browser.getWindowHandles().slice(-1)[0]; - - browser.closeWindow(); - browser.switchToWindow(lastWindowHandle); -}; diff --git a/tests/src/support/action/deleteCookies.js b/tests/src/support/action/deleteCookies.js deleted file mode 100644 index 196a20628..000000000 --- a/tests/src/support/action/deleteCookies.js +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Delete a cookie - * @param {String} name The name of the cookie to delete - */ -export default (name) => { - browser.deleteCookies(name); -}; diff --git a/tests/src/support/action/dragElement.js b/tests/src/support/action/dragElement.js deleted file mode 100644 index f7a783984..000000000 --- a/tests/src/support/action/dragElement.js +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Drag a element to a given destination - * @param {String} selector The selector for the source element - * @param {String} destination The selector for the destination element - */ -export default (selector, destination) => { - $(selector).dragAndDrop($(destination)); -}; diff --git a/tests/src/support/action/focusLastOpenedWindow.js b/tests/src/support/action/focusLastOpenedWindow.js deleted file mode 100644 index b11f16247..000000000 --- a/tests/src/support/action/focusLastOpenedWindow.js +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Focus the last opened window - * @param {String} obsolete Type of object to focus to (window or tab) - */ -/* eslint-disable no-unused-vars */ -export default (obsolete) => { - /* eslint-enable no-unused-vars */ - /** - * The last opened window - * @type {Object} - */ - const lastWindowHandle = browser.getWindowHandles().slice(-1)[0]; - - browser.switchToWindow(lastWindowHandle); -}; diff --git a/tests/src/support/action/handleModal.js b/tests/src/support/action/handleModal.js deleted file mode 100644 index 0098369ae..000000000 --- a/tests/src/support/action/handleModal.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Handle a modal - * @param {String} action Action to perform on the modal (accept, dismiss) - * @param {String} modalType Type of modal (alertbox, confirmbox, prompt) - */ -export default (action, modalType) => { - /** - * The command to perform on the browser object - * @type {String} - */ - let command = `${action.slice(0, 1).toLowerCase()}${action.slice(1)}Alert`; - - /** - * Alert boxes can't be dismissed, this causes Chrome to crash during tests - */ - if (modalType === "alertbox") { - command = "acceptAlert"; - } - - browser[command](); -}; diff --git a/tests/src/support/action/matsClearParameter.js b/tests/src/support/action/matsClearParameter.js deleted file mode 100644 index 0fc817e31..000000000 --- a/tests/src/support/action/matsClearParameter.js +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Clear a given input parameter - * @param {String} parameter Element selector - */ -export default (parameter) => { - $(`[id='controlButton-${parameter}-value']`).scrollIntoView(); - $(`#controlButton-${parameter}-value`).click(); - if ($(`[id*='${parameter}-select-clear']`).waitForDisplayed()) { - $(`[id*='${parameter}-select-clear']`).click(); - } - $(`#controlButton-${parameter}-value`).click(); -}; diff --git a/tests/src/support/action/matsDebug.js b/tests/src/support/action/matsDebug.js deleted file mode 100644 index 5e719fb2e..000000000 --- a/tests/src/support/action/matsDebug.js +++ /dev/null @@ -1,6 +0,0 @@ -/** - * browser.debug - */ -export default () => { - browser.debug(); -}; diff --git a/tests/src/support/action/matsRefreshBrowser.js b/tests/src/support/action/matsRefreshBrowser.js deleted file mode 100644 index fe5a9b09d..000000000 --- a/tests/src/support/action/matsRefreshBrowser.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * refresh the browser - * Creates a new Selenium session with your current capabilities. - * This is useful if you test highly stateful application where you need - * to clean the browser session between the tests in your spec file - * to avoid creating hundreds of single test files with WDIO. - */ -export default () => { - browser.reloadSession(); -}; diff --git a/tests/src/support/action/matsRefreshPage.js b/tests/src/support/action/matsRefreshPage.js deleted file mode 100644 index c90389b1e..000000000 --- a/tests/src/support/action/matsRefreshPage.js +++ /dev/null @@ -1,6 +0,0 @@ -/** - * Refresh the current page. - */ -export default () => { - browser.refresh(); -}; diff --git a/tests/src/support/action/moveTo.js b/tests/src/support/action/moveTo.js deleted file mode 100644 index decc45b37..000000000 --- a/tests/src/support/action/moveTo.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Move to the given selector with an optional offset on a X and Y position - * @param {String} selector Element selector - * @param {String} x X coordinate to move to - * @param {String} y Y coordinate to move to - */ -export default (selector, x, y) => { - /** - * X coordinate - * @type {Int} - */ - const intX = parseInt(x, 10) || undefined; - - /** - * Y coordinate - * @type {Int} - */ - const intY = parseInt(y, 10) || undefined; - - $(selector).moveTo(intX, intY); -}; diff --git a/tests/src/support/action/openWebsite.js b/tests/src/support/action/openWebsite.js deleted file mode 100644 index 9609cef17..000000000 --- a/tests/src/support/action/openWebsite.js +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Open the given URL - * @param {String} type Type of navigation (getUrl or site) - * @param {String} page The URL to navigate to - */ -export default (type, page) => { - /** - * The URL to navigate to - * @type {String} - */ - const url = type === "url" ? page : browser.options.baseUrl + page; - browser.url(url); -}; diff --git a/tests/src/support/action/openWebsiteAndWaitForPlotType.js b/tests/src/support/action/openWebsiteAndWaitForPlotType.js deleted file mode 100644 index 1766ad29e..000000000 --- a/tests/src/support/action/openWebsiteAndWaitForPlotType.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Open the given URL - * @param {String} page The URL to navigate to - */ - -export default (page) => { - /** - * The URL to navigate to - * @type {String} - */ - const url = browser.options.baseUrl + page; - browser.url(url); - const ms = 120000; - // wait for the curve label selector to exist - // noinspection JSJQueryEfficiency - $("#controlButton-label-value").waitForExist({ timeout: ms }); - // noinspection JSJQueryEfficiency - $("#controlButton-label-value").waitForEnabled({ timeout: ms }); - $("#controlButton-label-value").waitForClickable({ timeout: ms }); -}; diff --git a/tests/src/support/action/pause.js b/tests/src/support/action/pause.js deleted file mode 100644 index 1b591da14..000000000 --- a/tests/src/support/action/pause.js +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Pause execution for a given number of milliseconds - * @param {String} ms Number of milliseconds to pause - */ -export default (ms) => { - /** - * Number of milliseconds - * @type {Int} - */ - const intMs = parseInt(ms, 10); - - browser.pause(intMs); -}; diff --git a/tests/src/support/action/pressButton.js b/tests/src/support/action/pressButton.js deleted file mode 100644 index 79b7fdf7f..000000000 --- a/tests/src/support/action/pressButton.js +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Perform a key press - * @param {String} key The key to press - */ -export default (key) => { - browser.keys(key); -}; diff --git a/tests/src/support/action/saveMatsParameters.js b/tests/src/support/action/saveMatsParameters.js deleted file mode 100644 index fd32325df..000000000 --- a/tests/src/support/action/saveMatsParameters.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Check if the previously stored parameters match the current parameters. - */ -export default () => { - browser.saveMatsParameters = $$(".control-button").map((element) => - element.getText() - ); - // console.log(browser.saveMatsParameters); - // browser.debug(); -}; diff --git a/tests/src/support/action/scroll.js b/tests/src/support/action/scroll.js deleted file mode 100644 index 11ec4c931..000000000 --- a/tests/src/support/action/scroll.js +++ /dev/null @@ -1,7 +0,0 @@ -/** - * Scroll the page to the given element - * @param {String} selector Element selector - */ -export default (selector) => { - $(selector).scrollIntoView(); -}; diff --git a/tests/src/support/action/selectOption.js b/tests/src/support/action/selectOption.js deleted file mode 100644 index c9fd19eed..000000000 --- a/tests/src/support/action/selectOption.js +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Select an option of a select element - * @param {String} selectionType Type of method to select by (name, value or - * text) - * @param {String} selectionValue Value to select by - * @param {String} selector Element selector - */ -export default (selectionType, selectionValue, selector) => { - /** - * The method to use for selecting the option - * @type {String} - */ - let command = ""; - const commandArguments = [selectionValue]; - - switch (selectionType) { - case "name": { - command = "selectByAttribute"; - - // The selectByAttribute command expects the attribute name as it - // second argument so let's add it - commandArguments.unshift("name"); - - break; - } - - case "value": { - // The selectByAttribute command expects the attribute name as it - // second argument so let's add it - commandArguments.unshift("value"); - command = "selectByAttribute"; - break; - } - - case "text": { - command = "selectByVisibleText"; - break; - } - - default: { - throw new Error(`Unknown selection type "${selectionType}"`); - } - } - - $(selector)[command](...commandArguments); -}; diff --git a/tests/src/support/action/selectOptionByIndex.js b/tests/src/support/action/selectOptionByIndex.js deleted file mode 100644 index b806e32a1..000000000 --- a/tests/src/support/action/selectOptionByIndex.js +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Select a option from a select element by it's index - * @param {String} index The index of the option - * @param {String} obsolete The ordinal indicator of the index (unused) - * @param {String} selector Element selector - * - * @todo merge with selectOption - */ -export default (index, obsolete, selector) => { - /** - * The index of the option to select - * @type {Int} - */ - const optionIndex = parseInt(index, 10); - - $(selector).selectByIndex(optionIndex); -}; diff --git a/tests/src/support/action/setCookie.js b/tests/src/support/action/setCookie.js deleted file mode 100644 index 06219c679..000000000 --- a/tests/src/support/action/setCookie.js +++ /dev/null @@ -1,12 +0,0 @@ -/** - * Set a given cookie to a given value. When the cookies does not exist it will - * be created - * @param {String} cookieName The name of the cookie - * @param {String} cookieContent The value of the cookie - */ -export default (cookieName, cookieContent) => { - browser.setCookies({ - name: cookieName, - value: cookieContent, - }); -}; diff --git a/tests/src/support/action/setInputField.js b/tests/src/support/action/setInputField.js deleted file mode 100644 index ea6200e32..000000000 --- a/tests/src/support/action/setInputField.js +++ /dev/null @@ -1,26 +0,0 @@ -import checkIfElementExists from "../lib/checkIfElementExists"; - -/** - * Set the value of the given input field to a new value or add a value to the - * current selector value - * @param {String} method The method to use (add or set) - * @param {String} value The value to set the selector to - * @param {String} selector Element selector - */ -export default (method, value, selector) => { - /** - * The command to perform on the browser object (addValue or setValue) - * @type {String} - */ - const command = method === "add" ? "addValue" : "setValue"; - - let checkValue = value; - - checkIfElementExists(selector, false, 1); - - if (!value) { - checkValue = ""; - } - - $(selector)[command](checkValue); -}; diff --git a/tests/src/support/action/setMatsCurveDateRange.js b/tests/src/support/action/setMatsCurveDateRange.js deleted file mode 100644 index 4874cf685..000000000 --- a/tests/src/support/action/setMatsCurveDateRange.js +++ /dev/null @@ -1,38 +0,0 @@ -import pause from "./pause"; - -/** - * Set the curve date range to a predefined range - * @param {String} value The range to set the selector to - * */ -export default (value) => { - const dateRange = value; - const dates = dateRange.split(" - "); - $("#controlButton-curve-dates-value").waitForDisplayed(); - $("#controlButton-curve-dates-value").scrollIntoView(); - $("#controlButton-curve-dates-value").waitForClickable(); - $("#controlButton-curve-dates-value").click(); // brings up date menu - $$('input[name="daterangepicker_start"]')[0].setValue(""); - $$('input[name="daterangepicker_start"]')[0].setValue(dates[0]); - $$('input[name="daterangepicker_end"]')[0].setValue(""); - $$('input[name="daterangepicker_end"]')[0].setValue(dates[1]); - $$("/html/body/div[2]/div[1]/div/button[1]")[0].waitForClickable(); - $$("/html/body/div[2]/div[1]/div/button[1]")[0].click(); - let datesValue = ""; - let count = 0; - while (count < 10 && datesValue !== dateRange) { - datesValue = $("#controlButton-curve-dates-value").getText(); - if (datesValue !== dateRange) { - pause(2000); - } - count += 1; - } - if (datesValue !== dateRange) { - console.log(`value is ${value}`); - // browser.debug(); - } - - expect(datesValue).toEqual( - value, - `"curve date range" should be ${value} but was ${datesValue}` - ); -}; diff --git a/tests/src/support/action/setMatsDateRange.js b/tests/src/support/action/setMatsDateRange.js deleted file mode 100644 index 682305e38..000000000 --- a/tests/src/support/action/setMatsDateRange.js +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Set the date range to a predefined range - * @param {String} value The range to set the selector to - * */ -import pause from "./pause"; - -export default (value) => { - const dateRange = value; - const dates = dateRange.split(" - "); - $("#controlButton-dates-value").waitForDisplayed(); - $("#controlButton-dates-value").scrollIntoView(); - $("#controlButton-dates-value").waitForClickable(); - $("#controlButton-dates-value").click(); // brings up date menu - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_start"]')[ - $$('input[name="daterangepicker_start"]').length - 1 - ].setValue(""); - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_start"]')[ - $$('input[name="daterangepicker_start"]').length - 1 - ].setValue(dates[0]); - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_end"]')[ - $$('input[name="daterangepicker_end"]').length - 1 - ].setValue(""); - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_end"]')[ - $$('input[name="daterangepicker_end"]').length - 1 - ].setValue(dates[1]); - $$("/html/body/div[3]/div[1]/div/button[1]")[0].waitForClickable(); - $$("/html/body/div[3]/div[1]/div/button[1]")[0].click(); - let datesValue = ""; - let count = 0; - while (count < 10 && datesValue !== dateRange) { - datesValue = $("#controlButton-dates-value").getText(); - if (datesValue !== dateRange) { - pause(2000); - } - count += 1; - } - if (datesValue !== dateRange) { - console.log(`value is ${value}`); - // browser.debug(); - } - expect(datesValue).toEqual( - value, - `"date range" should be ${value} but was ${datesValue}` - ); -}; diff --git a/tests/src/support/action/setMatsPlotFormat.js b/tests/src/support/action/setMatsPlotFormat.js deleted file mode 100644 index 1521e1a6e..000000000 --- a/tests/src/support/action/setMatsPlotFormat.js +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Set the plotFormat - * @param {String} plotFormat The plot format - * */ -export default (plotFormat) => { - switch (plotFormat) { - case "matching diffs": - $("#plotFormat-radioGroup-matching").scrollIntoView(); - $("#plotFormat-radioGroup-matching").click(); - break; - case "pairwise diffs": - $("#plotFormat-radioGroup-pairwise").scrollIntoView(); - $("#plotFormat-radioGroup-pairwise").click(); - break; - case "no diffs": - $("#plotFormat-radioGroup-none").scrollIntoView(); - $("#plotFormat-radioGroup-none").click(); - break; - default: - throw new Error("invalid plotFormat in setMatsPlotFormat"); - } -}; diff --git a/tests/src/support/action/setMatsPlotType.js b/tests/src/support/action/setMatsPlotType.js deleted file mode 100644 index 81361d031..000000000 --- a/tests/src/support/action/setMatsPlotType.js +++ /dev/null @@ -1,10 +0,0 @@ -/** - * Set the date range to a predefined range - * @param {String} plotType The type of date range selector (curve or date) - * */ -export default (plotType) => { - $("#plotTypes-selector").scrollIntoView(); - $("#plotTypes-selector").click(); - $(`#plot-type-${plotType}`).scrollIntoView(); - $(`#plot-type-${plotType}`).click(); -}; diff --git a/tests/src/support/action/setMatsSelectedOption.js b/tests/src/support/action/setMatsSelectedOption.js deleted file mode 100644 index 68fb1c428..000000000 --- a/tests/src/support/action/setMatsSelectedOption.js +++ /dev/null @@ -1,73 +0,0 @@ -/** - * Select an option of a select element - * @param {String} parameter Element selector label - * @param {String} selectionValue Value to select by - */ -import pause from "./pause"; - -export default (parameter, selectionValue) => { - /** - * The method to use for selecting the option - * @type {String} - */ - - // console.log(`$('#controlButton-${parameter}')`); - // console.log(`$('.select2-results__option=${selectionValue}')`); - // console.log(`$('.select2-results__option=${selectionValue}')`); - // browser.debug(); - - // noinspection JSJQueryEfficiency - $(`#controlButton-${parameter}`).waitForDisplayed(); - let len = $$(`.select2-results__option=${selectionValue}`).length; - // it might already be clicked! Sometimes the click doesn't seem to take on the first try. - let c = 0; - while (len === 0 && c < 20) { - $(`#controlButton-${parameter}`).waitForClickable(); - $(`#controlButton-${parameter}`).click(); - len = $$(`.select2-results__option=${selectionValue}`).length; - pause(1000); - c += 1; - } - let multi = false; - if ($(`#${parameter}-select-clear`).isDisplayed()) { - multi = true; - // if it is a multi-select selector it has a clear button. Better clear it - $(`#${parameter}-select-clear`).waitForClickable(); - $(`#${parameter}-select-clear`).click(); - } - // noinspection JSJQueryEfficiency - $(`.select2-results__option=${selectionValue}`).scrollIntoView(); - // noinspection JSJQueryEfficiency - $(`.select2-results__option=${selectionValue}`).waitForClickable(); - $(`.select2-results__option=${selectionValue}`).click(); - if ($(`#${parameter}-select-done`).isDisplayed()) { - // if it is a multi-select selector, have to click the done button - $(`#${parameter}-select-done`).waitForClickable(); - $(`#${parameter}-select-done`).click(); - } - - let matchValue = selectionValue; - if (multi === true) { - // multi-selects have a range value - matchValue = `${selectionValue} .. ${selectionValue}`; - } - let text = ""; - let count = 0; - // this is essentially giving the parameter 20 seconds to show the new value - // this is mostly for when it is really busy doing parallel instances - while (count < 20 && text !== matchValue) { - text = $(`#controlButton-${parameter}-value`).getText(); - if (text !== matchValue) { - pause(2000); - } - count += 1; - } - if (text !== matchValue) { - console.log(`parameter is ${parameter}, selectionValue is ${selectionValue}`); - // browser.debug(); - } - expect(text).toEqual( - matchValue, - `Expexted ${text} to be ${matchValue} for parameter: ${parameter}` - ); -}; diff --git a/tests/src/support/action/setPredefinedDateRange.js b/tests/src/support/action/setPredefinedDateRange.js deleted file mode 100644 index ff41ae6c2..000000000 --- a/tests/src/support/action/setPredefinedDateRange.js +++ /dev/null @@ -1,46 +0,0 @@ -// noinspection JSIgnoredPromiseFromCall -/** * - * Set the date range to a predefined range - * @param {String} parameterType The type of date range selector (curve or date) - * @param {String} value The range to set the selector to - ** */ -export default (parameterType, value) => { - let definedRange = value; - if (!definedRange) { - definedRange = ""; - } - expect(parameterType === "curve-date" || parameterType === "date").toEqual( - true, - `Expected element "${parameterType}" to be "curve-date OR date"` - ); - if (parameterType === "curve-date") { - $("#controlButton-curve-dates-value").scrollIntoView(); - $("#controlButton-curve-dates-value").click(); // brings up date menu - } else if (parameterType === "date") { - $("#controlButton-dates-value").scrollIntoView(); - $("#controlButton-dates-value").click(); - } - browser.execute((dRange) => { - // eslint-disable-next-line no-undef - const dateRangePickers = document.getElementsByClassName("show-calendar"); - let dateRangePicker = null; - for (let dri = 0; dri < dateRangePickers.length; dri += 1) { - if (dateRangePickers[dri].style.display === "block") { - dateRangePicker = dateRangePickers[dri]; - break; - } - } - expect(dateRangePicker).not.toBe(null, "no dateRangePickerFound!"); - // noinspection JSObjectNullOrUndefined - const liTags = dateRangePicker.getElementsByTagName("li"); - let item = null; - for (let i = 0; i < liTags.length; i += 1) { - if (liTags[i].textContent === dRange) { - item = liTags[i]; - break; - } - } - // noinspection JSObjectNullOrUndefined - item.click(); - }, definedRange); -}; diff --git a/tests/src/support/action/setPromptText.js b/tests/src/support/action/setPromptText.js deleted file mode 100644 index 31af52347..000000000 --- a/tests/src/support/action/setPromptText.js +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Set the text of the current prompt - * @param {String} modalText The text to set to the prompt - */ -export default (modalText) => { - try { - browser.sendAlertText(modalText); - } catch (e) { - assert(e, "A prompt was not open when it should have been open"); - } -}; diff --git a/tests/src/support/action/setScatterDateRange.js b/tests/src/support/action/setScatterDateRange.js deleted file mode 100644 index 1c1e7ad24..000000000 --- a/tests/src/support/action/setScatterDateRange.js +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Set the curve date range to a predefined range - * @param {String} value The range to set the selector to - * */ -export default (value) => { - const dates = value.split(" - "); - $("#controlButton-dates-value").scrollIntoView(); - $("#controlButton-dates-value").click(); // brings up date menu - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_start"]')[ - $$('input[name="daterangepicker_start"]').length - 3 - ].setValue(""); - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_start"]')[ - $$('input[name="daterangepicker_start"]').length - 3 - ].setValue(dates[0]); - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_end"]')[ - $$('input[name="daterangepicker_end"]').length - 3 - ].setValue(""); - // eslint-disable-next-line max-len - $$('input[name="daterangepicker_end"]')[ - $$('input[name="daterangepicker_end"]').length - 3 - ].setValue(dates[1]); - // eslint-disable-next-line max-len - $$("/html/body/div[1]/div[1]/div/button")[ - $$("/html/body/div[3]/div[1]/div/button").length - 3 - ].click(); -}; diff --git a/tests/src/support/action/setWindowSize.js b/tests/src/support/action/setWindowSize.js deleted file mode 100644 index 957715787..000000000 --- a/tests/src/support/action/setWindowSize.js +++ /dev/null @@ -1,8 +0,0 @@ -/** - * Resize the browser window - * @param {String} screenWidth The width of the window to resize to - * @param {String} screenHeight The height of the window to resize to - */ -export default (screenWidth, screenHeight) => { - browser.setWindowSize(parseInt(screenWidth, 10), parseInt(screenHeight, 10)); -}; diff --git a/tests/src/support/action/waitFor.js b/tests/src/support/action/waitFor.js deleted file mode 100644 index 83f18e16d..000000000 --- a/tests/src/support/action/waitFor.js +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Wait for the given element to be enabled, displayed, or to exist - * @param {String} selector Element selector - * @param {String} ms Wait duration (optional) - * @param {String} falseState Check for opposite state - * @param {String} state State to check for (default - * existence) - */ -export default (selector, ms, falseState, state) => { - /** - * Maximum number of milliseconds to wait, default 3000 - * @type {Int} - */ - const intMs = parseInt(ms, 10) || 3000; - - /** - * Command to perform on the browser object - * @type {String} - */ - let command = "waitForExist"; - - /** - * Boolean interpretation of the false state - * @type {Boolean} - */ - let boolFalseState = !!falseState; - - /** - * Parsed interpretation of the state - * @type {String} - */ - let parsedState = ""; - - if (falseState || state) { - parsedState = - state.indexOf(" ") > -1 ? state.split(/\s/)[state.split(/\s/).length - 1] : state; - - if (parsedState) { - command = `waitFor${parsedState[0].toUpperCase()}` + `${parsedState.slice(1)}`; - } - } - - if (typeof falseState === "undefined") { - boolFalseState = false; - } - - $(selector)[command](intMs, boolFalseState); -}; diff --git a/tests/src/support/action/waitForDisplayed.js b/tests/src/support/action/waitForDisplayed.js deleted file mode 100644 index d0dea2bfc..000000000 --- a/tests/src/support/action/waitForDisplayed.js +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Wait for the given element to become visible - * @param {String} selector Element selector - * @param {String} falseCase Whether or not to expect a visible or hidden - * state - * - * @todo merge with waitfor - */ -export default (selector, falseCase) => { - /** - * Maximum number of milliseconds to wait for - * @type {Int} - */ - const ms = 10000; - - $(selector).waitForDisplayed(ms, !!falseCase); -}; diff --git a/tests/src/support/check/checkClass.js b/tests/src/support/check/checkClass.js deleted file mode 100644 index 972ec31ef..000000000 --- a/tests/src/support/check/checkClass.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Check if the given element has the given class - * @param {String} selector Element selector - * @param {String} falseCase Whether to check for the class to exist - * or not ('has', 'does not have') - * @param {String} expectedClassName The class name to check - */ -export default (selector, falseCase, expectedClassName) => { - /** - * List of all the classes of the element - * @type {Array} - */ - const classesList = $(selector).getAttribute("className").split(" "); - - if (falseCase === "does not have") { - expect(classesList).not.toContain( - expectedClassName, - `Element ${selector} should not have the class ${expectedClassName}` - ); - } else { - expect(classesList).toContain( - expectedClassName, - `Element ${selector} should have the class ${expectedClassName}` - ); - } -}; diff --git a/tests/src/support/check/checkContainsAnyText.js b/tests/src/support/check/checkContainsAnyText.js deleted file mode 100644 index a5bbef569..000000000 --- a/tests/src/support/check/checkContainsAnyText.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Check if the given elements contains text - * @param {String} elementType Element type (element or button) - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the content contains - * text or not - */ -export default (elementType, selector, falseCase) => { - /** - * The command to perform on the browser object - * @type {String} - */ - let command = "getValue"; - - if (elementType === "button" || $(selector).getAttribute("value") === null) { - command = "getText"; - } - - /** - * False case - * @type {Boolean} - */ - let boolFalseCase; - - /** - * The text of the element - * @type {String} - */ - const text = $(selector)[command](); - - if (typeof falseCase === "undefined") { - boolFalseCase = false; - } else { - boolFalseCase = !!falseCase; - } - - if (boolFalseCase) { - expect(text).toBe(""); - } else { - expect(text).not.toBe(""); - } -}; diff --git a/tests/src/support/check/checkContainsText.js b/tests/src/support/check/checkContainsText.js deleted file mode 100644 index 194cdc3b0..000000000 --- a/tests/src/support/check/checkContainsText.js +++ /dev/null @@ -1,55 +0,0 @@ -/** - * Check if the given elements contains text - * @param {String} elementType Element type (element or button) - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the content contains - * the given text or not - * @param {String} expectedText The text to check against - */ -export default (elementType, selector, falseCase, expectedText) => { - /** - * The command to perform on the browser object - * @type {String} - */ - let command = "getValue"; - - if ( - ["button", "container"].includes(elementType) || - $(selector).getAttribute("value") === null - ) { - command = "getText"; - } - - /** - * False case - * @type {Boolean} - */ - let boolFalseCase; - - /** - * The expected text - * @type {String} - */ - let stringExpectedText = expectedText; - - /** - * The text of the element - * @type {String} - */ - const elem = $(selector); - elem.waitForDisplayed(); - const text = elem[command](); - - if (typeof expectedText === "undefined") { - stringExpectedText = falseCase; - boolFalseCase = false; - } else { - boolFalseCase = falseCase === " not"; - } - - if (boolFalseCase) { - expect(text).not.toContain(stringExpectedText); - } else { - expect(text).toContain(stringExpectedText); - } -}; diff --git a/tests/src/support/check/checkCookieContent.js b/tests/src/support/check/checkCookieContent.js deleted file mode 100644 index 374e28b2c..000000000 --- a/tests/src/support/check/checkCookieContent.js +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Check the content of a cookie against a given value - * @param {String} name The name of the cookie - * @param {String} falseCase Whether or not to check if the value matches - * or not - * @param {String} expectedValue The value to check against - */ -export default (name, falseCase, expectedValue) => { - /** - * The cookie retrieved from the browser object - * @type {Object} - */ - const cookie = browser.getCookies(name)[0]; - expect(cookie.name).toBe(name, `no cookie found with the name "${name}"`); - - if (falseCase) { - expect(cookie.value).not.toBe( - expectedValue, - `expected cookie "${name}" not to have value "${expectedValue}"` - ); - } else { - expect(cookie.value).toBe( - expectedValue, - `expected cookie "${name}" to have value "${expectedValue}"` + - ` but got "${cookie.value}"` - ); - } -}; diff --git a/tests/src/support/check/checkCookieExists.js b/tests/src/support/check/checkCookieExists.js deleted file mode 100644 index daab2d03f..000000000 --- a/tests/src/support/check/checkCookieExists.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Check if a cookie with the given name exists - * @param {[type]} name The name of the cookie - * @param {[type]} falseCase Whether or not to check if the cookie exists or - * not - */ -export default (name, falseCase) => { - /** - * The cookie as retrieved from the browser - * @type {Object} - */ - const cookie = browser.getCookies(name); - - if (falseCase) { - expect(cookie).toHaveLength( - 0, - `Expected cookie "${name}" not to exists but it does` - ); - } else { - expect(cookie).not.toHaveLength( - 0, - `Expected cookie "${name}" to exists but it does not` - ); - } -}; diff --git a/tests/src/support/check/checkDimension.js b/tests/src/support/check/checkDimension.js deleted file mode 100644 index f5738fa57..000000000 --- a/tests/src/support/check/checkDimension.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Check the dimensions of the given element - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the dimensions match or - * not - * @param {String} expectedSize Expected size - * @param {String} dimension Dimension to check (broad or tall) - */ -export default (selector, falseCase, expectedSize, dimension) => { - /** - * The size of the given element - * @type {Object} - */ - const elementSize = $(selector).getSize(); - - /** - * Parsed size to check for - * @type {Int} - */ - const intExpectedSize = parseInt(expectedSize, 10); - - /** - * The size property to check against - * @type {Int} - */ - let originalSize = elementSize.height; - - /** - * The label of the checked property - * @type {String} - */ - let label = "height"; - - if (dimension === "broad") { - originalSize = elementSize.width; - label = "width"; - } - - if (falseCase) { - expect(originalSize).not.toBe( - intExpectedSize, - `Element "${selector}" should not have a ${label} of ` + `${intExpectedSize}px` - ); - } else { - expect(originalSize).toBe( - intExpectedSize, - `Element "${selector}" should have a ${label} of ` + - `${intExpectedSize}px, but is ${originalSize}px` - ); - } -}; diff --git a/tests/src/support/check/checkElementExists.js b/tests/src/support/check/checkElementExists.js deleted file mode 100644 index dbc632014..000000000 --- a/tests/src/support/check/checkElementExists.js +++ /dev/null @@ -1,21 +0,0 @@ -import checkIfElementExists from "../lib/checkIfElementExists"; - -/** - * Check if the given element exists - * @param {String} isExisting Whether the element should be existing or not - * (an or no) - * @param {String} selector Element selector - */ -export default (isExisting, selector) => { - /** - * Falsecase assertion - * @type {Boolean} - */ - let falseCase = true; - - if (isExisting === "an") { - falseCase = false; - } - - checkIfElementExists(selector, falseCase); -}; diff --git a/tests/src/support/check/checkEqualsText.js b/tests/src/support/check/checkEqualsText.js deleted file mode 100644 index 1558a768a..000000000 --- a/tests/src/support/check/checkEqualsText.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Check if the given elements text is the same as the given text - * @param {String} elementType Element type (element or button) - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the content equals the - * given text or not - * @param {String} expectedText The text to validate against - */ -export default (elementType, selector, falseCase, expectedText) => { - /** - * The command to execute on the browser object - * @type {String} - */ - let command = "getValue"; - - if (elementType === "button" || $(selector).getAttribute("value") === null) { - command = "getText"; - } - - /** - * The expected text to validate against - * @type {String} - */ - let parsedExpectedText = expectedText; - - /** - * Whether to check if the content equals the given text or not - * @type {Boolean} - */ - let boolFalseCase = !!falseCase; - - // Check for empty element - if (typeof parsedExpectedText === "function") { - parsedExpectedText = ""; - - boolFalseCase = !boolFalseCase; - } - - if (parsedExpectedText === undefined && falseCase === undefined) { - parsedExpectedText = ""; - boolFalseCase = true; - } - - const text = browser[command](selector); - - if (boolFalseCase) { - expect(parsedExpectedText).not.toBe(text); - } else { - expect(parsedExpectedText).toBe(text); - } -}; diff --git a/tests/src/support/check/checkFocus.js b/tests/src/support/check/checkFocus.js deleted file mode 100644 index 4371c1a5a..000000000 --- a/tests/src/support/check/checkFocus.js +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Check if the given element has the focus - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the given element has focus - * or not - */ -export default (selector, falseCase) => { - /** - * Value of the hasFocus function for the given element - * @type {Boolean} - */ - const hasFocus = $(selector).isFocused(); - - if (falseCase) { - expect(hasFocus).not.toBe(true, "Expected element to not be focused, but it is"); - } else { - expect(hasFocus).toBe(true, "Expected element to be focused, but it is not"); - } -}; diff --git a/tests/src/support/check/checkFontProperty.js b/tests/src/support/check/checkFontProperty.js deleted file mode 100644 index fbdca5b30..000000000 --- a/tests/src/support/check/checkFontProperty.js +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Check the given property of the given element - * @param {String} isCSS Whether to check for a CSS property or an - * attribute - * @param {String} attrName The name of the attribute to check - * @param {String} elem Element selector - * @param {String} falseCase Whether to check if the value of the - * attribute matches or not - * @param {String} expectedValue The value to match against - */ -export default (isCSS, attrName, elem, falseCase, expectedValue) => { - /** - * The command to use for fetching the expected value - * @type {String} - */ - const command = isCSS ? "getCssProperty" : "getAttribute"; - - /** - * Te label to identify the attribute by - * @type {String} - */ - const attrType = isCSS ? "CSS attribute" : "Attribute"; - - /** - * The actual attribute value - * @type {Mixed} - */ - let attributeValue = browser[command](elem, attrName); - - /** - * when getting something with a color or font-weight WebdriverIO returns a - * object but we want to assert against a string - */ - if (attrName.match(/(font-size|line-height|display|font-weight)/)) { - attributeValue = attributeValue.value; - } - - if (falseCase) { - expect(attributeValue).not.toBe( - expectedValue, - `${attrType}: ${attrName} of element "${elem}" should not ` + - `contain "${attributeValue}"` - ); - } else { - expect(attributeValue).toBe( - expectedValue, - `${attrType}: ${attrName} of element "${elem}" should contain ` + - `"${attributeValue}", but "${expectedValue}"` - ); - } -}; diff --git a/tests/src/support/check/checkInURLPath.js b/tests/src/support/check/checkInURLPath.js deleted file mode 100644 index a0306dbd7..000000000 --- a/tests/src/support/check/checkInURLPath.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Check if the given string is in the URL path - * @param {String} falseCase Whether to check if the given string is in - * the URL path or not - * @param {String} expectedUrlPart The string to check for - */ -export default (falseCase, expectedUrlPart) => { - /** - * The URL of the current browser window - * @type {String} - */ - const currentUrl = browser.getUrl(); - - if (falseCase) { - expect(currentUrl).not.toContain( - expectedUrlPart, - `Expected URL "${currentUrl}" not to contain ` + `"${expectedUrlPart}"` - ); - } else { - expect(currentUrl).toContain( - expectedUrlPart, - `Expected URL "${currentUrl}" to contain "${expectedUrlPart}"` - ); - } -}; diff --git a/tests/src/support/check/checkIsEmpty.js b/tests/src/support/check/checkIsEmpty.js deleted file mode 100644 index acf8a4008..000000000 --- a/tests/src/support/check/checkIsEmpty.js +++ /dev/null @@ -1,13 +0,0 @@ -import checkContainsAnyText from "./checkContainsAnyText"; - -export default (elementType, element, falseCase) => { - let newFalseCase = true; - - if (typeof falseCase === "function") { - newFalseCase = false; - } else if (falseCase === " not") { - newFalseCase = false; - } - - checkContainsAnyText(elementType, element, newFalseCase); -}; diff --git a/tests/src/support/check/checkIsOpenedInNewWindow.js b/tests/src/support/check/checkIsOpenedInNewWindow.js deleted file mode 100644 index 95571f7db..000000000 --- a/tests/src/support/check/checkIsOpenedInNewWindow.js +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Check if the given URL was opened in a new window - * @param {String} expectedUrl The URL to check for - */ -/* eslint-disable no-unused-vars */ -export default (expectedUrl, type) => { - /* eslint-enable no-unused-vars */ - /** - * All the current window handles - * @type {Object} - */ - const windowHandles = browser.getWindowHandles(); - - expect(windowHandles).not.toHaveLength(1, "A popup was not opened"); - - /** - * The last opened window handle - * @type {Object} - */ - const lastWindowHandle = windowHandles.slice(-1); - - // Make sure we focus on the last opened window handle - browser.switchToWindow(lastWindowHandle[0]); - - /** - * Get the URL of the current browser window - * @type {String} - */ - const windowUrl = browser.getUrl(); - - expect(windowUrl).toContain(expectedUrl, "The popup has a incorrect getUrl"); - - browser.closeWindow(); -}; diff --git a/tests/src/support/check/checkMatsAppTitle.js b/tests/src/support/check/checkMatsAppTitle.js deleted file mode 100644 index 028e5b42f..000000000 --- a/tests/src/support/check/checkMatsAppTitle.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Check the title (in the plottype element) - * @param {String} title the selection parameter - */ -export default (title) => { - const command = "getText"; - - /** - * The expected text - * @type {String} - */ - const stringExpectedText = title; - - /** - * The text of the element - * @type {String} - */ - const elem = $("#plotType"); - elem.waitForDisplayed(); - const text = elem[command](); - - expect(text).toContain(stringExpectedText); -}; diff --git a/tests/src/support/check/checkMatsCurveDatesValue.js b/tests/src/support/check/checkMatsCurveDatesValue.js deleted file mode 100644 index 9b6353a97..000000000 --- a/tests/src/support/check/checkMatsCurveDatesValue.js +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Check the selected state of the given element - * @param {String} value the expected value - */ -export default (value) => { - /** - * The expected value - * @type {string} - */ - const datesValue = $("#controlButton-curve-dates-value").getText(); - expect(datesValue).toEqual( - value, - `"daterange" should be ${value} but was ${datesValue}` - ); -}; diff --git a/tests/src/support/check/checkMatsCurveIsAdded.js b/tests/src/support/check/checkMatsCurveIsAdded.js deleted file mode 100644 index a250dd8b8..000000000 --- a/tests/src/support/check/checkMatsCurveIsAdded.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Check the curve has been added - * @param {String} the label of the curve - */ - -export default (curve) => { - const command = "getText"; - const selector = $(`#curveItem-${curve}`); - /** - * The expected text - * @type {String} - */ - const stringExpectedText = curve; - - /** - * The text of the element - * @type {String} - */ - selector.waitForDisplayed(); - const text = selector[command](); - - expect(text).toContain(stringExpectedText); -}; diff --git a/tests/src/support/check/checkMatsCurveListContains.js b/tests/src/support/check/checkMatsCurveListContains.js deleted file mode 100644 index 1e4e99060..000000000 --- a/tests/src/support/check/checkMatsCurveListContains.js +++ /dev/null @@ -1,25 +0,0 @@ -import pause from "../action/pause"; - -/** - * Check if the given elements contains text - * @param {String} expected The textual list to check against - */ -export default (expected) => { - /** - * Check that the curve list contains specific curve label - * @curveNumber {Number} - */ - const expectedList = expected.split(",").sort(); - $(".displayItemLabelSpan").waitForDisplayed(); - pause(1000); - const actualList = $$(".displayItemLabelSpan") - .map((elem) => elem.getText()) - .sort(); - const expectedText = expectedList.join(","); - const actualText = actualList.join(","); - const matches = expectedText === actualText; - expect(matches).toBe( - true, - `expected list ${expectedList} does not match actualList ${actualList}` - ); -}; diff --git a/tests/src/support/check/checkMatsCurveNumber.js b/tests/src/support/check/checkMatsCurveNumber.js deleted file mode 100644 index b292b91cb..000000000 --- a/tests/src/support/check/checkMatsCurveNumber.js +++ /dev/null @@ -1,39 +0,0 @@ -import pause from "../action/pause"; - -/** - * Check if the given elements contains text - * @param {number} curveNumber The text to check against - */ -export default (curveNumber) => { - /** - * Check that the graph contains curveNumber of curves - * @curveNumber {Number} - * @type {String} - */ - if (curveNumber === 0) { - // there won't be any curvelist - let count = 0; - let exists = $("#curveList").isExisting() && $("#curveList").isDisplayed(); - while (count < 10 && exists !== false) { - if (exists !== false) { - pause(2000); - exists = $("#curveList").isExisting() && $("#curveList").isDisplayed(); - count += 1; - } - } - expect(exists).toEqual(false, "There should be no curves remaining"); - } else { - let count = 0; - $("#curveList").waitForDisplayed(20000); - let curveItemsLength = $$("[id|='curveItem']").length; - while (count < 5 && curveItemsLength !== curveNumber) { - pause(1000); - curveItemsLength = $$("[id|='curveItem']").length; - count += 1; - } - expect(curveItemsLength).toEqual( - curveNumber, - `The expected number of curves #{curveNumber} does not match ${curveItemsLength}` - ); - } -}; diff --git a/tests/src/support/check/checkMatsDatesValue.js b/tests/src/support/check/checkMatsDatesValue.js deleted file mode 100644 index 7e954ba3f..000000000 --- a/tests/src/support/check/checkMatsDatesValue.js +++ /dev/null @@ -1,15 +0,0 @@ -/** - * Check the selected state of the given element - * @param {String} value the expected value - */ -export default (value) => { - /** - * The expected value - * @type {string} - */ - const datesValue = $("#controlButton-dates-value").getText(); - expect(datesValue).toEqual( - value, - `"daterange" should be ${value} but was ${datesValue}` - ); -}; diff --git a/tests/src/support/check/checkMatsGraphPlotType.js b/tests/src/support/check/checkMatsGraphPlotType.js deleted file mode 100644 index 484e67ca4..000000000 --- a/tests/src/support/check/checkMatsGraphPlotType.js +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Check if the given elements contains text - * @param {String} plotType The text to check against - */ -export default (plotType) => { - /** - * Check that the header contains the plot type - * @plotType {String} - */ - const command = "getText"; - - const stringExpectedText = plotType; - - const elem = $("#header"); - elem.waitForDisplayed(); - const text = elem[command](); - - expect(text).toContain(stringExpectedText); -}; diff --git a/tests/src/support/check/checkMatsInfoMessage.js b/tests/src/support/check/checkMatsInfoMessage.js deleted file mode 100644 index 2cad7753f..000000000 --- a/tests/src/support/check/checkMatsInfoMessage.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Check the message in the Mats info modal - * @param {String} message the modal message - */ -export default (message) => { - const command = "getText"; - const expectedText = message; - const elem = $("#info").$(".modal-body").$("

"); - elem.waitForDisplayed(); - const text = elem[command](); - // notice that the expectedText contains the actual text. - // that is because the expected text has a leading "Info: " - // that the actual modal text selector filters out - expect(expectedText).toContain( - text, - `The info modal does not contain the expected text ${expectedText}` - ); -}; diff --git a/tests/src/support/check/checkMatsLegendListContains.js b/tests/src/support/check/checkMatsLegendListContains.js deleted file mode 100644 index a6bb4d1c8..000000000 --- a/tests/src/support/check/checkMatsLegendListContains.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Check if the given elements contains text - * @param {String} expected The textual list to check against - */ -export default (expected) => { - /** - * Check that the curve list contains specific curve label - * @curveNumber {Number} - */ - const expectedList = expected.split(",").sort(); - const actualList = $$(".legendtext") - .map((elem) => elem.getText().split(":")[0]) - .sort(); - const expectedText = expectedList.join(","); - const actualText = actualList.join(","); - const matches = expectedText === actualText; - expect(matches).toBe( - true, - `expected list ${expectedList} does not match actualList ${actualList}` - ); -}; diff --git a/tests/src/support/check/checkMatsParameters.js b/tests/src/support/check/checkMatsParameters.js deleted file mode 100644 index 395139ad3..000000000 --- a/tests/src/support/check/checkMatsParameters.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Check if the previously stored parameters match the current parameters. - */ -export default () => { - const currentMatsParameters = $$(".control-button").map((element) => - element.getText() - ); - const matches = - currentMatsParameters.sort().join(",") === - browser.saveMatsParameters.sort().join(","); - expect(matches).toEqual( - true, - "saved MATS parameters do not match current parameters" - ); - // browser.debug(); -}; diff --git a/tests/src/support/check/checkMatsPlotNumber.js b/tests/src/support/check/checkMatsPlotNumber.js deleted file mode 100644 index 756710c14..000000000 --- a/tests/src/support/check/checkMatsPlotNumber.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Check if the given elements contains text - * @param {number} curveNumber The text to check against - */ -export default (curveNumber) => { - /** - * Check that the graph contains curveNumber of curves - * @curveNumber {Number} - * @type {String} - */ - $("#curves").waitForDisplayed(20000); - // use the heatMapVisibility button (it's a unique selector) to count the curves - const curveItems = $$(".traces"); - // eslint-disable-next-line no-template-curly-in-string - expect(curveItems).toHaveLength(curveNumber, 'Should have "${curveNumber}" curves'); -}; diff --git a/tests/src/support/check/checkModal.js b/tests/src/support/check/checkModal.js deleted file mode 100644 index 6fe2c36b9..000000000 --- a/tests/src/support/check/checkModal.js +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Check if a modal was opened - * @param {String} modalType The type of modal that is expected (alertbox, - * confirmbox or prompt) - * @param {String} falseState Whether to check if the modal was opened or not - */ -export default (modalType, falseState) => { - /** - * The text of the prompt - * @type {String} - */ - let promptText = ""; - - try { - promptText = browser.getAlertText(); - - if (falseState) { - expect(promptText).not.toEqual( - null, - `A ${modalType} was opened when it shouldn't` - ); - } - } catch (e) { - if (!falseState) { - expect(promptText).toEqual( - null, - `A ${modalType} was not opened when it should have been` - ); - } - } -}; diff --git a/tests/src/support/check/checkModalText.js b/tests/src/support/check/checkModalText.js deleted file mode 100644 index 2b39da90d..000000000 --- a/tests/src/support/check/checkModalText.js +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Check the text of a modal - * @param {String} modalType The type of modal that is expected - * (alertbox, confirmbox or prompt) - * @param {String} falseState Whether to check if the text matches or not - * @param {String} expectedText The text to check against - */ -export default (modalType, falseState, expectedText) => { - try { - /** - * The text of the current modal - * @type {String} - */ - const text = browser.getAlertText(); - - if (falseState) { - expect(text).not.toEqual( - expectedText, - `Expected the text of ${modalType} not to equal ` + `"${expectedText}"` - ); - } else { - expect(text).toEqual( - expectedText, - `Expected the text of ${modalType} to equal ` + - `"${expectedText}", instead found "${text}"` - ); - } - } catch (e) { - throw new Error(`A ${modalType} was not opened when it should have been opened`); - } -}; diff --git a/tests/src/support/check/checkNewWindow.js b/tests/src/support/check/checkNewWindow.js deleted file mode 100644 index cd42026bf..000000000 --- a/tests/src/support/check/checkNewWindow.js +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Check if a new window or tab is opened - * @param {String} obsolete The type of opened object (window or tab) - * @param {String} falseCase Whether to check if a new window/tab was opened - * or not - */ -export default (obsolete, falseCase) => { - /** - * The handles of all open windows/tabs - * @type {Object} - */ - const windowHandles = browser.getWindowHandles(); - - if (falseCase) { - expect(windowHandles).toHaveLength(1, "A new window should not have been opened"); - } else { - expect(windowHandles).not.toHaveLength(1, "A new window has been opened"); - } -}; diff --git a/tests/src/support/check/checkOffset.js b/tests/src/support/check/checkOffset.js deleted file mode 100644 index 0722c56b0..000000000 --- a/tests/src/support/check/checkOffset.js +++ /dev/null @@ -1,36 +0,0 @@ -/** - * Check the offset of the given element - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the offset matches - * or not - * @param {String} expectedPosition The position to check against - * @param {String} axis The axis to check on (x or y) - */ -export default (selector, falseCase, expectedPosition, axis) => { - /** - * Get the location of the element on the given axis - * @type {[type]} - */ - const location = $(selector).getLocation(axis); - - /** - * Parsed expected position - * @type {Int} - */ - const intExpectedPosition = parseFloat(expectedPosition); - - if (falseCase) { - expect(location).not.toEqual( - intExpectedPosition, - `Element "${selector}" should not be positioned at ` + - `${intExpectedPosition}px on the ${axis} axis` - ); - } else { - expect(location).toEqual( - intExpectedPosition, - `Element "${selector}" should be positioned at ` + - `${intExpectedPosition}px on the ${axis} axis, but was found ` + - `at ${location}px` - ); - } -}; diff --git a/tests/src/support/check/checkParameterValue.js b/tests/src/support/check/checkParameterValue.js deleted file mode 100644 index b0fabf15b..000000000 --- a/tests/src/support/check/checkParameterValue.js +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Check the selected state of the given element - * @param {String} parameter the selection parameter - * @param {String} option the selected option - */ -export default (parameter, option) => { - /** - * The selected state - * @type {Boolean} - */ - const isSelected = $(parameter).isSelected(); - - if (option) { - expect(isSelected).not.toEqual(true, `"${option}" should not be selected`); - } else { - expect(isSelected).toEqual(true, `"${option}" should be selected`); - } -}; diff --git a/tests/src/support/check/checkProperty.js b/tests/src/support/check/checkProperty.js deleted file mode 100644 index 9c0426db2..000000000 --- a/tests/src/support/check/checkProperty.js +++ /dev/null @@ -1,53 +0,0 @@ -/** - * Check the given property of the given element - * @param {String} isCSS Whether to check for a CSS property or an - * attribute - * @param {String} attrName The name of the attribute to check - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the value of the - * attribute matches or not - * @param {String} expectedValue The value to match against - */ -export default (isCSS, attrName, selector, falseCase, expectedValue) => { - /** - * The command to use for fetching the expected value - * @type {String} - */ - const command = isCSS ? "getCSSProperty" : "getAttribute"; - - /** - * Te label to identify the attribute by - * @type {String} - */ - const attrType = isCSS ? "CSS attribute" : "Attribute"; - - /** - * The actual attribute value - * @type {Mixed} - */ - let attributeValue = $(selector)[command](attrName); - - // eslint-disable-next-line - expectedValue = isFinite(expectedValue) ? parseFloat(expectedValue) : expectedValue; - - /** - * when getting something with a color or font-weight WebdriverIO returns a - * object but we want to assert against a string - */ - if (attrName.match(/(color|font-weight)/)) { - attributeValue = attributeValue.value; - } - if (falseCase) { - expect(attributeValue).not.toEqual( - expectedValue, - `${attrType}: ${attrName} of element "${selector}" should ` + - `not contain "${attributeValue}"` - ); - } else { - expect(attributeValue).toEqual( - expectedValue, - `${attrType}: ${attrName} of element "${selector}" should ` + - `contain "${attributeValue}", but "${expectedValue}"` - ); - } -}; diff --git a/tests/src/support/check/checkSelected.js b/tests/src/support/check/checkSelected.js deleted file mode 100644 index 1e961cde3..000000000 --- a/tests/src/support/check/checkSelected.js +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Check the selected state of the given element - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the element is elected or - * not - */ -export default (selector, falseCase) => { - /** - * The selected state - * @type {Boolean} - */ - const isSelected = $(selector).isSelected(); - - if (falseCase) { - expect(isSelected).not.toEqual(true, `"${selector}" should not be selected`); - } else { - expect(isSelected).toEqual(true, `"${selector}" should be selected`); - } -}; diff --git a/tests/src/support/check/checkTitle.js b/tests/src/support/check/checkTitle.js deleted file mode 100644 index e5733ad87..000000000 --- a/tests/src/support/check/checkTitle.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Check the title of the current browser window - * @param {Type} falseCase Whether to check if the title matches the - * expected value or not - * @param {Type} expectedTitle The expected title - */ -export default (falseCase, expectedTitle) => { - /** - * The title of the current browser window - * @type {String} - */ - const title = browser.getTitle(); - - if (falseCase) { - expect(title).not.toEqual( - expectedTitle, - `Expected title not to be "${expectedTitle}"` - ); - } else { - expect(title).toEqual( - expectedTitle, - `Expected title to be "${expectedTitle}" but found "${title}"` - ); - } -}; diff --git a/tests/src/support/check/checkTitleContains.js b/tests/src/support/check/checkTitleContains.js deleted file mode 100644 index 8e10efc04..000000000 --- a/tests/src/support/check/checkTitleContains.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Check the title of the current browser window contains expected text/title - * @param {Type} falseCase Whether to check if the title contains the - * expected value or not - * @param {Type} expectedTitle The expected title - */ -export default (falseCase, expectedTitle) => { - /** - * The actual title of the current browser window - * @type {String} - */ - const title = browser.getTitle(); - - if (falseCase) { - expect(title).not.toContain( - expectedTitle, - `Expected title not to contain "${expectedTitle}"` - ); - } else { - expect(title).toContain( - expectedTitle, - `Expected title to contain "${expectedTitle}" but found "${title}"` - ); - } -}; diff --git a/tests/src/support/check/checkURL.js b/tests/src/support/check/checkURL.js deleted file mode 100644 index d14355504..000000000 --- a/tests/src/support/check/checkURL.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Check the URL of the given browser window - * @param {String} falseCase Whether to check if the URL matches the - * expected value or not - * @param {String} expectedUrl The expected URL to check against - */ -export default (falseCase, expectedUrl) => { - /** - * The current browser window's URL - * @type {String} - */ - const currentUrl = browser.getUrl(); - - if (falseCase) { - expect(currentUrl).not.toEqual( - expectedUrl, - `expected url not to be "${currentUrl}"` - ); - } else { - expect(currentUrl).toEqual( - expectedUrl, - `expected url to be "${expectedUrl}" but found ` + `"${currentUrl}"` - ); - } -}; diff --git a/tests/src/support/check/checkURLPath.js b/tests/src/support/check/checkURLPath.js deleted file mode 100644 index 722f6681d..000000000 --- a/tests/src/support/check/checkURLPath.js +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Check if the current URL path matches the given path - * @param {String} falseCase Whether to check if the path matches the - * expected value or not - * @param {String} expectedPath The expected path to match against - */ -export default (falseCase, expectedPath) => { - /** - * The URL of the current browser window - * @type {String} - */ - let currentUrl = browser.getUrl().replace(/http(s?):\/\//, ""); - - /** - * The base URL of the current browser window - * @type {Object} - */ - const domain = `${currentUrl.split("/")[0]}`; - - currentUrl = currentUrl.replace(domain, ""); - - if (falseCase) { - expect(currentUrl).not.toEqual( - expectedPath, - `expected path not to be "${currentUrl}"` - ); - } else { - expect(currentUrl).toEqual( - expectedPath, - `expected path to be "${expectedPath}" but found ` + `"${currentUrl}"` - ); - } -}; diff --git a/tests/src/support/check/checkWithinViewport.js b/tests/src/support/check/checkWithinViewport.js deleted file mode 100644 index eeda306f5..000000000 --- a/tests/src/support/check/checkWithinViewport.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Check if the given element is visible inside the current viewport - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the element is visible - * within the current viewport or not - */ -export default (selector, falseCase) => { - /** - * The state of visibility of the given element inside the viewport - * @type {Boolean} - */ - const isDisplayed = $(selector).isDisplayedInViewport(); - - if (falseCase) { - expect(isDisplayed).not.toEqual( - true, - `Expected element "${selector}" to be outside the viewport` - ); - } else { - expect(isDisplayed).toEqual( - true, - `Expected element "${selector}" to be inside the viewport` - ); - } -}; diff --git a/tests/src/support/check/compareText.js b/tests/src/support/check/compareText.js deleted file mode 100644 index 578500cb4..000000000 --- a/tests/src/support/check/compareText.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Compare the contents of two elements with each other - * @param {String} selector1 Element selector for the first element - * @param {String} falseCase Whether to check if the contents of both - * elements match or not - * @param {String} selector2 Element selector for the second element - */ -export default (selector1, falseCase, selector2) => { - /** - * The text of the first element - * @type {String} - */ - const text1 = $(selector1).getText(); - - /** - * The text of the second element - * @type {String} - */ - const text2 = $(selector2).getText(); - - if (falseCase) { - expect(text1).not.toEqual(text2, `Expected text not to be "${text1}"`); - } else { - expect(text1).toEqual(text2, `Expected text to be "${text1}" but found "${text2}"`); - } -}; diff --git a/tests/src/support/check/isDisplayed.js b/tests/src/support/check/isDisplayed.js deleted file mode 100644 index b402d9ce3..000000000 --- a/tests/src/support/check/isDisplayed.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Check if the given element is (not) visible - * @param {String} selector Element selector - * @param {String} falseCase Check for a visible or a hidden element - */ -export default (selector, falseCase) => { - /** - * Visible state of the give element - * @type {String} - */ - const isDisplayed = $(selector).isDisplayed(); - - if (falseCase) { - expect(isDisplayed).not.toEqual( - true, - `Expected element "${selector}" not to be displayed` - ); - } else { - expect(isDisplayed).toEqual(true, `Expected element "${selector}" to be displayed`); - } -}; diff --git a/tests/src/support/check/isEnabled.js b/tests/src/support/check/isEnabled.js deleted file mode 100644 index ef9a9f78b..000000000 --- a/tests/src/support/check/isEnabled.js +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Check if the given selector is enabled - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the given selector - * is enabled or not - */ -export default (selector, falseCase) => { - /** - * The enabled state of the given selector - * @type {Boolean} - */ - const isEnabled = $(selector).isEnabled(); - - if (falseCase) { - expect(isEnabled).not.toEqual( - true, - `Expected element "${selector}" not to be enabled` - ); - } else { - expect(isEnabled).toEqual(true, `Expected element "${selector}" to be enabled`); - } -}; diff --git a/tests/src/support/check/isExisting.js b/tests/src/support/check/isExisting.js deleted file mode 100644 index be5e032a6..000000000 --- a/tests/src/support/check/isExisting.js +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Check if the given element exists in the current DOM - * @param {String} selector Element selector - * @param {String} falseCase Whether to check if the element exists or not - */ -export default (selector, falseCase) => { - /** - * Elements found in the DOM - * @type {Object} - */ - const elements = $$(selector); - - if (falseCase) { - expect(elements).toHaveLength(0, `Expected element "${selector}" not to exist`); - } else { - expect(elements.length).toBeGreaterThan( - 0, - `Expected element "${selector}" to exist` - ); - } -}; diff --git a/tests/src/support/check/isGroupValue.js b/tests/src/support/check/isGroupValue.js deleted file mode 100644 index 92e74be8f..000000000 --- a/tests/src/support/check/isGroupValue.js +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Check the current value (selected element of named group) of the given radio group selector against a provided value - * @param {String} selector Element group selector - * @param {String} the expected value - */ -export default (name, value) => { - /** - * The selected state - * @type {Boolean} - */ - const selector = `input[name=${name}]`; - const selectedValue = $(selector).getValue(); - expect(selectedValue).toBe(value, `"${name}" should should have selected ${value}`); -}; diff --git a/tests/src/support/check/isMainDisplayed.js b/tests/src/support/check/isMainDisplayed.js deleted file mode 100644 index 75a1e84aa..000000000 --- a/tests/src/support/check/isMainDisplayed.js +++ /dev/null @@ -1,11 +0,0 @@ -/** - * Check if the graph page is visible - */ -export default () => { - /** - * Is the main page visible? - */ - const isDisplayed = $("#plotType").isDisplayed(); - - expect(isDisplayed).toEqual(true, 'Expected element "plotType" to be displayed'); -}; diff --git a/tests/src/support/check/isMatsButtonEnabled.js b/tests/src/support/check/isMatsButtonEnabled.js deleted file mode 100644 index 603ac35f6..000000000 --- a/tests/src/support/check/isMatsButtonEnabled.js +++ /dev/null @@ -1,42 +0,0 @@ -/** - * Check if the given MATS button is enabled - * @param {String} buttonLabel Button label - * @param {String} falseCase Whether to check if the button is enabled or not - * */ -export default (buttonLabel, falseCase) => { - /** - * Visible state of the give element - * @type {String} - * @type {Boolean} - */ - let selector; - let boolFalseCase; - switch (buttonLabel) { - case "Add Curve": - selector = $("#add"); - break; - case "Back": - selector = $("#backButton"); - break; - case "Plot Matched": - selector = $("#plotMatched"); - break; - case "Plot Unmatched": - selector = $("#plotUnmatched"); - break; - default: - } - - if (typeof falseCase === "undefined") { - boolFalseCase = false; - } else { - boolFalseCase = !!falseCase; - } - if (boolFalseCase) { - const isEnabled = selector.isEnabled(); - expect(isEnabled).toEqual(false, `Expected element "${buttonLabel}" to be enabled`); - } else { - const isEnabled = selector.waitForEnabled(); - expect(isEnabled).toEqual(true, `Expected element "${buttonLabel}" to be enabled`); - } -}; diff --git a/tests/src/support/check/isMatsButtonVisible.js b/tests/src/support/check/isMatsButtonVisible.js deleted file mode 100644 index 7af29d2a9..000000000 --- a/tests/src/support/check/isMatsButtonVisible.js +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Check if the given MATS button is visible - * @param {String} buttonLabel Button label */ -export default (buttonLabel) => { - /** - * Visible state of the give element - * @type {String} - */ - let selector; - if (buttonLabel === "Remove All") { - selector = $("#remove-all"); - } else if (buttonLabel === "Remove all the curves") { - // this is the 'Remove all the curves' confirm button - // I don't know why there are two of the confirm buttons - // there has to be a better way to handle this - // eslint-disable-next-line prefer-destructuring - selector = $$("#confirm-remove-all")[1]; - } else if (buttonLabel.match("Remove curve .*")) { - // this is the 'Remove curve curvelabel' confirm button - // $$('#curve-list-remove*=Curve0').length - const cPart = buttonLabel.replace("Remove curve ", ""); - // eslint-disable-next-line no-template-curly-in-string - selector = $(`#curve-list-remove*=${cPart}`); - } else if (buttonLabel.match("Remove .*")) { - // This is the 'Remove curvelabel' remove button - // $$('#curve-list-remove*=Curve0').length - const cPart = buttonLabel.replace("Remove ", ""); - // eslint-disable-next-line no-template-curly-in-string - selector = $(`#curve-list-remove*=${cPart}`); - } else { - switch (buttonLabel) { - case "Add Curve": - selector = $("#add"); - break; - case "Back": - selector = $("#backButton"); - break; - case "Plot Matched": - selector = $("#plotMatched"); - break; - case "Plot Unmatched": - selector = $("#plotUnmatched"); - break; - default: - } - } - const ms = 10000; - selector.waitForDisplayed({ timeout: ms }); - const isDisplayed = selector.isDisplayed({ timeout: ms }); - expect(isDisplayed).toEqual( - true, - `Expected element "${buttonLabel}" to be displayed` - ); -}; diff --git a/tests/src/support/check/isMatsCurveColor.js b/tests/src/support/check/isMatsCurveColor.js deleted file mode 100644 index 7cd36e6fc..000000000 --- a/tests/src/support/check/isMatsCurveColor.js +++ /dev/null @@ -1,14 +0,0 @@ -/** - * Check the curve has the correct color - * @param {String} label of the curve - * @param {String} color of the curve - */ - -export default (curve, color) => { - const actualColor = $(`#${curve}-color-value`).getValue(); - const expectedColor = color; - expect(actualColor).toEqual( - expectedColor, - `expected color ${expectedColor} does not match actualColor ${actualColor}` - ); -}; diff --git a/tests/src/support/check/isMatsGraphDisplayed.js b/tests/src/support/check/isMatsGraphDisplayed.js deleted file mode 100644 index f2c32faeb..000000000 --- a/tests/src/support/check/isMatsGraphDisplayed.js +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Check if the graph page becomes visible - */ - -import pause from "../action/pause"; - -export default () => { - /** - * Is the graph page visible within ms milliseconds? - */ - - let count = 0; - let isDisplayed = false; - while (count < 10 && isDisplayed !== true) { - isDisplayed = $("#graph-container").waitForDisplayed(); - if (isDisplayed !== true) { - pause(1000); - } - count += 1; - } - expect(isDisplayed).toEqual( - true, - 'Expected element "#graph-container" to be displayed' - ); -}; diff --git a/tests/src/support/check/isMatsInfoVisible.js b/tests/src/support/check/isMatsInfoVisible.js deleted file mode 100644 index 49167c773..000000000 --- a/tests/src/support/check/isMatsInfoVisible.js +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Visible state of the info modal - * @type {String} - */ - -export default (falseCase) => { - const selector = $("#info"); - if (falseCase) { - const isDisplayed = selector.isDisplayed(); - expect(isDisplayed).not.toEqual( - true, - "Expected info modal to NOT be displayed and it is visible" - ); - } else { - const ms = 120000; - selector.waitForDisplayed({ timeout: ms }); - const isDisplayed = selector.isDisplayed({ timeout: ms }); - expect(isDisplayed).toEqual( - true, - "Expected info modal to be displayed and it is not visible" - ); - } -}; diff --git a/tests/src/support/check/isMatsPlotFormat.js b/tests/src/support/check/isMatsPlotFormat.js deleted file mode 100644 index 34f4054f8..000000000 --- a/tests/src/support/check/isMatsPlotFormat.js +++ /dev/null @@ -1,17 +0,0 @@ -/** - * Check the current value (selected element of named group) - * of the plot-format selector against a provided value - * @param {String} value the expected value - */ -export default (value) => { - /** - * The selected state - * @type {Boolean} - */ - const selected = $$("input[name=plotFormat]").find((elem) => elem.isSelected()); - const selectedValue = $(selected).getValue(); - expect(selectedValue).toBe( - value, - `"plot format" should be ${value} and is ${selectedValue}` - ); -}; diff --git a/tests/src/support/check/isMatsPlotType.js b/tests/src/support/check/isMatsPlotType.js deleted file mode 100644 index 4de455c4c..000000000 --- a/tests/src/support/check/isMatsPlotType.js +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Check the current value (selected element of named group) - * of the plot-type selector against a provided value - * @param {String} value the expected value - */ -export default (value) => { - /** - * The selected state - * @type {Boolean} - */ - const selectedValue = $("#plotTypes-selector").getValue(); - expect(selectedValue).toBe( - value, - `"plot type" should be ${value} and is ${selectedValue}` - ); -}; diff --git a/tests/src/support/check/isMatsSelectedOption.js b/tests/src/support/check/isMatsSelectedOption.js deleted file mode 100644 index dacd0ba1e..000000000 --- a/tests/src/support/check/isMatsSelectedOption.js +++ /dev/null @@ -1,29 +0,0 @@ -import pause from "../action/pause"; - -export default (parameter, value) => { - /** - * Check the selected state of the given element - * @param {String} parameter paramaeter - * @param {String} value the selected option - */ - $(`#controlButton-${parameter}-value`).waitForDisplayed(); - $(`#controlButton-${parameter}-value`).scrollIntoView(); - let count = 0; - let text = ""; - while (count < 10 && text !== value) { - text = $(`#controlButton-${parameter}-value`).getText(); - if (text !== value) { - if (text.includes(" .. ") && !value.includes(" .. ")) { - // this is a multiselect, which have different display formats than regular selectors. - // we need to reformat our expected value to match. - value = `${value} .. ${value}`; - } - pause(1000); - } - count += 1; - } - expect(text).toEqual( - value, - `Expexted ${text} to be ${value} for parameter: ${parameter}` - ); -}; diff --git a/tests/src/support/hooks.js b/tests/src/support/hooks.js deleted file mode 100644 index 34bb7274c..000000000 --- a/tests/src/support/hooks.js +++ /dev/null @@ -1,151 +0,0 @@ -// -// ===== -// Hooks -// ===== -// WebdriverIO provides a several hooks you can use to interfere the test process in order to -// enhance it and build services around it. You can either apply a single function to it or -// an array of methods. If one of them returns with a promise, -// WebdriverIO will wait until that promise is resolved to continue. -// -exports.hooks = { - /** - * Gets executed once before all workers get launched. - * @param {Object} config wdio configuration object - * @param {Array.} capabilities list of capabilities details - */ - // onPrepare: function (config, capabilities) { - // }, - /** - * Gets executed before a worker process is spawned & can be used to initialize specific service - * for that worker as well as modify runtime environments in an async fashion. - * @param {String} cid capability id (e.g 0-0) - * @param {[type]} caps object containing capabilities for session - * @param {[type]} specs specs to be run in the worker process - * @param {[type]} args object that will be merged with the main - * configuration once worker is initialized - * @param {[type]} execArgv list of string arguments passed to the worker process - */ - // onWorkerStart: function (cid, caps, specs, args, execArgv) { - // }, - /** - * Gets executed just before initializing the webdriver session and test framework. - * It allows you to manipulate configurations depending on the capability or spec. - * @param {Object} config wdio configuration object - * @param {Array.} capabilities list of capabilities details - * @param {Array.} specs List of spec file paths that are to be run - */ - // beforeSession: function (config, capabilities, specs) { - // }, - /** - * Gets executed before test execution begins. At this point you can access to all global - * variables like `browser`. It is the perfect place to define custom commands. - * @param {Array.} capabilities list of capabilities details - * @param {Array.} specs List of spec file paths that are to be run - */ - // before: function (capabilities, specs) { - // }, - /** - * Gets executed before the suite starts. - * @param {Object} suite suite details - */ - // beforeSuite: function (suite) { - // }, - /** - * This hook gets executed _before_ every hook within the suite starts. - * (For example, this runs before calling `before`, `beforeEach`, `after`) - * - * (`stepData` and `world` are Cucumber-specific.) - * - */ - // beforeHook: function (test, context, stepData, world) { - // }, - /** - * Hook that gets executed _after_ every hook within the suite ends. - * (For example, this runs after calling `before`, `beforeEach`, `after`, `afterEach` in Mocha.) - * - * (`stepData` and `world` are Cucumber-specific.) - */ - // afterHook:function(test,context,{error, result, duration, passed, retries}, stepData,world) { - // }, - /** - * Function to be executed before a test (in Mocha/Jasmine) starts. - */ - // beforeTest: function (test, context) { - // }, - /** - * Runs before a WebdriverIO command is executed. - * @param {String} commandName hook command name - * @param {Array} args arguments that the command would receive - */ - // beforeCommand: function (commandName, args) { - // }, - /** - * Runs after a WebdriverIO command gets executed - * @param {String} commandName hook command name - * @param {Array} args arguments that command would receive - * @param {Number} result 0 - command success, 1 - command error - * @param {Object} error error object, if any - */ - // afterCommand: function (commandName, args, result, error) { - // }, - /** - * Function to be executed after a test (in Mocha/Jasmine) - */ - // afterTest: function (test, context, {error, result, duration, passed, retries}) { - // }, - /** - * Hook that gets executed after the suite has ended. - * @param {Object} suite suite details - */ - // afterSuite: function (suite) { - // }, - /** - * Gets executed after all tests are done. You still have access to all global variables from - * the test. - * @param {Number} result 0 - test pass, 1 - test fail - * @param {Array.} capabilities list of capabilities details - * @param {Array.} specs List of spec file paths that ran - */ - // after: function (result, capabilities, specs) { - // }, - /** - * Gets executed right after terminating the webdriver session. - * @param {Object} config wdio configuration object - * @param {Array.} capabilities list of capabilities details - * @param {Array.} specs List of spec file paths that ran - */ - // afterSession: function (config, capabilities, specs) { - // }, - /** - * Gets executed after all workers have shut down and the process is about to exit. - * An error thrown in the `onComplete` hook will result in the test run failing. - * @param {Object} exitCode 0 - success, 1 - fail - * @param {Object} config wdio configuration object - * @param {Array.} capabilities list of capabilities details - * @param {} results object containing test results - */ - // onComplete: function (exitCode, config, capabilities, results) { - // }, - /** - * Gets executed when a refresh happens. - * @param {String} oldSessionId session ID of the old session - * @param {String} newSessionId session ID of the new session - */ - // onReload: function (oldSessionId, newSessionId) { - // }, - /** - * Cucumber-specific hooks - */ - // beforeFeature: function (uri, feature, scenarios) { - // }, - // beforeScenario: function (uri, feature, scenario, sourceLocation) { - // }, - // beforeStep: function ({uri, feature, step}, context) { - // }, - // afterStep: function ({uri, feature, step}, context, {error, result, duration, passed}) { - // }, - // afterScenario: function (uri, feature, scenario, result, sourceLocation) { - // }, - // afterFeature: function (uri, feature, scenarios) { - // } -}; diff --git a/tests/src/support/lib/checkIfElementExists.js b/tests/src/support/lib/checkIfElementExists.js deleted file mode 100644 index 1d03cca68..000000000 --- a/tests/src/support/lib/checkIfElementExists.js +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Check if the given element exists in the DOM one or more times - * @param {String} selector Element selector - * @param {Boolean} falseCase Check if the element (does not) exists - * @param {Number} exactly Check if the element exists exactly this number - * of times - */ -export default (selector, falseCase, exactly) => { - /** - * The number of elements found in the DOM - * @type {Int} - */ - const nrOfElements = $$(selector); - - if (falseCase === true) { - expect(nrOfElements).toHaveLength( - 0, - `Element with selector "${selector}" should not exist on the page` - ); - } else if (exactly) { - expect(nrOfElements).toHaveLength( - exactly, - `Element with selector "${selector}" should exist exactly ` + `${exactly} time(s)` - ); - } else { - expect(nrOfElements.length).toBeGreaterThanOrEqual( - 1, - `Element with selector "${selector}" should exist on the page` - ); - } -}; From bfb87c55df1fc3bd9e659ee10f96d0109818d720 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 16 Aug 2024 14:21:53 -0600 Subject: [PATCH 37/55] Further linting through graph.js --- .../imports/startup/client/graph_util.js | 29 +- .../imports/startup/client/routes.js | 6 +- .../imports/startup/client/select_util.js | 166 ++-- .../imports/startup/server/data_util.js | 2 +- .../imports/startup/server/publications.js | 4 +- .../imports/startup/ui/layouts/appBody.html | 2 +- .../imports/startup/ui/layouts/appBody.js | 7 +- meteor_packages/mats-common/lib/math_util.js | 99 --- meteor_packages/mats-common/lib/param_util.js | 242 +++--- meteor_packages/mats-common/lib/plot_util.js | 130 +-- meteor_packages/mats-common/lib/regression.js | 85 +- meteor_packages/mats-common/package.js | 6 +- .../mats-common/public/MATSReleaseNotes.html | 1 + meteor_packages/mats-common/server/main.js | 8 +- .../mats-common/templates/CustomHome.html | 2 +- .../mats-common/templates/Home.html | 2 +- .../mats-common/templates/ScorecardHome.html | 2 +- .../mats-common/templates/about/about.html | 2 +- .../mats-common/templates/about/about.js | 8 +- .../changePlotType/changePlotType.js | 3 + .../templates/curves/curve_item.js | 191 +++-- .../templates/curves/curve_list.js | 18 +- .../curves/curve_param_item_group.js | 40 +- .../templates/curves/scorecard_curve_list.js | 7 +- .../mats-common/templates/error/error.js | 3 + .../mats-common/templates/footer/footer.js | 1 + .../graph/displayFunctions/graph_plotly.js | 31 - .../mats-common/templates/graph/graph.js | 801 +++++++++--------- .../graphStandAlone/graphStandAlone.html | 2 +- .../graphStandAlone/graphStandAlone.js | 159 ++-- 30 files changed, 1017 insertions(+), 1042 deletions(-) delete mode 100644 meteor_packages/mats-common/lib/math_util.js delete mode 100644 meteor_packages/mats-common/templates/graph/displayFunctions/graph_plotly.js diff --git a/meteor_packages/mats-common/imports/startup/client/graph_util.js b/meteor_packages/mats-common/imports/startup/client/graph_util.js index f5a1d5fee..a2e6d217c 100644 --- a/meteor_packages/mats-common/imports/startup/client/graph_util.js +++ b/meteor_packages/mats-common/imports/startup/client/graph_util.js @@ -2,7 +2,7 @@ * Copyright (c) 2021 Colorado State University and Regents of the University of Colorado. All rights reserved. */ -import { matsTypes } from "meteor/randyp:mats-common"; +import { matsTypes, matsCurveUtils } from "meteor/randyp:mats-common"; /* global $, Session */ /* eslint-disable no-console */ @@ -562,6 +562,32 @@ const setDefaultView = function () { window.onbeforeunload = null; }; +const graphPlotly = function () { + // get plot info + const route = Session.get("route"); + + // get dataset info and options + const resultSet = matsCurveUtils.getGraphResult(); + if (resultSet === null || resultSet === undefined || resultSet.data === undefined) { + return false; + } + + // set options + const { options } = resultSet; + if (route !== undefined && route !== "") { + options.selection = []; + } + + // initialize show/hide button labels + const dataset = resultSet.data; + if (Session.get("graphPlotType") !== matsTypes.PlotTypes.map) { + setNoDataLabels(dataset); + } else { + setNoDataLabelsMap(dataset); + } + return null; +}; + const downloadFile = function (fileURL, fileName) { // for non-IE if (!window.ActiveXObject) { @@ -609,6 +635,7 @@ export default matsGraphUtils = { setGraphView, standAloneSetGraphView, setDefaultView, + graphPlotly, downloadFile, setScorecardDisplayView, }; diff --git a/meteor_packages/mats-common/imports/startup/client/routes.js b/meteor_packages/mats-common/imports/startup/client/routes.js index aca60f44c..26ddabdf3 100644 --- a/meteor_packages/mats-common/imports/startup/client/routes.js +++ b/meteor_packages/mats-common/imports/startup/client/routes.js @@ -44,7 +44,7 @@ FlowRouter.route("/JSON/:graphFunction/:key/:matching/:appName", { FlowRouter.route("/preview/:graphFunction/:key/:matching/:appName", { name: "preview", action(params) { - this.render("GraphStandAlone", params); + this.render("graphStandAlone", params); }, }); @@ -111,7 +111,7 @@ FlowRouter.route( { name: "preview", action(params) { - this.render("GraphStandAlone", params); + this.render("graphStandAlone", params); }, } ); @@ -185,7 +185,7 @@ FlowRouter.route( { name: "preview", action(params) { - this.render("GraphStandAlone", params); + this.render("graphStandAlone", params); }, } ); diff --git a/meteor_packages/mats-common/imports/startup/client/select_util.js b/meteor_packages/mats-common/imports/startup/client/select_util.js index 2860281a1..2c43088c8 100644 --- a/meteor_packages/mats-common/imports/startup/client/select_util.js +++ b/meteor_packages/mats-common/imports/startup/client/select_util.js @@ -9,28 +9,7 @@ import { matsTypes, } from "meteor/randyp:mats-common"; -// method to refresh the peers of the current selector -const refreshPeer = function (event, param) { - try { - const { peerName } = param; - if (peerName !== undefined) { - // refresh the peer - const targetParam = matsParamUtils.getParameterForName(peerName); - const targetId = `${targetParam.name}-${targetParam.type}`; - const targetElem = document.getElementById(targetId); - const refreshMapEvent = new CustomEvent("refresh", { - detail: { - refElement: null, - }, - }); - targetElem.dispatchEvent(refreshMapEvent); - } - refreshDependents(event, param); - } catch (e) { - e.message = `INFO: Error in select.js refreshPeer: ${e.message}`; - setInfo(e.message); - } -}; +/* global $, _, Session, setInfo */ // method to refresh the dependents of the current selector const refreshDependents = function (event, param) { @@ -43,10 +22,10 @@ const refreshDependents = function (event, param) { ) { // refresh the dependents let selectAllbool = false; - for (let i = 0; i < dependentNames.length; i++) { + for (let i = 0; i < dependentNames.length; i += 1) { const name = dependentNames[i]; const targetParam = matsParamUtils.getParameterForName(name); - var targetId; + let targetId; if (targetParam.type === matsTypes.InputTypes.dateRange) { targetId = `element-${targetParam.name}`; } else { @@ -75,14 +54,14 @@ const refreshDependents = function (event, param) { const select = true; if (targetElem.multiple && elements !== undefined && elements.length > 0) { if (selectAllbool) { - for (let i1 = 0; i1 < elements.length; i1++) { + for (let i1 = 0; i1 < elements.length; i1 += 1) { elements[i1].selected = select; } matsParamUtils.setValueTextForParamName(name, ""); } else { - const previously_selected = Session.get("selected"); - for (let i2 = 0; i2 < elements.length; i2++) { - if (_.indexOf(previously_selected, elements[i2].text) !== -1) { + const previouslySelected = Session.get("selected"); + for (let i2 = 0; i2 < elements.length; i2 += 1) { + if (_.indexOf(previouslySelected, elements[i2].text) !== -1) { elements[i2].selected = select; } } @@ -96,6 +75,29 @@ const refreshDependents = function (event, param) { } }; +// method to refresh the peers of the current selector +const refreshPeer = function (event, param) { + try { + const { peerName } = param; + if (peerName !== undefined) { + // refresh the peer + const targetParam = matsParamUtils.getParameterForName(peerName); + const targetId = `${targetParam.name}-${targetParam.type}`; + const targetElem = document.getElementById(targetId); + const refreshMapEvent = new CustomEvent("refresh", { + detail: { + refElement: null, + }, + }); + targetElem.dispatchEvent(refreshMapEvent); + } + refreshDependents(event, param); + } catch (e) { + e.message = `INFO: Error in select.js refreshPeer: ${e.message}`; + setInfo(e.message); + } +}; + // check for enable controlled - This select might have control of another selector const checkDisableOther = function (param, firstRender) { try { @@ -103,7 +105,7 @@ const checkDisableOther = function (param, firstRender) { // this param controls the enable/disable properties of at least one other param. // Use the options to enable disable that param. const controlledSelectors = Object.keys(param.disableOtherFor); - for (let i = 0; i < controlledSelectors.length; i++) { + for (let i = 0; i < controlledSelectors.length; i += 1) { const elem = matsParamUtils.getInputElementForParamName(param.name); if (!elem) { return; @@ -142,7 +144,7 @@ const checkHideOther = function (param, firstRender) { if (param.hideOtherFor !== undefined) { // this param controls the visibility of at least one other param. const controlledSelectors = Object.keys(param.hideOtherFor); - for (let i = 0; i < controlledSelectors.length; i++) { + for (let i = 0; i < controlledSelectors.length; i += 1) { const elem = matsParamUtils.getInputElementForParamName(param.name); if (!elem) { return; @@ -151,7 +153,7 @@ const checkHideOther = function (param, firstRender) { let selectedText; if (param.type === matsTypes.InputTypes.radioGroup) { const radioButtons = elem.getElementsByTagName("input"); - for (let ridx = 0; ridx < radioButtons.length; ridx++) { + for (let ridx = 0; ridx < radioButtons.length; ridx += 1) { if (radioButtons[ridx].checked) { selectedOptions = radioButtons[ridx].id.split("-radioGroup-"); selectedText = selectedOptions[selectedOptions.length - 1]; @@ -174,7 +176,7 @@ const checkHideOther = function (param, firstRender) { param.superiorRadioGroups !== undefined ) { // if a superior radio group wants the target element hidden and it already is, leave it be. - for (let sidx = 0; sidx < param.superiorRadioGroups.length; sidx++) { + for (let sidx = 0; sidx < param.superiorRadioGroups.length; sidx += 1) { const superiorName = param.superiorRadioGroups[sidx]; const superiorHideOtherFor = matsCollections.PlotParams.findOne({ name: superiorName, @@ -183,10 +185,14 @@ const checkHideOther = function (param, firstRender) { .getInputElementForParamName(superiorName) .getElementsByTagName("input"); let superiorSelectedText = ""; - for (let sidx = 0; sidx < superiorInputElementOptions.length; sidx++) { - if (superiorInputElementOptions[sidx].checked) { + for ( + let seidx = 0; + seidx < superiorInputElementOptions.length; + seidx += 1 + ) { + if (superiorInputElementOptions[seidx].checked) { const superiorSelectedOptions = - superiorInputElementOptions[sidx].id.split("-radioGroup-"); + superiorInputElementOptions[seidx].id.split("-radioGroup-"); superiorSelectedText = superiorSelectedOptions[superiorSelectedOptions.length - 1]; break; @@ -208,7 +214,7 @@ const checkHideOther = function (param, firstRender) { const otherInputElement = matsParamUtils.getInputElementForParamName( controlledSelectors[i] ); - var selectorControlElem; + let selectorControlElem; if ( (firstRender && param.default.toString() === @@ -242,10 +248,13 @@ const checkHideOther = function (param, firstRender) { selectorControlElem.purposelyHidden = false; } } - otherInputElement && + if ( + otherInputElement && otherInputElement.options && - otherInputElement.selectedIndex >= 0 && + otherInputElement.selectedIndex >= 0 + ) { otherInputElement.options[otherInputElement.selectedIndex].scrollIntoView(); + } } } checkDisableOther(param, firstRender); @@ -301,30 +310,30 @@ const refresh = function (event, paramName) { if (superiorDimensionality === 1) { sNames = superiorNames; } else { - sNames = superiorNames[0]; + [sNames] = superiorNames; } - for (var sn = 0; sn < sNames.length; sn++) { - var superiorElement = matsParamUtils.getInputElementForParamName(sNames[sn]); - var selectedSuperiorValue = + for (let sn = 0; sn < sNames.length; sn += 1) { + const superiorElement = matsParamUtils.getInputElementForParamName(sNames[sn]); + let selectedSuperiorValue = superiorElement.options[superiorElement.selectedIndex] === undefined ? matsParamUtils.getParameterForName(sNames[sn]).default : superiorElement.options[superiorElement.selectedIndex].text; if (sNames[sn].includes("statistic") && isMetexpress) { - selectedSuperiorValue = statisticTranslations[selectedSuperiorValue][0]; + [selectedSuperiorValue] = statisticTranslations[selectedSuperiorValue]; } superiors[0] = superiors[0] === undefined ? [] : superiors[0]; superiors[0].push({ element: superiorElement, value: selectedSuperiorValue }); } - for (var sNameIndex = 1; sNameIndex < superiorDimensionality; sNameIndex++) { + for (let sNameIndex = 1; sNameIndex < superiorDimensionality; sNameIndex += 1) { sNames = superiorNames[sNameIndex]; - for (var sn = 0; sn < sNames.length; sn++) { - var superiorElement = matsParamUtils.getInputElementForParamName(sNames[sn]); - var selectedSuperiorValue = + for (let sn = 0; sn < sNames.length; sn += 1) { + const superiorElement = matsParamUtils.getInputElementForParamName(sNames[sn]); + let selectedSuperiorValue = superiorElement.options[superiorElement.selectedIndex] === undefined ? matsParamUtils.getParameterForName(sNames[sn]).default : superiorElement.options[superiorElement.selectedIndex].text; if (sNames[sn].includes("statistic") && isMetexpress) { - selectedSuperiorValue = statisticTranslations[selectedSuperiorValue][0]; + [selectedSuperiorValue] = statisticTranslations[selectedSuperiorValue]; } superiors[sNameIndex] = superiors[sNameIndex] === undefined ? [] : superiors[sNameIndex]; @@ -409,7 +418,7 @@ const refresh = function (event, paramName) { } const brothers = []; - for (var i = 0; i < elems.length; i++) { + for (let i = 0; i < elems.length; i += 1) { if (elems[i].id.indexOf(name) >= 0 && elems[i].id !== elem.id) brothers.push(elems[i]); } @@ -424,15 +433,15 @@ const refresh = function (event, paramName) { // These are the ancestral options. if (param.optionsMap) { let firstSuperiorOptions = optionsMap; - var theseSuperiors = + const theseSuperiors = superiors === undefined || superiors.length === 0 ? [] : superiors[0]; for ( - var theseSuperiorsIndex = 0; + let theseSuperiorsIndex = 0; theseSuperiorsIndex < theseSuperiors.length; - theseSuperiorsIndex++ + theseSuperiorsIndex += 1 ) { - var superior = theseSuperiors[theseSuperiorsIndex]; - var selectedSuperiorValue = superior.value; + const superior = theseSuperiors[theseSuperiorsIndex]; + const selectedSuperiorValue = superior.value; if (isScorecard) { firstSuperiorOptions = firstSuperiorOptions[selectedSuperiorValue] !== undefined @@ -483,29 +492,29 @@ const refresh = function (event, paramName) { */ // need to get the actual options here - for (var sNameIndex = 1; sNameIndex < superiorDimensionality; sNameIndex++) { + for (let sNameIndex = 1; sNameIndex < superiorDimensionality; sNameIndex += 1) { // index down through the options for the list of superiors // starting with the most superior down through the least superior // and get the options list for the first set of superiors. // These are the ancestral options. let nextSuperiorOptions = optionsMap; - var theseSuperiors = + const theseSuperiors = superiors === undefined || superiors.length === 0 ? [] : superiors[sNameIndex]; for ( - var theseSuperiorsIndex = 0; + let theseSuperiorsIndex = 0; theseSuperiorsIndex < theseSuperiors.length; - theseSuperiorsIndex++ + theseSuperiorsIndex += 1 ) { - var superior = theseSuperiors[theseSuperiorsIndex]; - var selectedSuperiorValue = superior.value; + const superior = theseSuperiors[theseSuperiorsIndex]; + const selectedSuperiorValue = superior.value; nextSuperiorOptions = nextSuperiorOptions[selectedSuperiorValue]; } // since we now have multiple options we have to intersect them myOptions = _.intersection(myOptions, nextSuperiorOptions); } - if (myOptions === []) { + if (myOptions && myOptions.length === 0) { // none used - set to [] matsParamUtils.setValueTextForParamName(name, matsTypes.InputTypes.unused); } @@ -525,7 +534,7 @@ const refresh = function (event, paramName) { // optionGroups are an ordered map. It probably has options that are in the disabledOption list // which are used as markers in the select options pulldown. This is typical for models const optionsGroupsKeys = Object.keys(optionsGroups); - for (let k = 0; k < optionsGroupsKeys.length; k++) { + for (let k = 0; k < optionsGroupsKeys.length; k += 1) { if (myOptions === null) { myOptions = []; myOptions.push(optionsGroupsKeys[k]); @@ -544,7 +553,7 @@ const refresh = function (event, paramName) { return; } let firstGroup = true; - for (var i = 0; i < myOptions.length; i++) { + for (let i = 0; i < myOptions.length; i += 1) { const dIndex = disabledOptions === undefined ? -1 : disabledOptions.indexOf(myOptions[i]); if (dIndex >= 0) { @@ -621,8 +630,8 @@ const refresh = function (event, paramName) { ); } if (elem.selectedIndex >= 0) { - for (let svi = 0; svi < selectedSuperiorValues.length; svi++) { - superior = superiors[svi]; + for (let svi = 0; svi < selectedSuperiorValues.length; svi += 1) { + const superior = superiors[svi]; if ( matsParamUtils.getControlElementForParamName(superior.element.name) .offsetParent !== null @@ -646,17 +655,15 @@ const refresh = function (event, paramName) { matsParamUtils.setValueTextForParamName(name, matsTypes.InputTypes.unused); } else { elem.selectedIndex = 0; - elem && - elem.options && - elem.selectedIndex >= 0 && + if (elem && elem.options && elem.selectedIndex >= 0) { elem.options[elem.selectedIndex].scrollIntoView(); - elem && - elem.options && - elem.selectedIndex >= 0 && + } + if (elem && elem.options && elem.selectedIndex >= 0) { matsParamUtils.setValueTextForParamName( name, elem.options[elem.selectedIndex].text ); + } } } else if (param.multiple && selectedOptionOverlap.length > 0) { // need to manually select all the desired options @@ -677,19 +684,17 @@ const refresh = function (event, paramName) { matsParamUtils.setValueTextForParamName(name, selectedOptionOverlap); } else { elem.selectedIndex = selectedOptionIndex; - elem && - elem.options && - elem.selectedIndex >= 0 && + if (elem && elem.options && elem.selectedIndex >= 0) { elem.options[elem.selectedIndex].scrollIntoView(); - elem && - elem.options && - elem.selectedIndex >= 0 && + } + if (elem && elem.options && elem.selectedIndex >= 0) { matsParamUtils.setValueTextForParamName( name, elem.options[elem.selectedIndex].text ); + } } - for (var i = 0; i < brothers.length; i++) { + for (let i = 0; i < brothers.length; i += 1) { const belem = brothers[i]; const belemSelectedOptions = $(belem.selectedOptions) .map(function () { @@ -698,7 +703,7 @@ const refresh = function (event, paramName) { .get(); if (belemSelectedOptions === undefined || belemSelectedOptions.length === 0) { belem.options = []; - for (let i1 = 0; i1 < myOptions.length; i1++) { + for (let i1 = 0; i1 < myOptions.length; i1 += 1) { belem.options[belem.options.length] = new Option( myOptions[i1], myOptions[i1], @@ -715,10 +720,11 @@ const refresh = function (event, paramName) { } // These need to be done in the right order! // always check to see if an "other" needs to be hidden or disabled before refreshing - matsSelectUtils.checkHideOther(param, false); + checkHideOther(param, false); refreshPeer(event, param); }; // refresh function +// eslint-disable-next-line no-undef export default matsSelectUtils = { refresh, refreshPeer, diff --git a/meteor_packages/mats-common/imports/startup/server/data_util.js b/meteor_packages/mats-common/imports/startup/server/data_util.js index 677d14bb7..84b685db1 100644 --- a/meteor_packages/mats-common/imports/startup/server/data_util.js +++ b/meteor_packages/mats-common/imports/startup/server/data_util.js @@ -6,6 +6,7 @@ import { matsTypes, matsCollections, matsMethods } from "meteor/randyp:mats-comm import { Meteor } from "meteor/meteor"; import { HTTP } from "meteor/jkuester:http"; +/* global Npm */ /* eslint-disable global-require */ /* eslint-disable no-console */ @@ -383,7 +384,6 @@ const doSettings = function ( metexpress: "production", }; const settingsId = settings._id; - // eslint-disable-next-line no-undef const os = Npm.require("os"); const hostname = os.hostname().split(".")[0]; settings.appVersion = version; diff --git a/meteor_packages/mats-common/imports/startup/server/publications.js b/meteor_packages/mats-common/imports/startup/server/publications.js index bf4fe60d5..c3077c1f5 100644 --- a/meteor_packages/mats-common/imports/startup/server/publications.js +++ b/meteor_packages/mats-common/imports/startup/server/publications.js @@ -8,7 +8,7 @@ import { curveParamsByApp } from "../both/mats-curve-params"; /* eslint-disable no-console */ -const _publishField = function (field) { +const publishField = function (field) { Meteor.publish(field, function () { const data = matsCollections[field].find({}); if (data) { @@ -31,7 +31,7 @@ if (Meteor.isServer) { let currParam; for (let i = 0; i < params.length; i += 1) { currParam = params[i]; - _publishField(currParam); + publishField(currParam); } Meteor.publish("CurveParamsInfo", function () { const data = matsCollections.CurveParamsInfo.find({}); diff --git a/meteor_packages/mats-common/imports/startup/ui/layouts/appBody.html b/meteor_packages/mats-common/imports/startup/ui/layouts/appBody.html index 2ba533c6b..c2439122e 100644 --- a/meteor_packages/mats-common/imports/startup/ui/layouts/appBody.html +++ b/meteor_packages/mats-common/imports/startup/ui/layouts/appBody.html @@ -1,3 +1,3 @@ -