Runner Cleanup: SuiteRunner & TestRunner classes

resources/benchmark-runner.mjs

@@ -1,6 +1,6 @@
 import { Metric } from "./metric.mjs";
 import { params } from "./params.mjs";
-import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
+import { SUITE_RUNNER_LOOKUP } from "./suite-runner.mjs";
 
 const performance = globalThis.performance;
 
@@ -223,7 +223,7 @@ function geomeanToScore(geomean) {
 // The WarmupSuite is used to make sure all runner helper functions and
 // classes are compiled, to avoid unnecessary pauses due to delayed
 // compilation of runner methods in the middle of the measuring cycle.
-const WarmupSuite = {
+export const WarmupSuite = {
     name: "Warmup",
     url: "warmup/index.html",
     async prepare(page) {
@@ -410,7 +410,7 @@ export class BenchmarkRunner {
         // FIXME: Encapsulate more state in the SuiteRunner.
         // FIXME: Return and use measured values from SuiteRunner.
         const suiteRunnerClass = SUITE_RUNNER_LOOKUP[suite.type ?? "default"];
-        const suiteRunner = new suiteRunnerClass(this._measuredValues, this._frame, this._page, this._client, suite);
+        const suiteRunner = new suiteRunnerClass(this._frame, this._page, params, suite, this._client, this._measuredValues);
         await suiteRunner.run();
     }
 
@@ -484,151 +484,3 @@ export class BenchmarkRunner {
             metric.computeAggregatedMetrics();
     }
 }
-
-// FIXME: Create AsyncSuiteRunner subclass.
-// FIXME: Create RemoteSuiteRunner subclass.
-export class SuiteRunner {
-    constructor(measuredValues, frame, page, client, suite) {
-        // FIXME: Create SuiteRunner-local measuredValues.
-        this._suiteResults = measuredValues.tests[suite.name];
-        if (!this._suiteResults) {
-            this._suiteResults = { tests: {}, total: 0 };
-            measuredValues.tests[suite.name] = this._suiteResults;
-        }
-        this._measuredValues = measuredValues;
-        this._frame = frame;
-        this._page = page;
-        this._client = client;
-        this._suite = suite;
-    }
-
-    async run() {
-        await this._prepareSuite();
-        await this._runSuite();
-    }
-
-    async _prepareSuite() {
-        const suiteName = this._suite.name;
-        const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
-        const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
-
-        performance.mark(suitePrepareStartLabel);
-        await this._loadFrame();
-        await this._suite.prepare(this._page);
-        performance.mark(suitePrepareEndLabel);
-
-        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
-    }
-
-    async _runSuite() {
-        const suiteName = this._suite.name;
-        const suiteStartLabel = `suite-${suiteName}-start`;
-        const suiteEndLabel = `suite-${suiteName}-end`;
-
-        performance.mark(suiteStartLabel);
-        for (const test of this._suite.tests)
-            await this._runTestAndRecordResults(test);
-        performance.mark(suiteEndLabel);
-
-        performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
-        this._validateSuiteTotal();
-    }
-
-    _validateSuiteTotal() {
-        // When the test is fast and the precision is low (for example with Firefox'
-        // privacy.resistFingerprinting preference), it's possible that the measured
-        // total duration for an entire is 0.
-        const suiteTotal = this._suiteResults.total;
-        if (suiteTotal === 0)
-            throw new Error(`Got invalid 0-time total for suite ${this._suite.name}: ${suiteTotal}`);
-    }
-
-    async _loadFrame() {
-        return new Promise((resolve, reject) => {
-            const frame = this._page._frame;
-            frame.onload = () => resolve();
-            frame.onerror = () => reject();
-            frame.src = this._suite.url;
-        });
-    }
-
-    async _runTestAndRecordResults(test) {
-        if (this._client?.willRunTest)
-            await this._client.willRunTest(this._suite, test);
-
-        // Prepare all mark labels outside the measuring loop.
-        const suiteName = this._suite.name;
-        const testName = test.name;
-        const startLabel = `${suiteName}.${testName}-start`;
-        const syncEndLabel = `${suiteName}.${testName}-sync-end`;
-        const asyncStartLabel = `${suiteName}.${testName}-async-start`;
-        const asyncEndLabel = `${suiteName}.${testName}-async-end`;
-
-        let syncTime;
-        let asyncStartTime;
-        let asyncTime;
-        const runSync = () => {
-            if (params.warmupBeforeSync) {
-                performance.mark("warmup-start");
-                const startTime = performance.now();
-                // Infinite loop for the specified ms.
-                while (performance.now() - startTime < params.warmupBeforeSync)
-                    continue;
-                performance.mark("warmup-end");
-            }
-            performance.mark(startLabel);
-            const syncStartTime = performance.now();
-            test.run(this._page);
-            const syncEndTime = performance.now();
-            performance.mark(syncEndLabel);
-
-            syncTime = syncEndTime - syncStartTime;
-
-            performance.mark(asyncStartLabel);
-            asyncStartTime = performance.now();
-        };
-        const measureAsync = () => {
-            // Some browsers don't immediately update the layout for paint.
-            // Force the layout here to ensure we're measuring the layout time.
-            const height = this._frame.contentDocument.body.getBoundingClientRect().height;
-            const asyncEndTime = performance.now();
-            asyncTime = asyncEndTime - asyncStartTime;
-            this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
-            performance.mark(asyncEndLabel);
-            if (params.warmupBeforeSync)
-                performance.measure("warmup", "warmup-start", "warmup-end");
-            const suiteName = this._suite.name;
-            const testName = test.name;
-            performance.measure(`${suiteName}.${testName}-sync`, startLabel, syncEndLabel);
-            performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
-        };
-
-        const report = () => this._recordTestResults(test, syncTime, asyncTime);
-        const invokerClass = TEST_INVOKER_LOOKUP[params.measurementMethod];
-        const invoker = new invokerClass(runSync, measureAsync, report, params);
-
-        return invoker.start();
-    }
-
-    async _recordTestResults(test, syncTime, asyncTime) {
-        // Skip reporting updates for the warmup suite.
-        if (this._suite === WarmupSuite)
-            return;
-
-        const total = syncTime + asyncTime;
-        this._suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
-        this._suiteResults.total += total;
-
-        if (this._client?.didRunTest)
-            await this._client.didRunTest(this._suite, test);
-    }
-}
-
-// FIXME: implement remote steps
-class RemoteSuiteRunner extends SuiteRunner {}
-
-const SUITE_RUNNER_LOOKUP = {
-    __proto__: null,
-    default: SuiteRunner,
-    remote: RemoteSuiteRunner,
-};

resources/suite-runner.mjs

@@ -0,0 +1,104 @@
+import { TestRunner } from "./test-runner.mjs";
+import { WarmupSuite } from "./benchmark-runner.mjs";
+
+// FIXME: Create AsyncSuiteRunner subclass.
+// FIXME: Create RemoteSuiteRunner subclass.
+export class SuiteRunner {
+    #frame;
+    #page;
+    #params;
+    #suite;
+    #client;
+    #suiteResults;
+
+    constructor(frame, page, params, suite, client, measuredValues) {
+        // FIXME: Create SuiteRunner-local measuredValues.
+        this.#suiteResults = measuredValues.tests[suite.name];
+        if (!this.#suiteResults) {
+            this.#suiteResults = { tests: {}, total: 0 };
+            measuredValues.tests[suite.name] = this.#suiteResults;
+        }
+        this.#frame = frame;
+        this.#page = page;
+        this.#client = client;
+        this.#suite = suite;
+        this.#params = params;
+    }
+
+    async run() {
+        await this._prepareSuite();
+        await this._runSuite();
+    }
+
+    async _prepareSuite() {
+        const suiteName = this.#suite.name;
+        const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
+        const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;
+
+        performance.mark(suitePrepareStartLabel);
+        await this._loadFrame();
+        await this.#suite.prepare(this.#page);
+        performance.mark(suitePrepareEndLabel);
+
+        performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
+    }
+
+    async _runSuite() {
+        const suiteName = this.#suite.name;
+        const suiteStartLabel = `suite-${suiteName}-start`;
+        const suiteEndLabel = `suite-${suiteName}-end`;
+
+        performance.mark(suiteStartLabel);
+        for (const test of this.#suite.tests) {
+            if (this.#client?.willRunTest)
+                await this.#client.willRunTest(this.#suite, test);
+
+            const testRunner = new TestRunner(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults);
+            await testRunner.runTest();
+        }
+        performance.mark(suiteEndLabel);
+
+        performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
+        this._validateSuiteTotal();
+    }
+
+    _validateSuiteTotal() {
+        // When the test is fast and the precision is low (for example with Firefox'
+        // privacy.resistFingerprinting preference), it's possible that the measured
+        // total duration for an entire is 0.
+        const suiteTotal = this.#suiteResults.total;
+        if (suiteTotal === 0)
+            throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`);
+    }
+
+    async _loadFrame() {
+        return new Promise((resolve, reject) => {
+            const frame = this.#frame;
+            frame.onload = () => resolve();
+            frame.onerror = () => reject();
+            frame.src = this.#suite.url;
+        });
+    }
+
+    _recordTestResults = async (test, syncTime, asyncTime) => {
+        // Skip reporting updates for the warmup suite.
+        if (this.#suite === WarmupSuite)
+            return;
+
+        const total = syncTime + asyncTime;
+        this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
+        this.#suiteResults.total += total;
+
+        if (this.#client?.didRunTest)
+            await this.#client.didRunTest(this.#suite, test);
+    };
+}
+
+// FIXME: implement remote steps
+class RemoteSuiteRunner extends SuiteRunner {}
+
+export const SUITE_RUNNER_LOOKUP = {
+    __proto__: null,
+    default: SuiteRunner,
+    remote: RemoteSuiteRunner,
+};

resources/test-runner.mjs

@@ -0,0 +1,79 @@
+import { TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
+
+export class TestRunner {
+    #frame;
+    #page;
+    #params;
+    #suite;
+    #test;
+    #callback;
+
+    constructor(frame, page, params, suite, test, callback) {
+        this.#suite = suite;
+        this.#test = test;
+        this.#params = params;
+        this.#callback = callback;
+
+        this.#page = page;
+        this.#frame = frame;
+    }
+
+    async runTest() {
+        // Prepare all mark labels outside the measuring loop.
+        const suiteName = this.#suite.name;
+        const testName = this.#test.name;
+        const syncStartLabel = `${suiteName}.${testName}-start`;
+        const syncEndLabel = `${suiteName}.${testName}-sync-end`;
+        const asyncStartLabel = `${suiteName}.${testName}-async-start`;
+        const asyncEndLabel = `${suiteName}.${testName}-async-end`;
+
+        let syncTime;
+        let asyncStartTime;
+        let asyncTime;
+
+        const runSync = () => {
+            if (this.#params.warmupBeforeSync) {
+                performance.mark("warmup-start");
+                const startTime = performance.now();
+                // Infinite loop for the specified ms.
+                while (performance.now() - startTime < this.#params.warmupBeforeSync)
+                    continue;
+                performance.mark("warmup-end");
+            }
+            performance.mark(syncStartLabel);
+            const syncStartTime = performance.now();
+            this.#test.run(this.#page);
+            const syncEndTime = performance.now();
+            performance.mark(syncEndLabel);
+
+            syncTime = syncEndTime - syncStartTime;
+
+            performance.mark(asyncStartLabel);
+            asyncStartTime = performance.now();
+        };
+        const measureAsync = () => {
+            const bodyReference = this.#frame ? this.#frame.contentDocument.body : document.body;
+            const windowReference = this.#frame ? this.#frame.contentWindow : window;
+            // Some browsers don't immediately update the layout for paint.
+            // Force the layout here to ensure we're measuring the layout time.
+            const height = bodyReference.getBoundingClientRect().height;
+            windowReference._unusedHeightValue = height; // Prevent dead code elimination.
+
+            const asyncEndTime = performance.now();
+            performance.mark(asyncEndLabel);
+
+            asyncTime = asyncEndTime - asyncStartTime;
+
+            if (this.#params.warmupBeforeSync)
+                performance.measure("warmup", "warmup-start", "warmup-end");
+            performance.measure(`${suiteName}.${testName}-sync`, syncStartLabel, syncEndLabel);
+            performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
+        };
+
+        const report = () => this.#callback(this.#test, syncTime, asyncTime);
+        const invokerClass = TEST_INVOKER_LOOKUP[this.#params.measurementMethod];
+        const invoker = new invokerClass(runSync, measureAsync, report, this.#params);
+
+        return invoker.start();
+    }
+}