Skip to content

Commit

Permalink
Merge branch 'main' into feature/async-runner
Browse files Browse the repository at this point in the history
  • Loading branch information
@flashdesignory
flashdesignory committed Nov 15, 2024
2 parents 5281af4 + 67bc21f commit d4fa570
Show file tree
Hide file tree
Showing 4 changed files with 303 additions and 221 deletions.
215 changes: 3 additions & 212 deletions resources/benchmark-runner.mjs
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Metric } from "./metric.mjs";
import { params } from "./params.mjs";
import { TEST_INVOKER_LOOKUP, ASYNC_TEST_INVOKER_LOOKUP } from "./test-invoker.mjs";
import { SUITE_RUNNER_LOOKUP } from "./suite-runner.mjs";

const performance = globalThis.performance;

Expand Down Expand Up @@ -223,7 +223,7 @@ function geomeanToScore(geomean) {
// The WarmupSuite is used to make sure all runner helper functions and
// classes are compiled, to avoid unnecessary pauses due to delayed
// compilation of runner methods in the middle of the measuring cycle.
const WarmupSuite = {
export const WarmupSuite = {
name: "Warmup",
url: "warmup/index.html",
async prepare(page) {
Expand Down Expand Up @@ -410,7 +410,7 @@ export class BenchmarkRunner {
// FIXME: Encapsulate more state in the SuiteRunner.
// FIXME: Return and use measured values from SuiteRunner.
const suiteRunnerClass = SUITE_RUNNER_LOOKUP[suite.type ?? "default"];
const suiteRunner = new suiteRunnerClass(this._measuredValues, this._frame, this._page, this._client, suite);
const suiteRunner = new suiteRunnerClass(this._frame, this._page, params, suite, this._client, this._measuredValues);
await suiteRunner.run();
}

Expand Down Expand Up @@ -484,212 +484,3 @@ export class BenchmarkRunner {
metric.computeAggregatedMetrics();
}
}

// FIXME: Create AsyncSuiteRunner subclass.
// FIXME: Create RemoteSuiteRunner subclass.
export class SuiteRunner {
constructor(measuredValues, frame, page, client, suite) {
// FIXME: Create SuiteRunner-local measuredValues.
this._suiteResults = measuredValues.tests[suite.name];
if (!this._suiteResults) {
this._suiteResults = { tests: {}, total: 0 };
measuredValues.tests[suite.name] = this._suiteResults;
}
this._measuredValues = measuredValues;
this._frame = frame;
this._page = page;
this._client = client;
this._suite = suite;
}

async run() {
await this._prepareSuite();
await this._runSuite();
}

async _prepareSuite() {
const suiteName = this._suite.name;
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;

performance.mark(suitePrepareStartLabel);
await this._loadFrame();
await this._suite.prepare(this._page);
performance.mark(suitePrepareEndLabel);

performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
}

async _runSuite() {
const suiteName = this._suite.name;
const suiteStartLabel = `suite-${suiteName}-start`;
const suiteEndLabel = `suite-${suiteName}-end`;

performance.mark(suiteStartLabel);
for (const test of this._suite.tests)
await this._runTestAndRecordResults(test);
performance.mark(suiteEndLabel);

performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
this._validateSuiteTotal();
}

_validateSuiteTotal() {
// When the test is fast and the precision is low (for example with Firefox'
// privacy.resistFingerprinting preference), it's possible that the measured
// total duration for an entire is 0.
const suiteTotal = this._suiteResults.total;
if (suiteTotal === 0)
throw new Error(`Got invalid 0-time total for suite ${this._suite.name}: ${suiteTotal}`);
}

async _loadFrame() {
return new Promise((resolve, reject) => {
const frame = this._page._frame;
frame.onload = () => resolve();
frame.onerror = () => reject();
frame.src = this._suite.url;
});
}

async _runTestAndRecordResults(test) {
if (this._client?.willRunTest)
await this._client.willRunTest(this._suite, test);

// Prepare all mark labels outside the measuring loop.
const suiteName = this._suite.name;
const testName = test.name;
const startLabel = `${suiteName}.${testName}-start`;
const syncEndLabel = `${suiteName}.${testName}-sync-end`;
const asyncStartLabel = `${suiteName}.${testName}-async-start`;
const asyncEndLabel = `${suiteName}.${testName}-async-end`;

let syncTime;
let asyncStartTime;
let asyncTime;
const runSync = () => {
if (params.warmupBeforeSync) {
performance.mark("warmup-start");
const startTime = performance.now();
// Infinite loop for the specified ms.
while (performance.now() - startTime < params.warmupBeforeSync)
continue;
performance.mark("warmup-end");
}
performance.mark(startLabel);
const syncStartTime = performance.now();
test.run(this._page);
const syncEndTime = performance.now();
performance.mark(syncEndLabel);

syncTime = syncEndTime - syncStartTime;

performance.mark(asyncStartLabel);
asyncStartTime = performance.now();
};
const measureAsync = () => {
// Some browsers don't immediately update the layout for paint.
// Force the layout here to ensure we're measuring the layout time.
const height = this._frame.contentDocument.body.getBoundingClientRect().height;
const asyncEndTime = performance.now();
asyncTime = asyncEndTime - asyncStartTime;
this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
performance.mark(asyncEndLabel);
if (params.warmupBeforeSync)
performance.measure("warmup", "warmup-start", "warmup-end");
const suiteName = this._suite.name;
const testName = test.name;
performance.measure(`${suiteName}.${testName}-sync`, startLabel, syncEndLabel);
performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
};

const report = () => this._recordTestResults(test, syncTime, asyncTime);
const invokerClass = TEST_INVOKER_LOOKUP[params.measurementMethod];
const invoker = new invokerClass(runSync, measureAsync, report, params);

return invoker.start();
}

async _recordTestResults(test, syncTime, asyncTime) {
// Skip reporting updates for the warmup suite.
if (this._suite === WarmupSuite)
return;

const total = syncTime + asyncTime;
this._suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
this._suiteResults.total += total;

if (this._client?.didRunTest)
await this._client.didRunTest(this._suite, test);
}
}

export class AsyncSuiteRunner extends SuiteRunner {
async _runTestAndRecordResults(test) {
if (this._client?.willRunTest)
await this._client.willRunTest(this._suite, test);

// Prepare all mark labels outside the measuring loop.
const suiteName = this._suite.name;
const testName = test.name;
const startLabel = `${suiteName}.${testName}-start`;
const syncEndLabel = `${suiteName}.${testName}-sync-end`;
const asyncStartLabel = `${suiteName}.${testName}-async-start`;
const asyncEndLabel = `${suiteName}.${testName}-async-end`;

let syncTime;
let asyncStartTime;
let asyncTime;
const runSync = async () => {
if (params.warmupBeforeSync) {
performance.mark("warmup-start");
const startTime = performance.now();
// Infinite loop for the specified ms.
while (performance.now() - startTime < params.warmupBeforeSync)
continue;
performance.mark("warmup-end");
}
performance.mark(startLabel);
const syncStartTime = performance.now();
await test.run(this._page);
const syncEndTime = performance.now();
performance.mark(syncEndLabel);

syncTime = syncEndTime - syncStartTime;

performance.mark(asyncStartLabel);
asyncStartTime = performance.now();
};
const measureAsync = () => {
// Some browsers don't immediately update the layout for paint.
// Force the layout here to ensure we're measuring the layout time.
const height = this._frame.contentDocument.body.getBoundingClientRect().height;
const asyncEndTime = performance.now();
asyncTime = asyncEndTime - asyncStartTime;
this._frame.contentWindow._unusedHeightValue = height; // Prevent dead code elimination.
performance.mark(asyncEndLabel);
if (params.warmupBeforeSync)
performance.measure("warmup", "warmup-start", "warmup-end");
const suiteName = this._suite.name;
const testName = test.name;
performance.measure(`${suiteName}.${testName}-sync`, startLabel, syncEndLabel);
performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
};

const report = () => this._recordTestResults(test, syncTime, asyncTime);
const invokerClass = ASYNC_TEST_INVOKER_LOOKUP[params.measurementMethod];
const invoker = new invokerClass(runSync, measureAsync, report, params);

return invoker.start();
}
}

// FIXME: implement remote steps
class RemoteSuiteRunner extends SuiteRunner {}

const SUITE_RUNNER_LOOKUP = {
__proto__: null,
default: SuiteRunner,
async: AsyncSuiteRunner,
remote: RemoteSuiteRunner,
};
105 changes: 105 additions & 0 deletions resources/suite-runner.mjs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import { TEST_RUNNER_LOOKUP } from "./test-runner.mjs";
import { WarmupSuite } from "./benchmark-runner.mjs";

// FIXME: Create RemoteSuiteRunner subclass.
export class SuiteRunner {
#frame;
#page;
#params;
#suite;
#client;
#suiteResults;

constructor(frame, page, params, suite, client, measuredValues) {
// FIXME: Create SuiteRunner-local measuredValues.
this.#suiteResults = measuredValues.tests[suite.name];
if (!this.#suiteResults) {
this.#suiteResults = { tests: {}, total: 0 };
measuredValues.tests[suite.name] = this.#suiteResults;
}
this.#frame = frame;
this.#page = page;
this.#client = client;
this.#suite = suite;
this.#params = params;
}

async run() {
await this._prepareSuite();
await this._runSuite();
}

async _prepareSuite() {
const suiteName = this.#suite.name;
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;

performance.mark(suitePrepareStartLabel);
await this._loadFrame();
await this.#suite.prepare(this.#page);
performance.mark(suitePrepareEndLabel);

performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
}

async _runSuite() {
const suiteName = this.#suite.name;
const suiteStartLabel = `suite-${suiteName}-start`;
const suiteEndLabel = `suite-${suiteName}-end`;

performance.mark(suiteStartLabel);
for (const test of this.#suite.tests) {
if (this.#client?.willRunTest)
await this.#client.willRunTest(this.#suite, test);

const testRunnerClass = TEST_RUNNER_LOOKUP[suite.type ?? "default"];
const testRunner = new testRunnerClass(this.#frame, this.#page, this.#params, this.#suite, test, this._recordTestResults);
await testRunner.runTest();
}
performance.mark(suiteEndLabel);

performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
this._validateSuiteTotal();
}

_validateSuiteTotal() {
// When the test is fast and the precision is low (for example with Firefox'
// privacy.resistFingerprinting preference), it's possible that the measured
// total duration for an entire is 0.
const suiteTotal = this.#suiteResults.total;
if (suiteTotal === 0)
throw new Error(`Got invalid 0-time total for suite ${this.#suite.name}: ${suiteTotal}`);
}

async _loadFrame() {
return new Promise((resolve, reject) => {
const frame = this.#frame;
frame.onload = () => resolve();
frame.onerror = () => reject();
frame.src = this.#suite.url;
});
}

_recordTestResults = async (test, syncTime, asyncTime) => {
// Skip reporting updates for the warmup suite.
if (this.#suite === WarmupSuite)
return;

const total = syncTime + asyncTime;
this.#suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
this.#suiteResults.total += total;

if (this.#client?.didRunTest)
await this.#client.didRunTest(this.#suite, test);
};
}

// FIXME: implement remote steps
class RemoteSuiteRunner extends SuiteRunner {}

export const SUITE_RUNNER_LOOKUP = {
__proto__: null,
default: SuiteRunner,
async: SuiteRunner,
remote: RemoteSuiteRunner,
};
Loading

0 comments on commit d4fa570

Please sign in to comment.