Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

SuiteRunner: use suite instance variables #437

Merged
merged 3 commits into from
Oct 29, 2024
Merged

SuiteRunner: use suite instance variables #437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8dd554e
use suite directly
camillobruni Oct 22, 2024
b129905
fix tests
camillobruni Oct 22, 2024
ccdcf84
Merge branch 'main' into 2024-10-22_improve_runner
camillobruni Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 34 additions & 29 deletions resources/benchmark-runner.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,65 +551,67 @@ export class SuiteRunner {
}

async run() {
// FIXME: use this._suite in all SuiteRunner methods directly.
await this._prepareSuite(this._suite);
await this._runSuite(this._suite);
await this._prepareSuite();
await this._runSuite();
}

async _prepareSuite(suite) {
const suiteName = suite.name;
async _prepareSuite() {
const suiteName = this._suite.name;
const suitePrepareStartLabel = `suite-${suiteName}-prepare-start`;
const suitePrepareEndLabel = `suite-${suiteName}-prepare-end`;

performance.mark(suitePrepareStartLabel);
await this._loadFrame(suite);
await suite.prepare(this._page);
await this._loadFrame();
await this._suite.prepare(this._page);
performance.mark(suitePrepareEndLabel);

performance.measure(`suite-${suiteName}-prepare`, suitePrepareStartLabel, suitePrepareEndLabel);
}

async _runSuite(suite) {
const suiteName = suite.name;
async _runSuite() {
const suiteName = this._suite.name;
const suiteStartLabel = `suite-${suiteName}-start`;
const suiteEndLabel = `suite-${suiteName}-end`;

performance.mark(suiteStartLabel);
for (const test of suite.tests)
await this._runTestAndRecordResults(suite, test);
for (const test of this._suite.tests)
await this._runTestAndRecordResults(test);
performance.mark(suiteEndLabel);

performance.measure(`suite-${suiteName}`, suiteStartLabel, suiteEndLabel);
this._validateSuiteTotal(suiteName);
}

_validateSuiteTotal(suiteName) {
_validateSuiteTotal() {
// When the test is fast and the precision is low (for example with Firefox'
// privacy.resistFingerprinting preference), it's possible that the measured
// total duration for an entire is 0.
const suiteName = this._suite.name;
const suiteTotal = this._measuredValues.tests[suiteName].total;
if (suiteTotal === 0)
throw new Error(`Got invalid 0-time total for suite ${suiteName}: ${suiteTotal}`);
}

async _loadFrame(suite) {
async _loadFrame() {
return new Promise((resolve, reject) => {
const frame = this._page._frame;
frame.onload = () => resolve();
frame.onerror = () => reject();
frame.src = suite.url;
frame.src = this._suite.url;
});
}

async _runTestAndRecordResults(suite, test) {
async _runTestAndRecordResults(test) {
if (this._client?.willRunTest)
await this._client.willRunTest(suite, test);
await this._client.willRunTest(this._suite, test);

// Prepare all mark labels outside the measuring loop.
const startLabel = `${suite.name}.${test.name}-start`;
const syncEndLabel = `${suite.name}.${test.name}-sync-end`;
const asyncStartLabel = `${suite.name}.${test.name}-async-start`;
const asyncEndLabel = `${suite.name}.${test.name}-async-end`;
const suiteName = this._suite.name;
const testName = test.name;
const startLabel = `${suiteName}.${testName}-start`;
const syncEndLabel = `${suiteName}.${testName}-sync-end`;
const asyncStartLabel = `${suiteName}.${testName}-async-start`;
const asyncEndLabel = `${suiteName}.${testName}-async-end`;

let syncTime;
let asyncStartTime;
Expand Down Expand Up @@ -644,28 +646,31 @@ export class SuiteRunner {
performance.mark(asyncEndLabel);
if (params.warmupBeforeSync)
performance.measure("warmup", "warmup-start", "warmup-end");
performance.measure(`${suite.name}.${test.name}-sync`, startLabel, syncEndLabel);
performance.measure(`${suite.name}.${test.name}-async`, asyncStartLabel, asyncEndLabel);
const suiteName = this._suite.name;
const testName = test.name;
performance.measure(`${suiteName}.${testName}-sync`, startLabel, syncEndLabel);
performance.measure(`${suiteName}.${testName}-async`, asyncStartLabel, asyncEndLabel);
};
const report = () => this._recordTestResults(suite, test, syncTime, asyncTime);

const report = () => this._recordTestResults(test, syncTime, asyncTime);
const invokerClass = TEST_INVOKER_LOOKUP[params.measurementMethod];
const invoker = new invokerClass(runSync, measureAsync, report);

return invoker.start();
}

async _recordTestResults(suite, test, syncTime, asyncTime) {
async _recordTestResults(test, syncTime, asyncTime) {
// Skip reporting updates for the warmup suite.
if (suite === WarmupSuite)
if (this._suite === WarmupSuite)
return;

const suiteResults = this._measuredValues.tests[suite.name] || { tests: {}, total: 0 };
const suiteName = this._suite.name;
const suiteResults = this._measuredValues.tests[suiteName] || { tests: {}, total: 0 };
const total = syncTime + asyncTime;
this._measuredValues.tests[suite.name] = suiteResults;
this._measuredValues.tests[suiteName] = suiteResults;
suiteResults.tests[test.name] = { tests: { Sync: syncTime, Async: asyncTime }, total: total };
suiteResults.total += total;

if (this._client?.didRunTest)
await this._client.didRunTest(suite, test);
await this._client.didRunTest(this._suite, test);
}
}
9 changes: 5 additions & 4 deletions tests/benchmark-runner-tests.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,10 +188,11 @@ describe("BenchmarkRunner", () => {
const suite = SUITES_FIXTURE[0];

before(async () => {
runner._suite = suite;
await runner._appendFrame();
performanceMarkSpy = spy(window.performance, "mark");
const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, runner._suite);
await suiteRunner._runTestAndRecordResults(suite, suite.tests[0]);
const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, suite);
await suiteRunner._runTestAndRecordResults(suite.tests[0]);
});

it("should run client pre and post hooks if present", () => {
Expand Down Expand Up @@ -225,8 +226,8 @@ describe("BenchmarkRunner", () => {
stubPerformanceNowCalls(syncStart, syncEnd, asyncStart, asyncEnd);

// instantiate recorded test results
const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, runner._suite);
await suiteRunner._runTestAndRecordResults(suite, suite.tests[0]);
const suiteRunner = new SuiteRunner(runner._measuredValues, runner._frame, runner._page, runner._client, suite);
await suiteRunner._runTestAndRecordResults(suite.tests[0]);

await runner._finalize();
});
Expand Down
Loading