From f8948c96fdcef0b0f96d27acaa59faacbabaf0f9 Mon Sep 17 00:00:00 2001 From: Timo Tijhof Date: Sat, 1 May 2021 18:10:36 +0100 Subject: [PATCH] Core: Make `tap` and `console` reporters generally available This change makes the reporters, from the js-reporters package, generally available through the primary distribution, instead of being limited to the QUnit CLI. The built-in reporters are exposed via `QUnit.reporters`. This is a major step toward truly supporting the TAP ecosystem, and popular runners such as Airtap for browser testing, and the node-tap CLI (why not). There is still more work to be done, though, such as making the library more friendly to require'ing, and further decoupling of our HTML Runner/Reporter. As such, this remains experimental as of yet. It can be tried out by calling `QUnit.reporters.tap.init(QUnit)` between importing QUnit and importing your tests, e.g. from an inline script or setup script. This overall direction is outlined in https://github.com/js-reporters/js-reporters/issues/133, and as such apart from bridging older versions, the js-reporters package will not be as actively developed going forward. I'm copying the reporters we developed there into QUnit repository for further developement here, without the indirection of that package. Co-authored-by: Florentin Simion Co-authored-by: Franziska Carstens Co-authored-by: Martin Olsson Co-authored-by: Robert Jackson Co-authored-by: Timo Tijhof Co-authored-by: Trent Willis Co-authored-by: Zachary Mulgrew Co-authored-by: jeberger --- .editorconfig | 3 + package-lock.json | 8 +- package.json | 1 + src/core.js | 2 + src/reporters.js | 7 + src/reporters/ConsoleReporter.js | 36 +++ src/reporters/TapReporter.js | 254 +++++++++++++++++++++ test/cli/ConsoleReporter.js | 37 +++ test/cli/TapReporter.js | 378 +++++++++++++++++++++++++++++++ 9 files changed, 725 insertions(+), 1 deletion(-) create mode 100644 src/reporters.js create mode 100644 src/reporters/ConsoleReporter.js create mode 100644 src/reporters/TapReporter.js create mode 100644 test/cli/ConsoleReporter.js create mode 100644 test/cli/TapReporter.js diff --git a/.editorconfig b/.editorconfig index 40c60644b..fa17e8c88 100644 --- a/.editorconfig +++ b/.editorconfig @@ -10,6 +10,9 @@ charset = utf-8 trim_trailing_whitespace = true insert_final_newline = true +[test/cli/TapReporter.js] +trim_trailing_whitespace = false + [*.{yml,md}] indent_style = space indent_size = 2 diff --git a/package-lock.json b/package-lock.json index 83853ac7b..b7ba8cfc2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "qunit", - "version": "2.14.1-pre", + "version": "2.15.0-pre", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -4566,6 +4566,12 @@ "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", "dev": true }, + "kleur": { + "version": "4.1.4", + "resolved": "https://registry.npmjs.org/kleur/-/kleur-4.1.4.tgz", + "integrity": "sha512-8QADVssbrFjivHWQU7KkMgptGTl6WAcSdlbBPY4uNF+mWr6DGcKrvY2w4FQJoXch7+fKMjj0dRrL75vk3k23OA==", + "dev": true + }, "lcov-parse": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/lcov-parse/-/lcov-parse-1.0.0.tgz", diff --git a/package.json b/package.json index bf81b102b..68a8527d4 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "grunt-eslint": "^23.0.0", "grunt-git-authors": "^3.2.0", "grunt-search": "^0.1.8", + "kleur": "4.1.4", "npm-reporter": "file:./test/cli/fixtures/npm-reporter", "nyc": "^15.1.0", "proxyquire": "^1.8.0", diff --git a/src/core.js b/src/core.js index dd5d98375..08789337f 100644 --- a/src/core.js +++ b/src/core.js @@ -7,6 +7,7 @@ import Assert from "./assert"; import Logger from "./logger"; import Test, { test, pushFailure } from "./test"; import exportQUnit from "./export"; +import reporters from "./reporters"; import config from "./core/config"; import { extend, objectType, is, now } from "./core/utilities"; @@ -42,6 +43,7 @@ extend( QUnit, { dump, equiv, + reporters, is, objectType, on, diff --git a/src/reporters.js b/src/reporters.js new file mode 100644 index 000000000..4e52753c8 --- /dev/null +++ b/src/reporters.js @@ -0,0 +1,7 @@ +import ConsoleReporter from "./reporters/ConsoleReporter.js"; +import TapReporter from "./reporters/TapReporter.js"; + +export default { + console: ConsoleReporter, + tap: TapReporter +}; diff --git a/src/reporters/ConsoleReporter.js b/src/reporters/ConsoleReporter.js new file mode 100644 index 000000000..fb042efa7 --- /dev/null +++ b/src/reporters/ConsoleReporter.js @@ -0,0 +1,36 @@ +import { console } from "../globals"; + +export default class ConsoleReporter { + constructor( runner, options = {} ) { + + // Cache references to console methods to ensure we can report failures + // from tests tests that mock the console object itself. + // https://github.com/qunitjs/qunit/issues/1340 + this.log = options.log || console.log.bind( console ); + + runner.on( "runStart", this.onRunStart.bind( this ) ); + runner.on( "testStart", this.onTestStart.bind( this ) ); + runner.on( "testEnd", this.onTestEnd.bind( this ) ); + runner.on( "runEnd", this.onRunEnd.bind( this ) ); + } + + static init( runner, options ) { + return new ConsoleReporter( runner, options ); + } + + onRunStart( runStart ) { + this.log( "runStart", runStart ); + } + + onTestStart( test ) { + this.log( "testStart", test ); + } + + onTestEnd( test ) { + this.log( "testEnd", test ); + } + + onRunEnd( runEnd ) { + this.log( "runEnd", runEnd ); + } +} diff --git a/src/reporters/TapReporter.js b/src/reporters/TapReporter.js new file mode 100644 index 000000000..7aea237be --- /dev/null +++ b/src/reporters/TapReporter.js @@ -0,0 +1,254 @@ +import kleur from "kleur"; +import { console } from "../globals"; +const hasOwn = Object.hasOwnProperty; + +/** + * Format a given value into YAML. + * + * YAML is a superset of JSON that supports all the same data + * types and syntax, and more. As such, it is always possible + * to fallback to JSON.stringfify, but we generally avoid + * that to make output easier to read for humans. + * + * Supported data types: + * + * - null + * - boolean + * - number + * - string + * - array + * - object + * + * Anything else (including NaN, Infinity, and undefined) + * must be described in strings, for display purposes. + * + * Note that quotes are optional in YAML strings if the + * strings are "simple", and as such we generally prefer + * that for improved readability. We output strings in + * one of three ways: + * + * - bare unquoted text, for simple one-line strings. + * - JSON (quoted text), for complex one-line strings. + * - YAML Block, for complex multi-line strings. + * + * Objects with cyclical references will be stringifed as + * "[Circular]" as they cannot otherwise be represented. + */ +function prettyYamlValue( value, indent = 4 ) { + if ( value === undefined ) { + + // Not supported in JSON/YAML, turn into string + // and let the below output it as bare string. + value = String( value ); + } + + // Support IE 9-11: Use isFinite instead of ES6 Number.isFinite + if ( typeof value === "number" && !isFinite( value ) ) { + + // Turn NaN and Infinity into simple strings. + // Paranoia: Don't return directly just in case there's + // a way to add special characters here. + value = String( value ); + } + + if ( typeof value === "number" ) { + + // Simple numbers + return JSON.stringify( value ); + } + + if ( typeof value === "string" ) { + + // If any of these match, then we can't output it + // as bare unquoted text, because that would either + // cause data loss or invalid YAML syntax. + // + // - Quotes, escapes, line breaks, or JSON-like stuff. + const rSpecialJson = /['"\\/[{}\]\r\n]/; + + // - Characters that are special at the start of a YAML value + const rSpecialYaml = /[-?:,[\]{}#&*!|=>'"%@`]/; + + // - Leading or trailing whitespace. + const rUntrimmed = /(^\s|\s$)/; + + // - Ambiguous as YAML number, e.g. '2', '-1.2', '.2', or '2_000' + const rNumerical = /^[\d._-]+$/; + + // - Ambiguous as YAML bool. + // Use case-insensitive match, although technically only + // fully-lower, fully-upper, or uppercase-first would be ambiguous. + // e.g. true/True/TRUE, but not tRUe. + const rBool = /^(true|false|y|n|yes|no|on|off)$/i; + + // Is this a complex string? + if ( + value === "" || + rSpecialJson.test( value ) || + rSpecialYaml.test( value[ 0 ] ) || + rUntrimmed.test( value ) || + rNumerical.test( value ) || + rBool.test( value ) + ) { + if ( !/\n/.test( value ) ) { + + // Complex one-line string, use JSON (quoted string) + return JSON.stringify( value ); + } + + // See also + // Support IE 9-11: Avoid ES6 String#repeat + const prefix = ( new Array( indent + 1 ) ).join( " " ); + + const trailingLinebreakMatch = value.match( /\n+$/ ); + const trailingLinebreaks = trailingLinebreakMatch ? + trailingLinebreakMatch[ 0 ].length : 0; + + if ( trailingLinebreaks === 1 ) { + + // Use the most straight-forward "Block" string in YAML + // without any "Chomping" indicators. + const lines = value + + // Ignore the last new line, since we'll get that one for free + // with the straight-forward Block syntax. + .replace( /\n$/, "" ) + .split( "\n" ) + .map( line => prefix + line ); + return "|\n" + lines.join( "\n" ); + } else { + + // This has either no trailing new lines, or more than 1. + // Use |+ so that YAML parsers will preserve it exactly. + const lines = value + .split( "\n" ) + .map( line => prefix + line ); + return "|+\n" + lines.join( "\n" ); + } + } else { + + // Simple string, use bare unquoted text + return value; + } + } + + // Handle null, boolean, array, and object + return JSON.stringify( decycledShallowClone( value ), null, 2 ); +} + +/** + * Creates a shallow clone of an object where cycles have + * been replaced with "[Circular]". + */ +function decycledShallowClone( object, ancestors = [] ) { + if ( ancestors.indexOf( object ) !== -1 ) { + return "[Circular]"; + } + + let clone; + + const type = Object.prototype.toString + .call( object ) + .replace( /^\[.+\s(.+?)]$/, "$1" ) + .toLowerCase(); + + switch ( type ) { + case "array": + ancestors.push( object ); + clone = object.map( function( element ) { + return decycledShallowClone( element, ancestors ); + } ); + ancestors.pop(); + break; + case "object": + ancestors.push( object ); + clone = {}; + Object.keys( object ).forEach( function( key ) { + clone[ key ] = decycledShallowClone( object[ key ], ancestors ); + } ); + ancestors.pop(); + break; + default: + clone = object; + } + + return clone; +} + +export default class TapReporter { + constructor( runner, options = {} ) { + + // Cache references to console methods to ensure we can report failures + // from tests tests that mock the console object itself. + // https://github.com/qunitjs/qunit/issues/1340 + this.log = options.log || console.log.bind( console ); + + this.testCount = 0; + + runner.on( "runStart", this.onRunStart.bind( this ) ); + runner.on( "testEnd", this.onTestEnd.bind( this ) ); + runner.on( "runEnd", this.onRunEnd.bind( this ) ); + } + + static init( runner, options ) { + return new TapReporter( runner, options ); + } + + onRunStart( _globalSuite ) { + this.log( "TAP version 13" ); + } + + onTestEnd( test ) { + this.testCount = this.testCount + 1; + + if ( test.status === "passed" ) { + this.log( `ok ${this.testCount} ${test.fullName.join( " > " )}` ); + } else if ( test.status === "skipped" ) { + this.log( + kleur.yellow( `ok ${this.testCount} # SKIP ${test.fullName.join( " > " )}` ) + ); + } else if ( test.status === "todo" ) { + this.log( + kleur.cyan( `not ok ${this.testCount} # TODO ${test.fullName.join( " > " )}` ) + ); + test.errors.forEach( ( error ) => this.logError( error, "todo" ) ); + } else { + this.log( + kleur.red( `not ok ${this.testCount} ${test.fullName.join( " > " )}` ) + ); + test.errors.forEach( ( error ) => this.logError( error ) ); + } + } + + onRunEnd( globalSuite ) { + this.log( `1..${globalSuite.testCounts.total}` ); + this.log( `# pass ${globalSuite.testCounts.passed}` ); + this.log( kleur.yellow( `# skip ${globalSuite.testCounts.skipped}` ) ); + this.log( kleur.cyan( `# todo ${globalSuite.testCounts.todo}` ) ); + this.log( kleur.red( `# fail ${globalSuite.testCounts.failed}` ) ); + } + + logError( error, severity ) { + let out = " ---"; + out += `\n message: ${prettyYamlValue( error.message || "failed" )}`; + out += `\n severity: ${prettyYamlValue( severity || "failed" )}`; + + if ( hasOwn.call( error, "actual" ) ) { + out += `\n actual : ${prettyYamlValue( error.actual )}`; + } + + if ( hasOwn.call( error, "expected" ) ) { + out += `\n expected: ${prettyYamlValue( error.expected )}`; + } + + if ( error.stack ) { + + // Since stacks aren't user generated, take a bit of liberty by + // adding a trailing new line to allow a straight-forward YAML Blocks. + out += `\n stack: ${prettyYamlValue( error.stack + "\n" )}`; + } + + out += "\n ..."; + this.log( out ); + } +} diff --git a/test/cli/ConsoleReporter.js b/test/cli/ConsoleReporter.js new file mode 100644 index 000000000..c12aaed79 --- /dev/null +++ b/test/cli/ConsoleReporter.js @@ -0,0 +1,37 @@ +const { EventEmitter } = require( "events" ); + +QUnit.module( "ConsoleReporter", hooks => { + let emitter; + let callCount; + + hooks.beforeEach( function() { + emitter = new EventEmitter(); + callCount = 0; + const con = { + log: () => { + callCount++; + } + }; + QUnit.reporters.console.init( emitter, con ); + } ); + + QUnit.test( "Event \"runStart\"", assert => { + emitter.emit( "runStart", {} ); + assert.equal( callCount, 1 ); + } ); + + QUnit.test( "Event \"runEnd\"", assert => { + emitter.emit( "runEnd", {} ); + assert.equal( callCount, 1 ); + } ); + + QUnit.test( "Event \"testStart\"", assert => { + emitter.emit( "testStart", {} ); + assert.equal( callCount, 1 ); + } ); + + QUnit.test( "Event \"testEnd\"", assert => { + emitter.emit( "testEnd", {} ); + assert.equal( callCount, 1 ); + } ); +} ); diff --git a/test/cli/TapReporter.js b/test/cli/TapReporter.js new file mode 100644 index 000000000..06e9589d9 --- /dev/null +++ b/test/cli/TapReporter.js @@ -0,0 +1,378 @@ +const kleur = require( "kleur" ); +const { EventEmitter } = require( "events" ); + +function mockStack( error ) { + error.stack = ` at Object. (/dev/null/test/unit/data.js:6:5) + at require (internal/modules/cjs/helpers.js:22:18) + at /dev/null/node_modules/mocha/lib/mocha.js:220:27 + at startup (internal/bootstrap/node.js:283:19)`; + return error; +} + +function makeFailingTestEnd( actualValue ) { + return { + name: "Failing", + suiteName: null, + fullName: [ "Failing" ], + status: "failed", + runtime: 0, + errors: [ { + passed: false, + actual: actualValue, + expected: "expected" + } ], + assertions: null + }; +} + +QUnit.module( "TapReporter", hooks => { + let emitter; + let last; + let buffer; + + function log( str ) { + buffer += str + "\n"; + last = str; + } + + hooks.beforeEach( function() { + emitter = new EventEmitter(); + last = undefined; + buffer = ""; + QUnit.reporters.tap.init( emitter, { + log: log + } ); + } ); + + QUnit.test( "output the TAP header", assert => { + emitter.emit( "runStart", {} ); + + assert.strictEqual( last, "TAP version 13" ); + } ); + + QUnit.test( "output ok for a passing test", assert => { + const expected = "ok 1 name"; + + emitter.emit( "testEnd", { + name: "name", + suiteName: null, + fullName: [ "name" ], + status: "passed", + runtime: 0, + errors: [], + assertions: [] + } ); + + assert.strictEqual( last, expected ); + } ); + + QUnit.test( "output ok for a skipped test", assert => { + const expected = kleur.yellow( "ok 1 # SKIP name" ); + + emitter.emit( "testEnd", { + name: "name", + suiteName: null, + fullName: [ "name" ], + status: "skipped", + runtime: 0, + errors: [], + assertions: [] + } ); + assert.strictEqual( last, expected ); + } ); + + QUnit.test( "output not ok for a todo test", assert => { + const expected = kleur.cyan( "not ok 1 # TODO name" ); + + emitter.emit( "testEnd", { + name: "name", + suiteName: null, + fullName: [ "name" ], + status: "todo", + runtime: 0, + errors: [], + assertions: [] + } ); + assert.strictEqual( last, expected ); + } ); + + QUnit.test( "output not ok for a failing test", assert => { + const expected = kleur.red( "not ok 1 name" ); + + emitter.emit( "testEnd", { + name: "name", + suiteName: null, + fullName: [ "name" ], + status: "failed", + runtime: 0, + errors: [], + assertions: [] + } ); + assert.strictEqual( last, expected ); + } ); + + QUnit.test( "output all errors for a failing test", assert => { + emitter.emit( "testEnd", { + name: "name", + suiteName: null, + fullName: [ "name" ], + status: "failed", + runtime: 0, + errors: [ + mockStack( new Error( "first error" ) ), + mockStack( new Error( "second error" ) ) + ], + assertions: [] + } ); + + assert.strictEqual( buffer, `${kleur.red( "not ok 1 name" )} + --- + message: first error + severity: failed + stack: | + at Object. (/dev/null/test/unit/data.js:6:5) + at require (internal/modules/cjs/helpers.js:22:18) + at /dev/null/node_modules/mocha/lib/mocha.js:220:27 + at startup (internal/bootstrap/node.js:283:19) + ... + --- + message: second error + severity: failed + stack: | + at Object. (/dev/null/test/unit/data.js:6:5) + at require (internal/modules/cjs/helpers.js:22:18) + at /dev/null/node_modules/mocha/lib/mocha.js:220:27 + at startup (internal/bootstrap/node.js:283:19) + ... +` + ); + } ); + + QUnit.test( "output expected value of Infinity", assert => { + emitter.emit( "testEnd", { + name: "Failing", + suiteName: null, + fullName: [ "Failing" ], + status: "failed", + runtime: 0, + errors: [ { + passed: false, + actual: "actual", + expected: Infinity + } ], + assertions: null + } ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : actual + expected: Infinity + ...` + ); + } ); + + QUnit.test( "output actual value of undefined", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( undefined ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : undefined + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value of Infinity", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( Infinity ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : Infinity + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value of a string", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( "abc" ) ); + + // No redundant quotes + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : abc + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value with one trailing line break", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( "abc\n" ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : | + abc + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value with two trailing line breaks", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( "abc\n\n" ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : |+ + abc + + + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value of a number string", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( "2" ) ); + + // Quotes required to disambiguate YAML value + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : "2" + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value of boolean string", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( "true" ) ); + + // Quotes required to disambiguate YAML value + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : "true" + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value of 0", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( 0 ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : 0 + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual assertion value of empty array", assert => { + emitter.emit( "testEnd", makeFailingTestEnd( [] ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : [] + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value with a cyclical structure", assert => { + + /// Creates an object that has a cyclical reference. + function createCyclical() { + const cyclical = { a: "example" }; + cyclical.cycle = cyclical; + return cyclical; + } + emitter.emit( "testEnd", makeFailingTestEnd( createCyclical() ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : { + "a": "example", + "cycle": "[Circular]" +} + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual value with a subobject cyclical structure", assert => { + + // Creates an object that has a cyclical reference in a subobject. + function createSubobjectCyclical() { + const cyclical = { a: "example", sub: {} }; + cyclical.sub.cycle = cyclical; + return cyclical; + } + emitter.emit( "testEnd", makeFailingTestEnd( createSubobjectCyclical() ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : { + "a": "example", + "sub": { + "cycle": "[Circular]" + } +} + expected: expected + ...` + ); + } ); + + QUnit.test( "output actual assertion value of an acyclical structure", assert => { + + // Creates an object that references another object more + // than once in an acyclical way. + function createDuplicateAcyclical() { + const duplicate = { + example: "value" + }; + return { + a: duplicate, + b: duplicate, + c: "unique" + }; + } + emitter.emit( "testEnd", makeFailingTestEnd( createDuplicateAcyclical() ) ); + assert.strictEqual( last, ` --- + message: failed + severity: failed + actual : { + "a": { + "example": "value" + }, + "b": { + "example": "value" + }, + "c": "unique" +} + expected: expected + ...` + ); + } ); + + QUnit.test( "output the total number of tests", assert => { + emitter.emit( "runEnd", { + testCounts: { + total: 6, + passed: 3, + failed: 2, + skipped: 1, + todo: 0 + } + } ); + + assert.strictEqual( buffer, `1..6 +# pass 3 +${kleur.yellow( "# skip 1" )} +${kleur.cyan( "# todo 0" )} +${kleur.red( "# fail 2" )} +` + ); + } ); +} );