diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js index 50022d1d91..99a559e7ce 100644 --- a/utils/doclint/check_public_api/test/test.js +++ b/utils/doclint/check_public_api/test/test.js @@ -61,7 +61,7 @@ describe('checkPublicAPI', function() { runner.run(); async function testLint(state, test) { - const dirPath = path.join(__dirname, test.name); + const dirPath = path.join(__dirname, test.name()); const {expect} = new Matchers({ toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) }); @@ -75,7 +75,7 @@ async function testLint(state, test) { } async function testMDBuilder(state, test) { - const dirPath = path.join(__dirname, test.name); + const dirPath = path.join(__dirname, test.name()); const {expect} = new Matchers({ toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) }); @@ -85,7 +85,7 @@ async function testMDBuilder(state, test) { } async function testJSBuilder(state, test) { - const dirPath = path.join(__dirname, test.name); + const dirPath = path.join(__dirname, test.name()); const {expect} = new Matchers({ toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath) }); diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js index e6b85688af..5f42685dc3 100644 --- a/utils/testrunner/Reporter.js +++ b/utils/testrunner/Reporter.js @@ -49,14 +49,14 @@ class Reporter { console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`); console.log(''); const focusedSuites = this._runner.focusedSuites().map(suite => ({ - id: suite.location.filePath + ':' + suite.location.lineNumber + ':' + suite.location.columnNumber, - fullName: suite.fullName, - location: suite.location, + id: suite.location().filePath + ':' + suite.location().lineNumber + ':' + suite.location().columnNumber, + fullName: suite.fullName(), + location: suite.location(), })); const focusedTests = this._runner.focusedTests().map(test => ({ - id: test.location.filePath + ':' + test.location.lineNumber + ':' + test.location.columnNumber, - fullName: test.fullName, - location: test.location, + id: test.location().filePath + ':' + test.location().lineNumber + ':' + test.location().columnNumber, + fullName: test.fullName(), + location: test.location(), })); const focusedEntities = new Map([ ...focusedSuites.map(suite => ([suite.id, suite])), @@ -121,7 +121,7 @@ class Reporter { if (markedAsFailingTests.length > 0) { console.log('\nMarked as failing:'); markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => { - console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`); + console.log(`${index + 1}) ${test.fullName()} (${formatLocation(test.location())})`); }); } if (this._showMarkedAsFailingTests < markedAsFailingTests.length) { @@ -140,7 +140,7 @@ class Reporter { for (let i = 0; i < slowTests.length; ++i) { const test = slowTests[i]; const duration = test.endTimestamp - test.startTimestamp; - console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName} (${formatLocation(test.location)})`); + console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName()} (${formatLocation(test.location())})`); } } @@ -195,23 +195,23 @@ class Reporter { if (this._runner.parallel() > 1 && workerId !== undefined) prefix += ' ' + colors.gray(`[worker = ${workerId}]`); if (test.result === 'ok') { - console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName} (${formatLocation(test.location)})`); + console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`); } else if (test.result === 'terminated') { - console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName} (${formatLocation(test.location)})`); + console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName()} (${formatLocation(test.location())})`); } else if (test.result === 'crashed') { - console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`); + console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName()} (${formatLocation(test.location())})`); } else if (test.result === 'skipped') { } else if (test.result === 'markedAsFailing') { - console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName} (${formatLocation(test.location)})`); + console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`); } else if (test.result === 'timedout') { - console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`); + console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`); if (test.output) { console.log(' Output:'); for (const line of test.output) console.log(' ' + line); } } else if (test.result === 'failed') { - console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName} (${formatLocation(test.location)})`); + console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName()} (${formatLocation(test.location())})`); if (test.error instanceof MatchError) { let lines = this._filePathToLines.get(test.error.location.filePath); if (!lines) { @@ -225,7 +225,7 @@ class Reporter { const lineNumber = test.error.location.lineNumber; if (lineNumber < lines.length) { const lineNumberLength = (lineNumber + 1 + '').length; - const FROM = Math.max(test.location.lineNumber - 1, lineNumber - 5); + const FROM = Math.max(test.location().lineNumber - 1, lineNumber - 5); const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n'); const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(test.error.location.columnNumber - 1) + '^'; console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n'); @@ -243,10 +243,10 @@ class Reporter { console.log(' Stack:'); let stack = test.error.stack; // Highlight first test location, if any. - const match = stack.match(new RegExp(test.location.filePath + ':(\\d+):(\\d+)')); + const match = stack.match(new RegExp(test.location().filePath + ':(\\d+):(\\d+)')); if (match) { const [, line, column] = match; - const fileName = `${test.location.fileName}:${line}:${column}`; + const fileName = `${test.location().fileName}:${line}:${column}`; stack = stack.substring(0, match.index) + stack.substring(match.index).replace(fileName, colors.yellow(fileName)); } console.log(padLines(stack, 4)); diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index 299220d863..de20a688fa 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -73,39 +73,173 @@ function isTestFailure(testResult) { } class Test { - constructor(suite, name, callback, declaredMode, expectation, timeout) { - this.suite = suite; - this.name = name; - this.fullName = (suite.fullName + ' ' + name).trim(); - this.declaredMode = declaredMode; - this.expectation = expectation; - this._callback = callback; - this.location = getCallerLocation(__filename); - this.timeout = timeout; + constructor(suite, name, callback, location) { + this._suite = suite; + this._name = name; + this._fullName = (suite.fullName() + ' ' + name).trim(); + this._mode = TestMode.Run; + this._expectation = TestExpectation.Ok; + this._body = callback; + this._location = location; + this._timeout = INFINITE_TIMEOUT; + this._repeat = 1; - // Test results + // Test results. TODO: make these private. this.result = null; this.error = null; this.startTimestamp = 0; this.endTimestamp = 0; + + this.Modes = { ...TestMode }; + this.Expectations = { ...TestExpectation }; + } + + _clone() { + // TODO: introduce TestRun instead? + const test = new Test(this._suite, this._name, this._body, this._location); + test._timeout = this._timeout; + test._mode = this._mode; + test._expectation = this._expectation; + return test; + } + + suite() { + return this._suite; + } + + name() { + return this._name; + } + + fullName() { + return this._fullName; + } + + location() { + return this._location; + } + + body() { + return this._body; + } + + mode() { + return this._mode; + } + + setMode(mode) { + if (this._mode !== TestMode.Focus) + this._mode = mode; + } + + timeout() { + return this._timeout; + } + + setTimeout(timeout) { + this._timeout = timeout; + } + + expectation() { + return this._expectation; + } + + setExpectation(expectation) { + this._expectation = expectation; + } + + repeat() { + return this._repeat; + } + + setRepeat(repeat) { + this._repeat = repeat; + } + + effectiveMode() { + for (let suite = this._suite; suite; suite = suite.parentSuite()) { + if (suite.mode() === TestMode.Skip) + return TestMode.Skip; + } + return this._mode; + } + + effectiveExpectation() { + for (let suite = this._suite; suite; suite = suite.parentSuite()) { + if (suite.expectation() === TestExpectation.Fail) + return TestExpectation.Fail; + } + return this._expectation; } } class Suite { - constructor(parentSuite, name, declaredMode, expectation) { - this.parentSuite = parentSuite; - this.name = name; - this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim(); - this.declaredMode = declaredMode; - this.expectation = expectation; - /** @type {!Array<(!Test|!Suite)>} */ - this.children = []; - this.location = getCallerLocation(__filename); + constructor(parentSuite, name, location) { + this._parentSuite = parentSuite; + this._name = name; + this._fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim(); + this._mode = TestMode.Run; + this._expectation = TestExpectation.Ok; + this._location = location; + this._repeat = 1; + // TODO: make these private. this.beforeAll = null; this.beforeEach = null; this.afterAll = null; this.afterEach = null; + + this.Modes = { ...TestMode }; + this.Expectations = { ...TestExpectation }; + } + + _clone() { + // TODO: introduce TestRun instead? + const suite = new Suite(this._parentSuite, this._name, this._location); + suite._mode = this._mode; + suite._expectation = this._expectation; + return suite; + } + + parentSuite() { + return this._parentSuite; + } + + name() { + return this._name; + } + + fullName() { + return this._fullName; + } + + mode() { + return this._mode; + } + + setMode(mode) { + if (this._mode !== TestMode.Focus) + this._mode = mode; + } + + location() { + return this._location; + } + + expectation() { + return this._expectation; + } + + setExpectation(expectation) { + this._expectation = expectation; + } + + repeat() { + return this._repeat; + } + + setRepeat(repeat) { + this._repeat = repeat; } } @@ -179,14 +313,14 @@ class TestWorker { if (this._markTerminated(test)) return; - if (test.declaredMode === TestMode.Skip) { + if (test.effectiveMode() === TestMode.Skip) { await this._testPass._willStartTest(this, test); test.result = TestResult.Skipped; await this._testPass._didFinishTest(this, test); return; } - if (test.expectation === TestExpectation.Fail && test.declaredMode !== TestMode.Focus) { + if (test.effectiveExpectation() === TestExpectation.Fail && test.effectiveMode() !== TestMode.Focus) { await this._testPass._willStartTest(this, test); test.result = TestResult.MarkedAsFailing; await this._testPass._didFinishTest(this, test); @@ -194,7 +328,7 @@ class TestWorker { } const suiteStack = []; - for (let suite = test.suite; suite; suite = suite.parentSuite) + for (let suite = test.suite(); suite; suite = suite.parentSuite()) suiteStack.push(suite); suiteStack.reverse(); @@ -230,7 +364,7 @@ class TestWorker { if (!test.error && !this._markTerminated(test)) { await this._testPass._willStartTestBody(this, test); - const { promise, terminate } = runUserCallback(test._callback, test.timeout, [this._state, test]); + const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, test]); this._runningTestTerminate = terminate; test.error = await promise; this._runningTestTerminate = null; @@ -259,7 +393,7 @@ class TestWorker { await this._testPass._willStartHook(this, suite, hook.location, hookName); const timeout = this._testPass._runner._timeout; - const { promise, terminate } = runUserCallback(hook.callback, timeout, [this._state, test]); + const { promise, terminate } = runUserCallback(hook.body, timeout, [this._state, test]); this._runningHookTerminate = terminate; let error = await promise; this._runningHookTerminate = null; @@ -272,7 +406,7 @@ class TestWorker { } let message; if (error === TimeoutError) { - message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hookName}" in suite "${suite.fullName}"`; + message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hookName}" in suite "${suite.fullName()}"`; error = null; } else if (error === TerminatedError) { // Do not report termination details - it's just noise. @@ -281,7 +415,7 @@ class TestWorker { } else { if (error.stack) await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); - message = `${locationString} - FAILED while running "${hookName}" in suite "${suite.fullName}": `; + message = `${locationString} - FAILED while running "${hookName}" in suite "${suite.fullName()}": `; } await this._testPass._didFailHook(this, suite, hook.location, hookName, message, error); test.error = error; @@ -411,78 +545,29 @@ class TestPass { } async _willStartTestBody(worker, test) { - debug('testrunner:test')(`[${worker._workerId}] starting "${test.fullName}" (${test.location.fileName + ':' + test.location.lineNumber})`); + debug('testrunner:test')(`[${worker._workerId}] starting "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`); } async _didFinishTestBody(worker, test) { - debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName}" (${test.location.fileName + ':' + test.location.lineNumber})`); + debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`); } async _willStartHook(worker, suite, location, hookName) { - debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" started for "${suite.fullName}" (${location.fileName + ':' + location.lineNumber})`); + debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" started for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`); } async _didFailHook(worker, suite, location, hookName, message, error) { - debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" FAILED for "${suite.fullName}" (${location.fileName + ':' + location.lineNumber})`); + debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" FAILED for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`); if (message) this._result.addError(message, error, worker); this._result.setResult(TestResult.Crashed, message); } async _didCompleteHook(worker, suite, location, hookName) { - debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" OK for "${suite.fullName}" (${location.fileName + ':' + location.lineNumber})`); + debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" OK for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`); } } -// TODO: merge spec with Test/Suite. -function createSpec(name, callback) { - let timeout = INFINITE_TIMEOUT; - let repeat = 1; - let expectation = TestExpectation.Ok; - let mode = TestMode.Run; - const spec = { - Modes: { ...TestMode }, - Expectations: { ...TestExpectation }, - name() { - return name; - }, - callback() { - return callback; - }, - mode() { - return mode; - }, - setMode(m) { - if (mode !== TestMode.Focus) - mode = m; - }, - expectations() { - return [expectation]; - }, - setExpectations(e) { - if (Array.isArray(e)) { - if (e.length > 1) - throw new Error('Only a single expectation is currently supported'); - e = e[0]; - } - expectation = e; - }, - timeout() { - return timeout; - }, - setTimeout(t) { - timeout = t; - }, - repeat() { - return repeat; - }, - setRepeat(r) { - repeat = r; - }, - }; - return spec; -} - class TestRunner extends EventEmitter { constructor(options = {}) { super(); @@ -495,15 +580,18 @@ class TestRunner extends EventEmitter { } = options; this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; this._sourceMapSupport = new SourceMapSupport(); - this._rootSuite = new Suite(null, '', TestMode.Run); + const dummyLocation = { fileName: '', filePath: '', lineNumber: 0, columnNumber: 0 }; + this._rootSuite = new Suite(null, '', dummyLocation); this._currentSuite = this._rootSuite; this._tests = []; this._suites = []; this._timeout = timeout === 0 ? INFINITE_TIMEOUT : timeout; this._parallel = parallel; this._breakOnFailure = breakOnFailure; - this._modifiers = new Map(); - this._attributes = new Map(); + this._suiteModifiers = new Map(); + this._suiteAttributes = new Map(); + this._testModifiers = new Map(); + this._testAttributes = new Map(); if (MAJOR_NODEJS_VERSION >= 8 && disableTimeoutWhenInspectorIsEnabled) { if (inspector.url()) { @@ -520,23 +608,27 @@ class TestRunner extends EventEmitter { this.afterAll = this._addHook.bind(this, 'afterAll'); this.afterEach = this._addHook.bind(this, 'afterEach'); - this.describe = this._specBuilder([], true); - this.it = this._specBuilder([], false); + this.describe = this._suiteBuilder([]); + this.it = this._testBuilder([]); - this._attributes.set('debug', t => { + this.testAttribute('debug', t => { t.setMode(t.Modes.Focus); t.setTimeout(INFINITE_TIMEOUT); - const N = t.callback().toString().split('\n').length; - const location = getCallerLocation(__filename); + const N = t.body().toString().split('\n').length; + const location = t.location(); for (let line = 0; line < N; ++line) this._debuggerLogBreakpointLines.set(location.filePath, line + location.lineNumber); }); - this._modifiers.set('skip', (t, condition) => condition && t.setMode(t.Modes.Skip)); - this._modifiers.set('fail', (t, condition) => condition && t.setExpectations(t.Expectations.Fail)); - this._modifiers.set('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3)); - this._modifiers.set('repeat', (t, count) => t.setRepeat(count)); - this._attributes.set('focus', t => t.setMode(t.Modes.Focus)); + this.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip)); + this.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip)); + this.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); + this.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); + this.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3)); + this.testModifier('repeat', (t, count) => t.setRepeat(count)); + this.suiteModifier('repeat', (s, count) => s.setRepeat(count)); + this.testAttribute('focus', t => t.setMode(t.Modes.Focus)); + this.suiteAttribute('focus', s => s.setMode(s.Modes.Focus)); this.fdescribe = this.describe.focus; this.xdescribe = this.describe.skip(true); this.fit = this.it.focus; @@ -544,86 +636,78 @@ class TestRunner extends EventEmitter { this.dit = this.it.debug; } - _specBuilder(callbacks, isSuite) { - return new Proxy(() => {}, { - apply: (target, thisArg, [name, callback]) => { - const spec = createSpec(name, callback); - spec.setTimeout(this._timeout); - for (const { callback, args } of callbacks) - callback(spec, ...args); - if (isSuite) - this._addSuite(spec, []); - else - this._addTest(spec); - }, + _suiteBuilder(callbacks) { + return new Proxy((name, callback, ...suiteArgs) => { + const location = getCallerLocation(__filename); + const suite = new Suite(this._currentSuite, name, location); + for (const { callback, args } of callbacks) + callback(suite, ...args); + for (let i = 0; i < suite.repeat(); i++) { + this._currentSuite = suite._clone(); + callback(...suiteArgs); + this._suites.push(this._currentSuite); + this._currentSuite = this._currentSuite.parentSuite(); + } + }, { get: (obj, prop) => { - if (this._modifiers.has(prop)) - return (...args) => this._specBuilder([...callbacks, { callback: this._modifiers.get(prop), args }], isSuite); - if (this._attributes.has(prop)) - return this._specBuilder([...callbacks, { callback: this._attributes.get(prop), args: [] }], isSuite); + if (this._suiteModifiers.has(prop)) + return (...args) => this._suiteBuilder([...callbacks, { callback: this._suiteModifiers.get(prop), args }]); + if (this._suiteAttributes.has(prop)) + return this._suiteBuilder([...callbacks, { callback: this._suiteAttributes.get(prop), args: [] }]); return obj[prop]; }, }); } - modifier(name, callback) { - this._modifiers.set(name, callback); + _testBuilder(callbacks) { + return new Proxy((name, callback) => { + const location = getCallerLocation(__filename); + const test = new Test(this._currentSuite, name, callback, location); + test.setTimeout(this._timeout); + for (const { callback, args } of callbacks) + callback(test, ...args); + for (let i = 0; i < test.repeat(); i++) + this._tests.push(test._clone()); + }, { + get: (obj, prop) => { + if (this._testModifiers.has(prop)) + return (...args) => this._testBuilder([...callbacks, { callback: this._testModifiers.get(prop), args }]); + if (this._testAttributes.has(prop)) + return this._testBuilder([...callbacks, { callback: this._testAttributes.get(prop), args: [] }]); + return obj[prop]; + }, + }); } - attribute(name, callback) { - this._attributes.set(name, callback); + testModifier(name, callback) { + this._testModifiers.set(name, callback); + } + + testAttribute(name, callback) { + this._testAttributes.set(name, callback); + } + + suiteModifier(name, callback) { + this._suiteModifiers.set(name, callback); + } + + suiteAttribute(name, callback) { + this._suiteAttributes.set(name, callback); } loadTests(module, ...args) { - if (typeof module.describe === 'function') { - const spec = createSpec('', module.describe); - spec.setMode(spec.Modes.Run); - this._addSuite(spec, args); - } - if (typeof module.fdescribe === 'function') { - const spec = createSpec('', module.fdescribe); - spec.setMode(spec.Modes.Focus); - this._addSuite(spec, args); - } - if (typeof module.xdescribe === 'function') { - const spec = createSpec('', module.xdescribe); - spec.setMode(spec.Modes.Skip); - this._addSuite(spec, args); - } - } - - _addTest(spec) { - for (let i = 0; i < spec.repeat(); i++) { - let expectation = spec.expectations()[0]; - let mode = spec.mode(); - for (let suite = this._currentSuite; suite; suite = suite.parentSuite) { - if (suite.expectation === TestExpectation.Fail) - expectation = TestExpectation.Fail; - if (suite.declaredMode === TestMode.Skip) - mode = TestMode.Skip; - } - const test = new Test(this._currentSuite, spec.name(), spec.callback(), mode, expectation, spec.timeout()); - this._currentSuite.children.push(test); - this._tests.push(test); - } - } - - _addSuite(spec, args) { - for (let i = 0; i < spec.repeat(); i++) { - const oldSuite = this._currentSuite; - const suite = new Suite(this._currentSuite, spec.name(), spec.mode(), spec.expectations()[0]); - this._suites.push(suite); - this._currentSuite.children.push(suite); - this._currentSuite = suite; - spec.callback()(...args); - this._currentSuite = oldSuite; - } + if (typeof module.describe === 'function') + this.describe('', module.describe, ...args); + if (typeof module.fdescribe === 'function') + this.describe.focus('', module.fdescribe, ...args); + if (typeof module.xdescribe === 'function') + this.describe.skip(true)('', module.xdescribe, ...args); } _addHook(hookName, callback) { assert(this._currentSuite[hookName] === null, `Only one ${hookName} hook available per suite`); const location = getCallerLocation(__filename); - this._currentSuite[hookName] = { callback, location }; + this._currentSuite[hookName] = { body: callback, location }; } async run(options = {}) { @@ -636,21 +720,17 @@ class TestRunner extends EventEmitter { if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) { result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI'); } else { + this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure); let timeoutId; - const timeoutPromise = new Promise(resolve => { - const timeoutResult = new Result(); - timeoutResult.setResult(TestResult.Crashed, `Total timeout of ${totalTimeout}ms reached.`); - if (totalTimeout) - timeoutId = setTimeout(resolve.bind(null, timeoutResult), totalTimeout); - }); + if (totalTimeout) { + timeoutId = setTimeout(() => { + this._runningPass._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */); + }, totalTimeout); + } try { - this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure); - result = await Promise.race([ - this._runningPass.run(runnableTests).catch(e => { console.error(e); throw e; }), - timeoutPromise, - ]); - this._runningPass = null; + result = await this._runningPass.run(runnableTests).catch(e => { console.error(e); throw e; }); } finally { + this._runningPass = null; clearTimeout(timeoutId); } } @@ -679,18 +759,18 @@ class TestRunner extends EventEmitter { // First pass: pick "fit" and blacklist parent suites for (let i = 0; i < this._tests.length; i++) { const test = this._tests[i]; - if (test.declaredMode !== TestMode.Focus) + if (test.mode() !== TestMode.Focus) continue; tests.push({ i, test }); - for (let suite = test.suite; suite; suite = suite.parentSuite) + for (let suite = test.suite(); suite; suite = suite.parentSuite()) blacklistSuites.add(suite); } // Second pass: pick all tests that belong to non-blacklisted "fdescribe" for (let i = 0; i < this._tests.length; i++) { const test = this._tests[i]; let insideFocusedSuite = false; - for (let suite = test.suite; suite; suite = suite.parentSuite) { - if (!blacklistSuites.has(suite) && suite.declaredMode === TestMode.Focus) { + for (let suite = test.suite(); suite; suite = suite.parentSuite()) { + if (!blacklistSuites.has(suite) && suite.mode() === TestMode.Focus) { insideFocusedSuite = true; break; } @@ -703,11 +783,11 @@ class TestRunner extends EventEmitter { } focusedSuites() { - return this._suites.filter(suite => suite.declaredMode === TestMode.Focus); + return this._suites.filter(suite => suite.mode() === TestMode.Focus); } focusedTests() { - return this._tests.filter(test => test.declaredMode === TestMode.Focus); + return this._tests.filter(test => test.effectiveMode() === TestMode.Focus); } hasFocusedTestsOrSuites() { @@ -716,8 +796,8 @@ class TestRunner extends EventEmitter { focusMatchingTests(fullNameRegex) { for (const test of this._tests) { - if (fullNameRegex.test(test.fullName)) - test.declaredMode = TestMode.Focus; + if (fullNameRegex.test(test.fullName())) + test.setMode(TestMode.Focus); } } @@ -726,7 +806,7 @@ class TestRunner extends EventEmitter { } failedTests() { - return this._tests.filter(test => test.result === TestResult.Failed || test.result === TestResult.TimedOut || test.result === TestResult.Crashed); + return this._tests.filter(test => isTestFailure(test.result)); } passedTests() { diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index 126f4052ae..2884dfc829 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -17,13 +17,13 @@ module.exports.addTests = function({testRunner, expect}) { t.it('uno', () => {}); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('uno'); - expect(test.declaredMode).toBe('run'); - expect(test.location.filePath).toEqual(__filename); - expect(test.location.fileName).toEqual('testrunner.spec.js'); - expect(test.location.lineNumber).toBeTruthy(); - expect(test.location.columnNumber).toBeTruthy(); + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('uno'); + expect(test.effectiveMode()).toBe('run'); + expect(test.location().filePath).toEqual(__filename); + expect(test.location().fileName).toEqual('testrunner.spec.js'); + expect(test.location().lineNumber).toBeTruthy(); + expect(test.location().columnNumber).toBeTruthy(); }); }); @@ -33,9 +33,9 @@ module.exports.addTests = function({testRunner, expect}) { t.xit('uno', () => {}); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('uno'); - expect(test.declaredMode).toBe('skip'); + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('uno'); + expect(test.effectiveMode()).toBe('skip'); }); }); @@ -45,9 +45,9 @@ module.exports.addTests = function({testRunner, expect}) { t.fit('uno', () => {}); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('uno'); - expect(test.declaredMode).toBe('focus'); + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('uno'); + expect(test.effectiveMode()).toBe('focus'); }); it('should run a failed focused test', async() => { const t = newTestRunner(); @@ -56,7 +56,7 @@ module.exports.addTests = function({testRunner, expect}) { expect(t.tests().length).toBe(1); await t.run(); expect(run).toBe(true); - expect(t.failedTests()[0].name).toBe('uno'); + expect(t.failedTests()[0].name()).toBe('uno'); }); }); @@ -68,12 +68,12 @@ module.exports.addTests = function({testRunner, expect}) { }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('suite uno'); - expect(test.declaredMode).toBe('run'); - expect(test.suite.name).toBe('suite'); - expect(test.suite.fullName).toBe('suite'); - expect(test.suite.declaredMode).toBe('run'); + expect(test.name()).toBe('uno'); + expect(test.fullName()).toBe('suite uno'); + expect(test.effectiveMode()).toBe('run'); + expect(test.suite().name()).toBe('suite'); + expect(test.suite().fullName()).toBe('suite'); + expect(test.suite().mode()).toBe('run'); }); }); @@ -85,8 +85,8 @@ module.exports.addTests = function({testRunner, expect}) { }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.declaredMode).toBe('skip'); - expect(test.suite.declaredMode).toBe('skip'); + expect(test.effectiveMode()).toBe('skip'); + expect(test.suite().mode()).toBe('skip'); }); it('focused tests inside a skipped suite are considered skipped', async() => { const t = newTestRunner(); @@ -95,8 +95,8 @@ module.exports.addTests = function({testRunner, expect}) { }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.declaredMode).toBe('skip'); - expect(test.suite.declaredMode).toBe('skip'); + expect(test.effectiveMode()).toBe('skip'); + expect(test.suite().mode()).toBe('skip'); }); }); @@ -108,8 +108,8 @@ module.exports.addTests = function({testRunner, expect}) { }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.declaredMode).toBe('run'); - expect(test.suite.declaredMode).toBe('focus'); + expect(test.effectiveMode()).toBe('run'); + expect(test.suite().mode()).toBe('focus'); }); it('skipped tests inside a focused suite should stay skipped', async() => { const t = newTestRunner(); @@ -118,8 +118,8 @@ module.exports.addTests = function({testRunner, expect}) { }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.declaredMode).toBe('skip'); - expect(test.suite.declaredMode).toBe('focus'); + expect(test.effectiveMode()).toBe('skip'); + expect(test.suite().mode()).toBe('focus'); }); it('should run all "run" tests inside a focused suite', async() => { const log = []; @@ -164,7 +164,7 @@ module.exports.addTests = function({testRunner, expect}) { const t = newTestRunner({timeout: 123}); const log = []; - t.modifier('foo', (t, ...args) => { + t.testModifier('foo', (t, ...args) => { log.push('foo'); expect(t.Modes.Run).toBeTruthy(); @@ -173,7 +173,7 @@ module.exports.addTests = function({testRunner, expect}) { expect(t.mode()).toBe(t.Modes.Run); expect(t.Expectations.Ok).toBeTruthy(); expect(t.Expectations.Fail).toBeTruthy(); - expect(t.expectations()).toEqual([t.Expectations.Ok]); + expect(t.expectation()).toBe(t.Expectations.Ok); expect(t.timeout()).toBe(123); expect(t.repeat()).toBe(1); @@ -182,17 +182,17 @@ module.exports.addTests = function({testRunner, expect}) { expect(args[1]).toBe('dos'); t.setMode(t.Modes.Focus); - t.setExpectations([t.Expectations.Fail]); + t.setExpectation(t.Expectations.Fail); t.setTimeout(234); t.setRepeat(42); }); - t.attribute('bar', t => { + t.testAttribute('bar', t => { log.push('bar'); expect(t.mode()).toBe(t.Modes.Focus); t.setMode(t.Modes.Skip); expect(t.mode()).toBe(t.Modes.Focus); - expect(t.expectations()).toEqual([t.Expectations.Fail]); + expect(t.expectation()).toBe(t.Expectations.Fail); expect(t.timeout()).toBe(234); expect(t.repeat()).toBe(42); }); @@ -339,6 +339,24 @@ module.exports.addTests = function({testRunner, expect}) { await t.run(); expect(ran).toBe(true); }); + it('should handle repeat', async() => { + const t = newTestRunner(); + let suite = 0; + let test = 0; + let beforeAll = 0; + let beforeEach = 0; + t.describe.repeat(2)('suite', () => { + suite++; + t.beforeAll(() => beforeAll++); + t.beforeEach(() => beforeEach++); + t.it.repeat(3)('uno', () => test++); + }); + await t.run(); + expect(suite).toBe(2); + expect(beforeAll).toBe(2); + expect(beforeEach).toBe(6); + expect(test).toBe(6); + }); it('should run tests if some fail', async() => { const t = newTestRunner(); const log = []; @@ -376,7 +394,7 @@ module.exports.addTests = function({testRunner, expect}) { t.beforeEach(state => state.FOO = 42); t.it('uno', (state, test) => { log.push('state.FOO=' + state.FOO); - log.push('test=' + test.name); + log.push('test=' + test.name()); }); await t.run(); expect(log.join()).toBe('state.FOO=42,test=uno'); @@ -454,6 +472,13 @@ module.exports.addTests = function({testRunner, expect}) { await t.run(); expect(log.join()).toBe('1,2,3,4'); }); + it('should respect total timeout', async() => { + const t = newTestRunner({timeout: 10000}); + t.it('uno', async () => { await new Promise(() => {}); }); + const result = await t.run({totalTimeout: 1}); + expect(t.tests()[0].result).toBe('terminated'); + expect(result.message).toContain('Total timeout'); + }); }); describe('TestRunner.run result', () => {