diff --git a/test/test.js b/test/test.js index 7f498da98e..8ce27cd958 100644 --- a/test/test.js +++ b/test/test.js @@ -43,17 +43,17 @@ const testRunner = new TestRunner({ breakOnFailure: process.argv.indexOf('--break-on-failure') !== -1, installCommonHelpers: false }); -testRunner.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip)); -testRunner.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip)); +testRunner.testModifier('skip', (t, condition) => condition && t.setSkipped(true)); +testRunner.suiteModifier('skip', (s, condition) => condition && s.setSkipped(true)); testRunner.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail)); testRunner.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail)); testRunner.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3)); testRunner.testModifier('repeat', (t, count) => t.setRepeat(count)); testRunner.suiteModifier('repeat', (s, count) => s.setRepeat(count)); -testRunner.testAttribute('focus', t => t.setMode(t.Modes.Focus)); -testRunner.suiteAttribute('focus', s => s.setMode(s.Modes.Focus)); +testRunner.testAttribute('focus', t => t.setFocused(true)); +testRunner.suiteAttribute('focus', s => s.setFocused(true)); testRunner.testAttribute('debug', t => { - t.setMode(t.Modes.Focus); + t.setFocused(true); t.setTimeout(100000000); let session; diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js index 5f42685dc3..8a51a20ea0 100644 --- a/utils/testrunner/Reporter.js +++ b/utils/testrunner/Reporter.js @@ -39,34 +39,23 @@ class Reporter { runner.on('testfinished', this._onTestFinished.bind(this)); } - _onStarted(runnableTests) { + _onStarted(testRuns) { this._testCounter = 0; this._timestamp = Date.now(); const allTests = this._runner.tests(); - if (allTests.length === runnableTests.length) { - console.log(`Running all ${colors.yellow(runnableTests.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`); + if (!this._runner.hasFocusedTestsOrSuites()) { + console.log(`Running all ${colors.yellow(testRuns.length)} tests on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}:\n`); } else { - console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`); + console.log(`Running ${colors.yellow(testRuns.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`); console.log(''); - const focusedSuites = this._runner.focusedSuites().map(suite => ({ - id: suite.location().filePath + ':' + suite.location().lineNumber + ':' + suite.location().columnNumber, - fullName: suite.fullName(), - location: suite.location(), - })); - const focusedTests = this._runner.focusedTests().map(test => ({ - id: test.location().filePath + ':' + test.location().lineNumber + ':' + test.location().columnNumber, - fullName: test.fullName(), - location: test.location(), - })); - const focusedEntities = new Map([ - ...focusedSuites.map(suite => ([suite.id, suite])), - ...focusedTests.map(test => ([test.id, test])), - ]); - if (focusedEntities.size) { + const focusedEntities = [ + ...this._runner.suites().filter(suite => suite.focused()), + ...this._runner.tests().filter(test => test.focused()), + ]; + if (focusedEntities.length) { console.log('Focused Suites and Tests:'); - const entities = [...focusedEntities.values()]; - for (let i = 0; i < entities.length; ++i) - console.log(` ${i + 1}) ${entities[i].fullName} (${formatLocation(entities[i].location)})`); + for (let i = 0; i < focusedEntities.length; ++i) + console.log(` ${i + 1}) ${focusedEntities[i].fullName()} (${formatLocation(focusedEntities[i].location())})`); console.log(''); } } @@ -80,168 +69,164 @@ class Reporter { } for (let i = 0; i < result.errors.length; i++) { - const { message, error, workerId, tests } = result.errors[i]; + const { message, error, runs } = result.errors[i]; console.log(`\n${colors.magenta('NON-TEST ERROR #' + i)}: ${message}`); if (error && error.stack) console.log(padLines(error.stack, 2)); - const lastTests = tests.slice(tests.length - Math.min(10, tests.length)); - if (lastTests.length) + const lastRuns = runs.slice(runs.length - Math.min(10, runs.length)); + if (lastRuns.length) console.log(`WORKER STATE`); - for (let j = 0; j < lastTests.length; j++) - this._printVerboseTestResult(j, lastTests[j], workerId); + for (let j = 0; j < lastRuns.length; j++) + this._printVerboseTestRunResult(j, lastRuns[j]); } console.log(''); console.log(''); } _onFinished(result) { - this._printTestResults(); + this._printTestResults(result); if (!result.ok()) this._printFailedResult(result); process.exitCode = result.exitCode; } - _printTestResults() { + _printTestResults(result) { // 2 newlines after completing all tests. console.log('\n'); - const failedTests = this._runner.failedTests(); - if (this._summary && failedTests.length > 0) { + const runs = result.runs; + const failedRuns = runs.filter(run => run.isFailure()); + const executedRuns = runs.filter(run => run.result()); + const okRuns = runs.filter(run => run.ok()); + const skippedRuns = runs.filter(run => run.result() === 'skipped'); + const markedAsFailingRuns = runs.filter(run => run.result() === 'markedAsFailing'); + + if (this._summary && failedRuns.length > 0) { console.log('\nFailures:'); - for (let i = 0; i < failedTests.length; ++i) { - const test = failedTests[i]; - this._printVerboseTestResult(i + 1, test); + for (let i = 0; i < failedRuns.length; ++i) { + this._printVerboseTestRunResult(i + 1, failedRuns[i]); console.log(''); } } - const skippedTests = this._runner.skippedTests(); - const markedAsFailingTests = this._runner.markedAsFailingTests(); - if (this._showMarkedAsFailingTests && this._summary && markedAsFailingTests.length) { - if (markedAsFailingTests.length > 0) { + if (this._showMarkedAsFailingTests && this._summary && markedAsFailingRuns.length) { + if (markedAsFailingRuns.length > 0) { console.log('\nMarked as failing:'); - markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => { - console.log(`${index + 1}) ${test.fullName()} (${formatLocation(test.location())})`); + markedAsFailingRuns.slice(0, this._showMarkedAsFailingTests).forEach((testRun, index) => { + console.log(`${index + 1}) ${testRun.test().fullName()} (${formatLocation(testRun.test().location())})`); }); } - if (this._showMarkedAsFailingTests < markedAsFailingTests.length) { + if (this._showMarkedAsFailingTests < markedAsFailingRuns.length) { console.log(''); - console.log(`... and ${colors.yellow(markedAsFailingTests.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`); + console.log(`... and ${colors.yellow(markedAsFailingRuns.length - this._showMarkedAsFailingTests)} more marked as failing tests ...`); } } if (this._showSlowTests) { - const slowTests = this._runner.passedTests().sort((a, b) => { - const aDuration = a.endTimestamp - a.startTimestamp; - const bDuration = b.endTimestamp - b.startTimestamp; - return bDuration - aDuration; - }).slice(0, this._showSlowTests); + const slowRuns = okRuns.sort((a, b) => b.duration() - a.duration()).slice(0, this._showSlowTests); console.log(`\nSlowest tests:`); - for (let i = 0; i < slowTests.length; ++i) { - const test = slowTests[i]; - const duration = test.endTimestamp - test.startTimestamp; - console.log(` (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName()} (${formatLocation(test.location())})`); + for (let i = 0; i < slowRuns.length; ++i) { + const run = slowRuns[i]; + console.log(` (${i + 1}) ${colors.yellow((run.duration() / 1000) + 's')} - ${run.test().fullName()} (${formatLocation(run.test().location())})`); } } - const tests = this._runner.tests(); - const executedTests = tests.filter(test => test.result); - const okTestsLength = executedTests.length - failedTests.length - markedAsFailingTests.length - skippedTests.length; let summaryText = ''; - if (failedTests.length || markedAsFailingTests.length) { - const summary = [`ok - ${colors.green(okTestsLength)}`]; - if (failedTests.length) - summary.push(`failed - ${colors.red(failedTests.length)}`); - if (markedAsFailingTests.length) - summary.push(`marked as failing - ${colors.yellow(markedAsFailingTests.length)}`); - if (skippedTests.length) - summary.push(`skipped - ${colors.yellow(skippedTests.length)}`); + if (failedRuns.length || markedAsFailingRuns.length) { + const summary = [`ok - ${colors.green(okRuns.length)}`]; + if (failedRuns.length) + summary.push(`failed - ${colors.red(failedRuns.length)}`); + if (markedAsFailingRuns.length) + summary.push(`marked as failing - ${colors.yellow(markedAsFailingRuns.length)}`); + if (skippedRuns.length) + summary.push(`skipped - ${colors.yellow(skippedRuns.length)}`); summaryText = ` (${summary.join(', ')})`; } - console.log(`\nRan ${executedTests.length}${summaryText} of ${tests.length} test${tests.length > 1 ? 's' : ''}`); + console.log(`\nRan ${executedRuns.length}${summaryText} of ${runs.length} test${runs.length > 1 ? 's' : ''}`); const milliseconds = Date.now() - this._timestamp; const seconds = milliseconds / 1000; console.log(`Finished in ${colors.yellow(seconds)} seconds`); } - _onTestStarted(test, workerId) { + _onTestStarted(testRun) { } - _onTestFinished(test, workerId) { + _onTestFinished(testRun) { if (this._verbose) { ++this._testCounter; - this._printVerboseTestResult(this._testCounter, test, workerId); + this._printVerboseTestRunResult(this._testCounter, testRun); } else { - if (test.result === 'ok') + if (testRun.result() === 'ok') process.stdout.write(colors.green('\u00B7')); - else if (test.result === 'skipped') + else if (testRun.result() === 'skipped') process.stdout.write(colors.yellow('\u00B7')); - else if (test.result === 'markedAsFailing') + else if (testRun.result() === 'markedAsFailing') process.stdout.write(colors.yellow('\u00D7')); - else if (test.result === 'failed') + else if (testRun.result() === 'failed') process.stdout.write(colors.red('F')); - else if (test.result === 'crashed') + else if (testRun.result() === 'crashed') process.stdout.write(colors.red('C')); - else if (test.result === 'terminated') + else if (testRun.result() === 'terminated') process.stdout.write(colors.magenta('.')); - else if (test.result === 'timedout') + else if (testRun.result() === 'timedout') process.stdout.write(colors.red('T')); } } - _printVerboseTestResult(resultIndex, test, workerId = undefined) { + _printVerboseTestRunResult(resultIndex, testRun) { + const test = testRun.test(); let prefix = `${resultIndex})`; - if (this._runner.parallel() > 1 && workerId !== undefined) - prefix += ' ' + colors.gray(`[worker = ${workerId}]`); - if (test.result === 'ok') { + if (this._runner.parallel() > 1) + prefix += ' ' + colors.gray(`[worker = ${testRun.workerId()}]`); + if (testRun.result() === 'ok') { console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (test.result === 'terminated') { + } else if (testRun.result() === 'terminated') { console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (test.result === 'crashed') { + } else if (testRun.result() === 'crashed') { console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (test.result === 'skipped') { - } else if (test.result === 'markedAsFailing') { + } else if (testRun.result() === 'skipped') { + } else if (testRun.result() === 'markedAsFailing') { console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`); - } else if (test.result === 'timedout') { + } else if (testRun.result() === 'timedout') { console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`); - if (test.output) { + if (testRun.output) { console.log(' Output:'); - for (const line of test.output) + for (const line of testRun.output) console.log(' ' + line); } - } else if (test.result === 'failed') { + } else if (testRun.result() === 'failed') { console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName()} (${formatLocation(test.location())})`); - if (test.error instanceof MatchError) { - let lines = this._filePathToLines.get(test.error.location.filePath); + if (testRun.error() instanceof MatchError) { + let lines = this._filePathToLines.get(testRun.error().location.filePath); if (!lines) { try { - lines = fs.readFileSync(test.error.location.filePath, 'utf8').split('\n'); + lines = fs.readFileSync(testRun.error().location.filePath, 'utf8').split('\n'); } catch (e) { lines = []; } - this._filePathToLines.set(test.error.location.filePath, lines); + this._filePathToLines.set(testRun.error().location.filePath, lines); } - const lineNumber = test.error.location.lineNumber; + const lineNumber = testRun.error().location.lineNumber; if (lineNumber < lines.length) { const lineNumberLength = (lineNumber + 1 + '').length; const FROM = Math.max(test.location().lineNumber - 1, lineNumber - 5); const snippet = lines.slice(FROM, lineNumber).map((line, index) => ` ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n'); - const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(test.error.location.columnNumber - 1) + '^'; + const pointer = ` ` + ' '.repeat(lineNumberLength) + ' ' + '~'.repeat(testRun.error().location.columnNumber - 1) + '^'; console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n'); } - console.log(padLines(test.error.formatter(), 4)); + console.log(padLines(testRun.error().formatter(), 4)); console.log(''); } else { console.log(' Message:'); - let message = '' + (test.error.message || test.error); - if (test.error.stack && message.includes(test.error.stack)) - message = message.substring(0, message.indexOf(test.error.stack)); + let message = '' + (testRun.error().message || testRun.error()); + if (testRun.error().stack && message.includes(testRun.error().stack)) + message = message.substring(0, message.indexOf(testRun.error().stack)); if (message) console.log(` ${colors.red(message)}`); - if (test.error.stack) { + if (testRun.error().stack) { console.log(' Stack:'); - let stack = test.error.stack; + let stack = testRun.error().stack; // Highlight first test location, if any. const match = stack.match(new RegExp(test.location().filePath + ':(\\d+):(\\d+)')); if (match) { @@ -252,9 +237,9 @@ class Reporter { console.log(padLines(stack, 4)); } } - if (test.output) { + if (testRun.output) { console.log(' Output:'); - for (const line of test.output) + for (const line of testRun.output) console.log(' ' + line); } } diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js index bbdf6bbd15..f81e4d640b 100644 --- a/utils/testrunner/TestRunner.js +++ b/utils/testrunner/TestRunner.js @@ -37,12 +37,6 @@ function runUserCallback(callback, timeout, args) { return { promise, terminate }; } -const TestMode = { - Run: 'run', - Skip: 'skip', - Focus: 'focus', -}; - const TestExpectation = { Ok: 'ok', Fail: 'fail', @@ -72,7 +66,8 @@ class Test { this._suite = suite; this._name = name; this._fullName = (suite.fullName() + ' ' + name).trim(); - this._mode = TestMode.Run; + this._skipped = false; + this._focused = false; this._expectation = TestExpectation.Ok; this._body = callback; this._location = location; @@ -80,26 +75,9 @@ class Test { this._repeat = 1; this._hooks = []; - // Test results. TODO: make these private. - this.result = null; - this.error = null; - this.startTimestamp = 0; - this.endTimestamp = 0; - - this.Modes = { ...TestMode }; this.Expectations = { ...TestExpectation }; } - _clone() { - // TODO: introduce TestRun instead? - const test = new Test(this._suite, this._name, this._body, this._location); - test._timeout = this._timeout; - test._mode = this._mode; - test._expectation = this._expectation; - test._hooks = this._hooks.slice(); - return test; - } - suite() { return this._suite; } @@ -120,13 +98,20 @@ class Test { return this._body; } - mode() { - return this._mode; + skipped() { + return this._skipped; } - setMode(mode) { - if (this._mode !== TestMode.Focus) - this._mode = mode; + setSkipped(skipped) { + this._skipped = skipped; + } + + focused() { + return this._focused; + } + + setFocused(focused) { + this._focused = focused; } timeout() { @@ -171,24 +156,16 @@ class Suite { this._parentSuite = parentSuite; this._name = name; this._fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim(); - this._mode = TestMode.Run; + this._skipped = false; + this._focused = false; this._expectation = TestExpectation.Ok; this._location = location; this._repeat = 1; this._hooks = []; - this.Modes = { ...TestMode }; this.Expectations = { ...TestExpectation }; } - _clone() { - // TODO: introduce TestRun instead? - const suite = new Suite(this._parentSuite, this._name, this._location); - suite._mode = this._mode; - suite._expectation = this._expectation; - return suite; - } - parentSuite() { return this._parentSuite; } @@ -201,13 +178,20 @@ class Suite { return this._fullName; } - mode() { - return this._mode; + skipped() { + return this._skipped; } - setMode(mode) { - if (this._mode !== TestMode.Focus) - this._mode = mode; + setSkipped(skipped) { + this._skipped = skipped; + } + + focused() { + return this._focused; + } + + setFocused(focused) { + this._focused = focused; } location() { @@ -251,12 +235,56 @@ class Suite { } } +class TestRun { + constructor(test) { + this._test = test; + this._result = null; + this._error = null; + this._startTimestamp = 0; + this._endTimestamp = 0; + this._workerId = null; + } + + finished() { + return this._result !== null && this._result !== 'running'; + } + + isFailure() { + return this._result === TestResult.Failed || this._result === TestResult.TimedOut || this._result === TestResult.Crashed; + } + + ok() { + return this._result === TestResult.Ok; + } + + result() { + return this._result; + } + + error() { + return this._error; + } + + duration() { + return this._endTimestamp - this._startTimestamp; + } + + test() { + return this._test; + } + + workerId() { + return this._workerId; + } +} + class Result { constructor() { this.result = TestResult.Ok; this.exitCode = 0; this.message = ''; this.errors = []; + this.runs = []; } setResult(result, message) { @@ -275,11 +303,9 @@ class Result { } addError(message, error, worker) { - const data = { message, error, tests: [] }; - if (worker) { - data.workerId = worker._workerId; - data.tests = worker._runTests.slice(); - } + const data = { message, error, runs: [] }; + if (worker) + data.runs = worker._runs.slice(); this.errors.push(data); } @@ -297,7 +323,7 @@ class TestWorker { this._workerId = workerId; this._runningTestTerminate = null; this._runningHookTerminate = null; - this._runTests = []; + this._runs = []; } terminate(terminateHooks) { @@ -308,36 +334,34 @@ class TestWorker { this._runningHookTerminate(); } - _markTerminated(test) { + _markTerminated(testRun) { if (!this._terminating) return false; - test.result = TestResult.Terminated; + testRun._result = TestResult.Terminated; return true; } - async runTest(test) { - this._runTests.push(test); + async run(testRun) { + this._runs.push(testRun); - if (this._markTerminated(test)) - return; - - let skipped = test.mode() === TestMode.Skip; + const test = testRun.test(); + let skipped = test.skipped() && !test.focused(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) - skipped = skipped || (suite.mode() === TestMode.Skip); + skipped = skipped || (suite.skipped() && !suite.focused()); if (skipped) { - await this._testPass._willStartTest(this, test); - test.result = TestResult.Skipped; - await this._testPass._didFinishTest(this, test); + await this._willStartTestRun(testRun); + testRun._result = TestResult.Skipped; + await this._didFinishTestRun(testRun); return; } let expectedToFail = test.expectation() === TestExpectation.Fail; for (let suite = test.suite(); suite; suite = suite.parentSuite()) expectedToFail = expectedToFail || (suite.expectation() === TestExpectation.Fail); - if (expectedToFail && test.mode() !== TestMode.Focus) { - await this._testPass._willStartTest(this, test); - test.result = TestResult.MarkedAsFailing; - await this._testPass._didFinishTest(this, test); + if (expectedToFail && !test.focused()) { + await this._willStartTestRun(testRun); + testRun._result = TestResult.MarkedAsFailing; + await this._didFinishTestRun(testRun); return; } @@ -351,80 +375,80 @@ class TestWorker { common++; while (this._suiteStack.length > common) { - if (this._markTerminated(test)) + if (this._markTerminated(testRun)) return; const suite = this._suiteStack.pop(); for (const hook of suite.hooks('afterAll')) { - if (!await this._runHook(test, hook, suite.fullName())) + if (!await this._runHook(testRun, hook, suite.fullName())) return; } } while (this._suiteStack.length < suiteStack.length) { - if (this._markTerminated(test)) + if (this._markTerminated(testRun)) return; const suite = suiteStack[this._suiteStack.length]; this._suiteStack.push(suite); for (const hook of suite.hooks('beforeAll')) { - if (!await this._runHook(test, hook, suite.fullName())) + if (!await this._runHook(testRun, hook, suite.fullName())) return; } } - if (this._markTerminated(test)) + if (this._markTerminated(testRun)) return; // From this point till the end, we have to run all hooks // no matter what happens. - await this._testPass._willStartTest(this, test); + await this._willStartTestRun(testRun); for (const suite of this._suiteStack) { for (const hook of suite.hooks('beforeEach')) - await this._runHook(test, hook, suite.fullName(), true); + await this._runHook(testRun, hook, suite.fullName(), true); } for (const hook of test.hooks('before')) - await this._runHook(test, hook, test.fullName(), true); + await this._runHook(testRun, hook, test.fullName(), true); - if (!test.error && !this._markTerminated(test)) { - await this._testPass._willStartTestBody(this, test); + if (!testRun._error && !this._markTerminated(testRun)) { + await this._willStartTestBody(testRun); const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, test]); this._runningTestTerminate = terminate; - test.error = await promise; + testRun._error = await promise; this._runningTestTerminate = null; - if (test.error && test.error.stack) - await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(test.error); - if (!test.error) - test.result = TestResult.Ok; - else if (test.error === TimeoutError) - test.result = TestResult.TimedOut; - else if (test.error === TerminatedError) - test.result = TestResult.Terminated; + if (testRun._error && testRun._error.stack) + await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(testRun._error); + if (!testRun._error) + testRun._result = TestResult.Ok; + else if (testRun._error === TimeoutError) + testRun._result = TestResult.TimedOut; + else if (testRun._error === TerminatedError) + testRun._result = TestResult.Terminated; else - test.result = TestResult.Failed; - await this._testPass._didFinishTestBody(this, test); + testRun._result = TestResult.Failed; + await this._didFinishTestBody(testRun); } for (const hook of test.hooks('after')) - await this._runHook(test, hook, test.fullName(), true); + await this._runHook(testRun, hook, test.fullName(), true); for (const suite of this._suiteStack.slice().reverse()) { for (const hook of suite.hooks('afterEach')) - await this._runHook(test, hook, suite.fullName(), true); + await this._runHook(testRun, hook, suite.fullName(), true); } - await this._testPass._didFinishTest(this, test); + await this._didFinishTestRun(testRun); } - async _runHook(test, hook, fullName, passTest = false) { - await this._testPass._willStartHook(this, hook, fullName); + async _runHook(testRun, hook, fullName, passTest = false) { + await this._willStartHook(hook, fullName); const timeout = this._testPass._runner._timeout; - const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, test] : [this._state]); + const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, testRun.test()] : [this._state]); this._runningHookTerminate = terminate; let error = await promise; this._runningHookTerminate = null; if (error) { const locationString = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`; - if (test.result !== TestResult.Terminated) { + if (testRun && testRun._result !== TestResult.Terminated) { // Prefer terminated result over any hook failures. - test.result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed; + testRun._result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed; } let message; if (error === TimeoutError) { @@ -439,20 +463,56 @@ class TestWorker { await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); message = `${locationString} - FAILED while running "${hook.name}" in suite "${fullName}": `; } - await this._testPass._didFailHook(this, hook, fullName, message, error); - test.error = error; + await this._didFailHook(hook, fullName, message, error); + if (testRun) + testRun._error = error; return false; } - await this._testPass._didCompleteHook(this, hook, fullName); + await this._didCompleteHook(hook, fullName); return true; } + async _willStartTestRun(testRun) { + testRun._startTimestamp = Date.now(); + testRun._workerId = this._workerId; + this._testPass._runner.emit(TestRunner.Events.TestStarted, testRun); + } + + async _didFinishTestRun(testRun) { + testRun._endTimestamp = Date.now(); + testRun._workerId = this._workerId; + this._testPass._runner.emit(TestRunner.Events.TestFinished, testRun); + } + + async _willStartTestBody(testRun) { + debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location().fileName + ':' + testRun.test().location().lineNumber})`); + } + + async _didFinishTestBody(testRun) { + debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location().fileName + ':' + testRun.test().location().lineNumber})`); + } + + async _willStartHook(hook, fullName) { + debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" started for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); + } + + async _didFailHook(hook, fullName, message, error) { + debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); + if (message) + this._testPass._result.addError(message, error, this); + this._testPass._result.setResult(TestResult.Crashed, message); + } + + async _didCompleteHook(hook, fullName) { + debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); + } + async shutdown() { while (this._suiteStack.length > 0) { const suite = this._suiteStack.pop(); for (const hook of suite.hooks('afterAll')) - await this._runHook({}, hook, suite.fullName()); + await this._runHook(null, hook, suite.fullName()); } } } @@ -469,7 +529,7 @@ class TestPass { this._terminating = false; } - async run(testList) { + async run(testRuns) { const terminations = [ createTermination.call(this, 'SIGINT', TestResult.Terminated, 'SIGINT received'), createTermination.call(this, 'SIGHUP', TestResult.Terminated, 'SIGHUP received'), @@ -480,24 +540,21 @@ class TestPass { for (const termination of terminations) process.on(termination.event, termination.handler); - for (const test of testList) { - test.result = null; - test.error = null; - } this._result = new Result(); + this._result.runs = testRuns; - const parallel = Math.min(this._parallel, testList.length); + const parallel = Math.min(this._parallel, testRuns.length); const workerPromises = []; for (let i = 0; i < parallel; ++i) { - const initialTestIndex = i * Math.floor(testList.length / parallel); - workerPromises.push(this._runWorker(initialTestIndex, testList, i)); + const initialTestRunIndex = i * Math.floor(testRuns.length / parallel); + workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i)); } await Promise.all(workerPromises); for (const termination of terminations) process.removeListener(termination.event, termination.handler); - if (this._runner.failedTests().length) + if (testRuns.some(run => run.isFailure())) this._result.setResult(TestResult.Failed, ''); return this._result; @@ -510,25 +567,25 @@ class TestPass { } } - async _runWorker(testIndex, testList, parallelIndex) { + async _runWorker(testRunIndex, testRuns, parallelIndex) { let worker = new TestWorker(this, this._nextWorkerId++, parallelIndex); this._workers[parallelIndex] = worker; while (!worker._terminating) { let skipped = 0; - while (skipped < testList.length && testList[testIndex].result !== null) { - testIndex = (testIndex + 1) % testList.length; + while (skipped < testRuns.length && testRuns[testRunIndex]._result !== null) { + testRunIndex = (testRunIndex + 1) % testRuns.length; skipped++; } - const test = testList[testIndex]; - if (test.result !== null) { + const testRun = testRuns[testRunIndex]; + if (testRun._result !== null) { // All tests have been run. break; } // Mark as running so that other workers do not run it again. - test.result = 'running'; - await worker.runTest(test); - if (isTestFailure(test.result)) { + testRun._result = 'running'; + await worker.run(testRun); + if (testRun.isFailure()) { // Something went wrong during test run, let's use a fresh worker. await worker.shutdown(); if (this._breakOnFailure) { @@ -556,39 +613,6 @@ class TestPass { this._result.addError(message, error, this._workers.length === 1 ? this._workers[0] : null); } } - - async _willStartTest(worker, test) { - test.startTimestamp = Date.now(); - this._runner.emit(TestRunner.Events.TestStarted, test, worker._workerId); - } - - async _didFinishTest(worker, test) { - test.endTimestamp = Date.now(); - this._runner.emit(TestRunner.Events.TestFinished, test, worker._workerId); - } - - async _willStartTestBody(worker, test) { - debug('testrunner:test')(`[${worker._workerId}] starting "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`); - } - - async _didFinishTestBody(worker, test) { - debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`); - } - - async _willStartHook(worker, hook, fullName) { - debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" started for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); - } - - async _didFailHook(worker, hook, fullName, message, error) { - debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); - if (message) - this._result.addError(message, error, worker); - this._result.setResult(TestResult.Crashed, message); - } - - async _didCompleteHook(worker, hook, fullName) { - debug('testrunner:hook')(`[${worker._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); - } } class TestRunner extends EventEmitter { @@ -625,10 +649,10 @@ class TestRunner extends EventEmitter { this.it = this._testBuilder([]); if (installCommonHelpers) { - this.fdescribe = this.describe.setup(s => s.setMode(s.Modes.Focus)); - this.xdescribe = this.describe.setup(s => s.setMode(s.Modes.Skip)); - this.fit = this.it.setup(t => t.setMode(t.Modes.Focus)); - this.xit = this.it.setup(t => t.setMode(t.Modes.Skip)); + this.fdescribe = this.describe.setup(s => s.setFocused(true)); + this.xdescribe = this.describe.setup(s => s.setSkipped(true)); + this.fit = this.it.setup(t => t.setFocused(true)); + this.xit = this.it.setup(t => t.setSkipped(true)); } } @@ -638,12 +662,10 @@ class TestRunner extends EventEmitter { const suite = new Suite(this._currentSuite, name, location); for (const { callback, args } of callbacks) callback(suite, ...args); - for (let i = 0; i < suite.repeat(); i++) { - this._currentSuite = suite._clone(); - callback(...suiteArgs); - this._suites.push(this._currentSuite); - this._currentSuite = this._currentSuite.parentSuite(); - } + this._currentSuite = suite; + callback(...suiteArgs); + this._suites.push(suite); + this._currentSuite = suite.parentSuite(); }, { get: (obj, prop) => { if (prop === 'setup') @@ -664,8 +686,7 @@ class TestRunner extends EventEmitter { test.setTimeout(this._timeout); for (const { callback, args } of callbacks) callback(test, ...args); - for (let i = 0; i < test.repeat(); i++) - this._tests.push(test._clone()); + this._tests.push(test); }, { get: (obj, prop) => { if (prop === 'setup') @@ -697,11 +718,19 @@ class TestRunner extends EventEmitter { async run(options = {}) { const { totalTimeout = 0 } = options; - const runnableTests = this.runnableTests(); - this.emit(TestRunner.Events.Started, runnableTests); + const testRuns = []; + for (const test of this._testsToRun()) { + let repeat = test.repeat(); + for (let suite = test.suite(); suite; suite = suite.parentSuite()) + repeat *= suite.repeat(); + for (let i = 0; i < repeat; i++) + testRuns.push(new TestRun(test)); + } + this.emit(TestRunner.Events.Started, testRuns); - let result = new Result(); + let result; if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) { + result = new Result(); result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI'); } else { this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure); @@ -712,7 +741,7 @@ class TestRunner extends EventEmitter { }, totalTimeout); } try { - result = await this._runningPass.run(runnableTests).catch(e => { console.error(e); throw e; }); + result = await this._runningPass.run(testRuns).catch(e => { console.error(e); throw e; }); } finally { this._runningPass = null; clearTimeout(timeoutId); @@ -722,13 +751,13 @@ class TestRunner extends EventEmitter { return result; } - runnableTests() { + _testsToRun() { if (!this.hasFocusedTestsOrSuites()) - return this._tests.slice(); + return this._tests; const notFocusedSuites = new Set(); // Mark parent suites of focused tests as not focused. for (const test of this._tests) { - if (test.mode() === TestMode.Focus) { + if (test.focused()) { for (let suite = test.suite(); suite; suite = suite.parentSuite()) notFocusedSuites.add(suite); } @@ -736,9 +765,9 @@ class TestRunner extends EventEmitter { // Pick all tests that are focused or belong to focused suites. const tests = []; for (const test of this._tests) { - let focused = test.mode() === TestMode.Focus; + let focused = test.focused(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) - focused = focused || (suite.mode() === TestMode.Focus && !notFocusedSuites.has(suite)); + focused = focused || (suite.focused() && !notFocusedSuites.has(suite)); if (focused) tests.push(test); } @@ -755,22 +784,14 @@ class TestRunner extends EventEmitter { return this._timeout; } - focusedSuites() { - return this._suites.filter(suite => suite.mode() === TestMode.Focus); - } - - focusedTests() { - return this._tests.filter(test => test.mode() === TestMode.Focus); - } - hasFocusedTestsOrSuites() { - return !!this.focusedTests().length || !!this.focusedSuites().length; + return this._tests.some(test => test.focused()) || this._suites.some(suite => suite.focused()); } focusMatchingTests(fullNameRegex) { for (const test of this._tests) { if (fullNameRegex.test(test.fullName())) - test.setMode(TestMode.Focus); + test.setFocused(true); } } @@ -778,20 +799,8 @@ class TestRunner extends EventEmitter { return this._tests.slice(); } - failedTests() { - return this._tests.filter(test => isTestFailure(test.result)); - } - - passedTests() { - return this._tests.filter(test => test.result === TestResult.Ok); - } - - skippedTests() { - return this._tests.filter(test => test.result === TestResult.Skipped); - } - - markedAsFailingTests() { - return this._tests.filter(test => test.result === TestResult.MarkedAsFailing); + suites() { + return this._suites.slice(); } parallel() { diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js index a76644dfbe..e1aff1073a 100644 --- a/utils/testrunner/test/testrunner.spec.js +++ b/utils/testrunner/test/testrunner.spec.js @@ -19,12 +19,21 @@ module.exports.addTests = function({testRunner, expect}) { const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('uno'); - expect(test.mode()).toBe('run'); + expect(test.focused()).toBe(false); + expect(test.skipped()).toBe(false); expect(test.location().filePath).toEqual(__filename); expect(test.location().fileName).toEqual('testrunner.spec.js'); expect(test.location().lineNumber).toBeTruthy(); expect(test.location().columnNumber).toBeTruthy(); }); + it('should run a test', async() => { + const t = newTestRunner(); + t.it('uno', () => {}); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('ok'); + }); }); describe('TestRunner.xit', () => { @@ -35,8 +44,16 @@ module.exports.addTests = function({testRunner, expect}) { const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('uno'); - expect(test.mode()).toBe('skip'); - expect(t.runnableTests()).toEqual([test]); + expect(test.focused()).toBe(false); + expect(test.skipped()).toBe(true); + }); + it('should not run a skipped test', async() => { + const t = newTestRunner(); + t.xit('uno', () => {}); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('skipped'); }); }); @@ -48,8 +65,16 @@ module.exports.addTests = function({testRunner, expect}) { const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('uno'); - expect(test.mode()).toBe('focus'); - expect(t.runnableTests()).toEqual([test]); + expect(test.focused()).toBe(true); + expect(test.skipped()).toBe(false); + }); + it('should run a focused test', async() => { + const t = newTestRunner(); + t.fit('uno', () => {}); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('ok'); }); it('should run a failed focused test', async() => { const t = newTestRunner(); @@ -57,10 +82,11 @@ module.exports.addTests = function({testRunner, expect}) { t.fit.setup(t => t.setExpectation(t.Expectations.Fail))('uno', () => { run = true; throw new Error('failure'); }); - expect(t.tests().length).toBe(1); - await t.run(); + const result = await t.run(); expect(run).toBe(true); - expect(t.failedTests()[0].name()).toBe('uno'); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('failed'); }); }); @@ -74,11 +100,12 @@ module.exports.addTests = function({testRunner, expect}) { const test = t.tests()[0]; expect(test.name()).toBe('uno'); expect(test.fullName()).toBe('suite uno'); - expect(test.mode()).toBe('run'); + expect(test.focused()).toBe(false); + expect(test.skipped()).toBe(false); expect(test.suite().name()).toBe('suite'); expect(test.suite().fullName()).toBe('suite'); - expect(test.suite().mode()).toBe('run'); - expect(t.runnableTests()).toEqual([test]); + expect(test.suite().focused()).toBe(false); + expect(test.suite().skipped()).toBe(false); }); }); @@ -90,20 +117,22 @@ module.exports.addTests = function({testRunner, expect}) { }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.mode()).toBe('run'); - expect(test.suite().mode()).toBe('skip'); - expect(t.runnableTests()).toEqual([test]); + expect(test.focused()).toBe(false); + expect(test.skipped()).toBe(false); + expect(test.suite().focused()).toBe(false); + expect(test.suite().skipped()).toBe(true); }); it('focused tests inside a skipped suite are not run', async() => { const t = newTestRunner(); + let run = false; t.xdescribe('suite', () => { - t.fit('uno', () => {}); + t.fit('uno', () => { run = true; }); }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.mode()).toBe('focus'); - expect(test.suite().mode()).toBe('skip'); - expect(t.runnableTests()).toEqual([test]); + const result = await t.run(); + expect(run).toBe(false); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('skipped'); }); }); @@ -115,20 +144,20 @@ module.exports.addTests = function({testRunner, expect}) { }); expect(t.tests().length).toBe(1); const test = t.tests()[0]; - expect(test.mode()).toBe('run'); - expect(test.suite().mode()).toBe('focus'); - expect(t.runnableTests()).toEqual([test]); + expect(test.focused()).toBe(false); + expect(test.skipped()).toBe(false); + expect(test.suite().focused()).toBe(true); + expect(test.suite().skipped()).toBe(false); }); it('skipped tests inside a focused suite should not be run', async() => { const t = newTestRunner(); t.fdescribe('suite', () => { t.xit('uno', () => {}); }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.mode()).toBe('skip'); - expect(test.suite().mode()).toBe('focus'); - expect(t.runnableTests()).toEqual([test]); + const result = await t.run(); + expect(result.runs.length).toBe(1); + expect(result.runs[0].test()).toBe(t.tests()[0]); + expect(result.runs[0].result()).toBe('skipped'); }); it('should run all "run" tests inside a focused suite', async() => { const log = []; @@ -176,10 +205,8 @@ module.exports.addTests = function({testRunner, expect}) { t.testModifier('foo', (t, ...args) => { log.push('foo'); - expect(t.Modes.Run).toBeTruthy(); - expect(t.Modes.Skip).toBeTruthy(); - expect(t.Modes.Focus).toBeTruthy(); - expect(t.mode()).toBe(t.Modes.Run); + expect(t.focused()).toBe(false); + expect(t.skipped()).toBe(false); expect(t.Expectations.Ok).toBeTruthy(); expect(t.Expectations.Fail).toBeTruthy(); expect(t.expectation()).toBe(t.Expectations.Ok); @@ -190,7 +217,7 @@ module.exports.addTests = function({testRunner, expect}) { expect(args[0]).toBe('uno'); expect(args[1]).toBe('dos'); - t.setMode(t.Modes.Focus); + t.setFocused(true); t.setExpectation(t.Expectations.Fail); t.setTimeout(234); t.setRepeat(42); @@ -198,9 +225,9 @@ module.exports.addTests = function({testRunner, expect}) { t.testAttribute('bar', t => { log.push('bar'); - expect(t.mode()).toBe(t.Modes.Focus); - t.setMode(t.Modes.Skip); - expect(t.mode()).toBe(t.Modes.Focus); + t.setSkipped(true); + expect(t.focused()).toBe(true); + expect(t.skipped()).toBe(true); expect(t.expectation()).toBe(t.Expectations.Fail); expect(t.timeout()).toBe(234); expect(t.repeat()).toBe(42); @@ -323,15 +350,15 @@ module.exports.addTests = function({testRunner, expect}) { const t = newTestRunner({timeout: 10000}); t.afterEach(() => { throw new Error('crash!'); }); t.it('uno', () => { t.terminate(); }); - await t.run(); - expect(t.tests()[0].result).toBe('terminated'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('terminated'); }); it('should report as terminated when terminated during hook', async() => { const t = newTestRunner({timeout: 10000}); t.afterEach(() => { t.terminate(); }); t.it('uno', () => { }); - await t.run(); - expect(t.tests()[0].result).toBe('terminated'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('terminated'); }); it('should unwind hooks properly when crashed', async() => { const log = []; @@ -389,8 +416,8 @@ module.exports.addTests = function({testRunner, expect}) { t.it.repeat(3)('uno', () => test++); }); await t.run(); - expect(suite).toBe(2); - expect(beforeAll).toBe(2); + expect(suite).toBe(1); + expect(beforeAll).toBe(1); expect(beforeEach).toBe(6); expect(test).toBe(6); }); @@ -513,7 +540,7 @@ module.exports.addTests = function({testRunner, expect}) { const t = newTestRunner({timeout: 10000}); t.it('uno', async () => { await new Promise(() => {}); }); const result = await t.run({totalTimeout: 1}); - expect(t.tests()[0].result).toBe('terminated'); + expect(result.runs[0].result()).toBe('terminated'); expect(result.message).toContain('Total timeout'); }); }); @@ -603,92 +630,64 @@ module.exports.addTests = function({testRunner, expect}) { }); }); - describe('TestRunner.passedTests', () => { - it('should work', async() => { - const t = newTestRunner(); - t.it('uno', () => {}); - await t.run(); - expect(t.failedTests().length).toBe(0); - expect(t.skippedTests().length).toBe(0); - expect(t.passedTests().length).toBe(1); - const [test] = t.passedTests(); - expect(test.result).toBe('ok'); - }); - }); - - describe('TestRunner.failedTests', () => { + describe('TestRunner result', () => { it('should work for both throwing and timeouting tests', async() => { const t = newTestRunner({timeout: 1}); t.it('uno', () => { throw new Error('boo');}); t.it('dos', () => new Promise(() => {})); - await t.run(); - expect(t.skippedTests().length).toBe(0); - expect(t.passedTests().length).toBe(0); - expect(t.failedTests().length).toBe(2); - const [test1, test2] = t.failedTests(); - expect(test1.result).toBe('failed'); - expect(test2.result).toBe('timedout'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('failed'); + expect(result.runs[1].result()).toBe('timedout'); }); it('should report crashed tests', async() => { const t = newTestRunner(); t.beforeEach(() => { throw new Error('woof');}); t.it('uno', () => {}); - await t.run(); - expect(t.failedTests().length).toBe(1); - expect(t.failedTests()[0].result).toBe('crashed'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('crashed'); }); - }); - - describe('TestRunner.skippedTests', () => { - it('should work for both throwing and timeouting tests', async() => { + it('skipped should work for both throwing and timeouting tests', async() => { const t = newTestRunner({timeout: 1}); t.xit('uno', () => { throw new Error('boo');}); - await t.run(); - expect(t.skippedTests().length).toBe(1); - expect(t.passedTests().length).toBe(0); - expect(t.failedTests().length).toBe(0); - const [test] = t.skippedTests(); - expect(test.result).toBe('skipped'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('skipped'); }); - }); - - describe('Test.result', () => { it('should return OK', async() => { const t = newTestRunner(); t.it('uno', () => {}); - await t.run(); - expect(t.tests()[0].result).toBe('ok'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('ok'); }); it('should return TIMEDOUT', async() => { const t = newTestRunner({timeout: 1}); t.it('uno', async() => new Promise(() => {})); - await t.run(); - expect(t.tests()[0].result).toBe('timedout'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('timedout'); }); it('should return SKIPPED', async() => { const t = newTestRunner(); t.xit('uno', () => {}); - await t.run(); - expect(t.tests()[0].result).toBe('skipped'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('skipped'); }); it('should return FAILED', async() => { const t = newTestRunner(); t.it('uno', async() => Promise.reject('woof')); - await t.run(); - expect(t.tests()[0].result).toBe('failed'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('failed'); }); it('should return TERMINATED', async() => { const t = newTestRunner(); t.it('uno', async() => t.terminate()); - await t.run(); - expect(t.tests()[0].result).toBe('terminated'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('terminated'); }); it('should return CRASHED', async() => { const t = newTestRunner(); t.it('uno', () => {}); t.afterEach(() => {throw new Error('foo');}); - await t.run(); - expect(t.tests()[0].result).toBe('crashed'); + const result = await t.run(); + expect(result.runs[0].result()).toBe('crashed'); }); });