/** * Copyright 2017 Google Inc. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ const EventEmitter = require('events'); const {SourceMapSupport} = require('./SourceMapSupport'); const debug = require('debug'); const {getCallerLocation} = require('./utils'); const INFINITE_TIMEOUT = 100000000; const TimeoutError = new Error('Timeout'); const TerminatedError = new Error('Terminated'); function runUserCallback(callback, timeout, args) { let terminateCallback; let timeoutId; const promise = Promise.race([ Promise.resolve().then(callback.bind(null, ...args)).then(() => null).catch(e => e), new Promise(resolve => { timeoutId = setTimeout(resolve.bind(null, TimeoutError), timeout); }), new Promise(resolve => terminateCallback = resolve), ]).catch(e => e).finally(() => clearTimeout(timeoutId)); const terminate = () => terminateCallback(TerminatedError); return { promise, terminate }; } const TestExpectation = { Ok: 'ok', Fail: 'fail', }; const TestResult = { Ok: 'ok', MarkedAsFailing: 'markedAsFailing', // User marked as failed Skipped: 'skipped', // User marked as skipped Failed: 'failed', // Exception happened during running TimedOut: 'timedout', // Timeout Exceeded while running Terminated: 'terminated', // Execution terminated Crashed: 'crashed', // If testrunner crashed due to this test }; function createHook(callback, name) { const location = getCallerLocation(__filename); return { name, body: callback, location }; } class Test { constructor(suite, name, callback, location) { this._suite = suite; this._name = name; this._fullName = (suite.fullName() + ' ' + name).trim(); this._skipped = false; this._focused = false; this._expectation = TestExpectation.Ok; this._body = callback; this._location = location; this._timeout = INFINITE_TIMEOUT; this._repeat = 1; this._hooks = []; this.Expectations = { ...TestExpectation }; } suite() { return this._suite; } name() { return this._name; } fullName() { return this._fullName; } location() { return this._location; } body() { return this._body; } skipped() { return this._skipped; } setSkipped(skipped) { this._skipped = skipped; return this; } focused() { return this._focused; } setFocused(focused) { this._focused = focused; return this; } timeout() { return this._timeout; } setTimeout(timeout) { this._timeout = timeout; return this; } expectation() { return this._expectation; } setExpectation(expectation) { this._expectation = expectation; return this; } repeat() { return this._repeat; } setRepeat(repeat) { this._repeat = repeat; return this; } before(callback) { this._hooks.push(createHook(callback, 'before')); return this; } after(callback) { this._hooks.push(createHook(callback, 'after')); return this; } hooks(name) { return this._hooks.filter(hook => !name || hook.name === name); } } class Suite { constructor(parentSuite, name, location) { this._parentSuite = parentSuite; this._name = name; this._fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim(); this._skipped = false; this._focused = false; this._expectation = TestExpectation.Ok; this._location = location; this._repeat = 1; this._hooks = []; this.Expectations = { ...TestExpectation }; } parentSuite() { return this._parentSuite; } name() { return this._name; } fullName() { return this._fullName; } skipped() { return this._skipped; } setSkipped(skipped) { this._skipped = skipped; return this; } focused() { return this._focused; } setFocused(focused) { this._focused = focused; return this; } location() { return this._location; } expectation() { return this._expectation; } setExpectation(expectation) { this._expectation = expectation; return this; } repeat() { return this._repeat; } setRepeat(repeat) { this._repeat = repeat; return this; } beforeEach(callback) { this._hooks.push(createHook(callback, 'beforeEach')); return this; } afterEach(callback) { this._hooks.push(createHook(callback, 'afterEach')); return this; } beforeAll(callback) { this._hooks.push(createHook(callback, 'beforeAll')); return this; } afterAll(callback) { this._hooks.push(createHook(callback, 'afterAll')); return this; } hooks(name) { return this._hooks.filter(hook => !name || hook.name === name); } } class TestRun { constructor(test) { this._test = test; this._result = null; this._error = null; this._startTimestamp = 0; this._endTimestamp = 0; this._workerId = null; } finished() { return this._result !== null && this._result !== 'running'; } isFailure() { return this._result === TestResult.Failed || this._result === TestResult.TimedOut || this._result === TestResult.Crashed; } ok() { return this._result === TestResult.Ok; } result() { return this._result; } error() { return this._error; } duration() { return this._endTimestamp - this._startTimestamp; } test() { return this._test; } workerId() { return this._workerId; } } class Result { constructor() { this.result = TestResult.Ok; this.exitCode = 0; this.message = ''; this.errors = []; this.runs = []; } setResult(result, message) { if (!this.ok()) return; this.result = result; this.message = message || ''; if (result === TestResult.Ok) this.exitCode = 0; else if (result === TestResult.Terminated) this.exitCode = 130; else if (result === TestResult.Crashed) this.exitCode = 2; else this.exitCode = 1; } addError(message, error, worker) { const data = { message, error, runs: [] }; if (worker) data.runs = worker._runs.slice(); this.errors.push(data); } ok() { return this.result === TestResult.Ok; } } class TestWorker { constructor(testPass, workerId, parallelIndex) { this._testPass = testPass; this._state = { parallelIndex }; this._suiteStack = []; this._terminating = false; this._workerId = workerId; this._runningTestTerminate = null; this._runningHookTerminate = null; this._runs = []; } terminate(terminateHooks) { this._terminating = true; if (this._runningTestTerminate) this._runningTestTerminate(); if (terminateHooks && this._runningHookTerminate) this._runningHookTerminate(); } _markTerminated(testRun) { if (!this._terminating) return false; testRun._result = TestResult.Terminated; return true; } async run(testRun) { this._runs.push(testRun); const test = testRun.test(); let skipped = test.skipped() && !test.focused(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) skipped = skipped || (suite.skipped() && !suite.focused()); if (skipped) { await this._willStartTestRun(testRun); testRun._result = TestResult.Skipped; await this._didFinishTestRun(testRun); return; } let expectedToFail = test.expectation() === TestExpectation.Fail; for (let suite = test.suite(); suite; suite = suite.parentSuite()) expectedToFail = expectedToFail || (suite.expectation() === TestExpectation.Fail); if (expectedToFail && !test.focused()) { await this._willStartTestRun(testRun); testRun._result = TestResult.MarkedAsFailing; await this._didFinishTestRun(testRun); return; } const suiteStack = []; for (let suite = test.suite(); suite; suite = suite.parentSuite()) suiteStack.push(suite); suiteStack.reverse(); let common = 0; while (common < suiteStack.length && this._suiteStack[common] === suiteStack[common]) common++; while (this._suiteStack.length > common) { if (this._markTerminated(testRun)) return; const suite = this._suiteStack.pop(); for (const hook of suite.hooks('afterAll')) { if (!await this._runHook(testRun, hook, suite.fullName())) return; } } while (this._suiteStack.length < suiteStack.length) { if (this._markTerminated(testRun)) return; const suite = suiteStack[this._suiteStack.length]; this._suiteStack.push(suite); for (const hook of suite.hooks('beforeAll')) { if (!await this._runHook(testRun, hook, suite.fullName())) return; } } if (this._markTerminated(testRun)) return; // From this point till the end, we have to run all hooks // no matter what happens. await this._willStartTestRun(testRun); for (const suite of this._suiteStack) { for (const hook of suite.hooks('beforeEach')) await this._runHook(testRun, hook, suite.fullName(), true); } for (const hook of test.hooks('before')) await this._runHook(testRun, hook, test.fullName(), true); if (!testRun._error && !this._markTerminated(testRun)) { await this._willStartTestBody(testRun); const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, test]); this._runningTestTerminate = terminate; testRun._error = await promise; this._runningTestTerminate = null; if (testRun._error && testRun._error.stack) await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(testRun._error); if (!testRun._error) testRun._result = TestResult.Ok; else if (testRun._error === TimeoutError) testRun._result = TestResult.TimedOut; else if (testRun._error === TerminatedError) testRun._result = TestResult.Terminated; else testRun._result = TestResult.Failed; await this._didFinishTestBody(testRun); } for (const hook of test.hooks('after')) await this._runHook(testRun, hook, test.fullName(), true); for (const suite of this._suiteStack.slice().reverse()) { for (const hook of suite.hooks('afterEach')) await this._runHook(testRun, hook, suite.fullName(), true); } await this._didFinishTestRun(testRun); } async _runHook(testRun, hook, fullName, passTest = false) { await this._willStartHook(hook, fullName); const timeout = this._testPass._runner._timeout; const { promise, terminate } = runUserCallback(hook.body, timeout, passTest ? [this._state, testRun.test()] : [this._state]); this._runningHookTerminate = terminate; let error = await promise; this._runningHookTerminate = null; if (error) { const locationString = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`; if (testRun && testRun._result !== TestResult.Terminated) { // Prefer terminated result over any hook failures. testRun._result = error === TerminatedError ? TestResult.Terminated : TestResult.Crashed; } let message; if (error === TimeoutError) { message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hook.name}" in "${fullName}"`; error = null; } else if (error === TerminatedError) { // Do not report termination details - it's just noise. message = ''; error = null; } else { if (error.stack) await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); message = `${locationString} - FAILED while running "${hook.name}" in suite "${fullName}": `; } await this._didFailHook(hook, fullName, message, error); if (testRun) testRun._error = error; return false; } await this._didCompleteHook(hook, fullName); return true; } async _willStartTestRun(testRun) { testRun._startTimestamp = Date.now(); testRun._workerId = this._workerId; this._testPass._runner.emit(TestRunner.Events.TestStarted, testRun); } async _didFinishTestRun(testRun) { testRun._endTimestamp = Date.now(); testRun._workerId = this._workerId; this._testPass._runner.emit(TestRunner.Events.TestFinished, testRun); } async _willStartTestBody(testRun) { debug('testrunner:test')(`[${this._workerId}] starting "${testRun.test().fullName()}" (${testRun.test().location().fileName + ':' + testRun.test().location().lineNumber})`); } async _didFinishTestBody(testRun) { debug('testrunner:test')(`[${this._workerId}] ${testRun._result.toUpperCase()} "${testRun.test().fullName()}" (${testRun.test().location().fileName + ':' + testRun.test().location().lineNumber})`); } async _willStartHook(hook, fullName) { debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" started for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); } async _didFailHook(hook, fullName, message, error) { debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" FAILED for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); if (message) this._testPass._result.addError(message, error, this); this._testPass._result.setResult(TestResult.Crashed, message); } async _didCompleteHook(hook, fullName) { debug('testrunner:hook')(`[${this._workerId}] "${hook.name}" OK for "${fullName}" (${hook.location.fileName + ':' + hook.location.lineNumber})`); } async shutdown() { while (this._suiteStack.length > 0) { const suite = this._suiteStack.pop(); for (const hook of suite.hooks('afterAll')) await this._runHook(null, hook, suite.fullName()); } } } class TestPass { constructor(runner, parallel, breakOnFailure) { this._runner = runner; this._workers = []; this._nextWorkerId = 1; this._parallel = parallel; this._breakOnFailure = breakOnFailure; this._errors = []; this._result = new Result(); this._terminating = false; } async run(testRuns) { const terminations = [ createTermination.call(this, 'SIGINT', TestResult.Terminated, 'SIGINT received'), createTermination.call(this, 'SIGHUP', TestResult.Terminated, 'SIGHUP received'), createTermination.call(this, 'SIGTERM', TestResult.Terminated, 'SIGTERM received'), createTermination.call(this, 'unhandledRejection', TestResult.Crashed, 'UNHANDLED PROMISE REJECTION'), createTermination.call(this, 'uncaughtException', TestResult.Crashed, 'UNHANDLED ERROR'), ]; for (const termination of terminations) process.on(termination.event, termination.handler); this._result = new Result(); this._result.runs = testRuns; const parallel = Math.min(this._parallel, testRuns.length); const workerPromises = []; for (let i = 0; i < parallel; ++i) { const initialTestRunIndex = i * Math.floor(testRuns.length / parallel); workerPromises.push(this._runWorker(initialTestRunIndex, testRuns, i)); } await Promise.all(workerPromises); for (const termination of terminations) process.removeListener(termination.event, termination.handler); if (testRuns.some(run => run.isFailure())) this._result.setResult(TestResult.Failed, ''); return this._result; function createTermination(event, result, message) { return { event, message, handler: error => this._terminate(result, message, event === 'SIGTERM', event.startsWith('SIG') ? null : error) }; } } async _runWorker(testRunIndex, testRuns, parallelIndex) { let worker = new TestWorker(this, this._nextWorkerId++, parallelIndex); this._workers[parallelIndex] = worker; while (!worker._terminating) { let skipped = 0; while (skipped < testRuns.length && testRuns[testRunIndex]._result !== null) { testRunIndex = (testRunIndex + 1) % testRuns.length; skipped++; } const testRun = testRuns[testRunIndex]; if (testRun._result !== null) { // All tests have been run. break; } // Mark as running so that other workers do not run it again. testRun._result = 'running'; await worker.run(testRun); if (testRun.isFailure()) { // Something went wrong during test run, let's use a fresh worker. await worker.shutdown(); if (this._breakOnFailure) { const message = `Terminating because a test has failed and |testRunner.breakOnFailure| is enabled`; await this._terminate(TestResult.Terminated, message, false /* force */, null /* error */); return; } worker = new TestWorker(this, this._nextWorkerId++, parallelIndex); this._workers[parallelIndex] = worker; } } await worker.shutdown(); } async _terminate(result, message, force, error) { debug('testrunner')(`TERMINATED result = ${result}, message = ${message}`); for (const worker of this._workers) worker.terminate(force /* terminateHooks */); this._result.setResult(result, message); if (this._result.message === 'SIGINT received' && message === 'SIGTERM received') this._result.message = message; if (error) { if (error.stack) await this._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error); this._result.addError(message, error, this._workers.length === 1 ? this._workers[0] : null); } } } class TestRunner extends EventEmitter { constructor(options = {}) { super(); const { timeout = 10 * 1000, // Default timeout is 10 seconds. parallel = 1, breakOnFailure = false, crashIfTestsAreFocusedOnCI = true, installCommonHelpers = true, } = options; this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI; this._sourceMapSupport = new SourceMapSupport(); const dummyLocation = { fileName: '', filePath: '', lineNumber: 0, columnNumber: 0 }; this._rootSuite = new Suite(null, '', dummyLocation); this._currentSuite = this._rootSuite; this._tests = []; this._suites = []; this._timeout = timeout === 0 ? INFINITE_TIMEOUT : timeout; this._parallel = parallel; this._breakOnFailure = breakOnFailure; this._suiteModifiers = new Map(); this._suiteAttributes = new Map(); this._testModifiers = new Map(); this._testAttributes = new Map(); this.beforeAll = (callback) => this._currentSuite.beforeAll(callback); this.beforeEach = (callback) => this._currentSuite.beforeEach(callback); this.afterAll = (callback) => this._currentSuite.afterAll(callback); this.afterEach = (callback) => this._currentSuite.afterEach(callback); this.describe = this._suiteBuilder([]); this.it = this._testBuilder([]); this.Expectations = { ...TestExpectation }; if (installCommonHelpers) { this.fdescribe = this._suiteBuilder([{ callback: s => s.setFocused(true), args: [] }]); this.xdescribe = this._suiteBuilder([{ callback: s => s.setSkipped(true), args: [] }]); this.fit = this._testBuilder([{ callback: t => t.setFocused(true), args: [] }]); this.xit = this._testBuilder([{ callback: t => t.setSkipped(true), args: [] }]); } } _suiteBuilder(callbacks) { return new Proxy((name, callback, ...suiteArgs) => { const location = getCallerLocation(__filename); const suite = new Suite(this._currentSuite, name, location); for (const { callback, args } of callbacks) callback(suite, ...args); this._currentSuite = suite; callback(...suiteArgs); this._suites.push(suite); this._currentSuite = suite.parentSuite(); return suite; }, { get: (obj, prop) => { if (this._suiteModifiers.has(prop)) return (...args) => this._suiteBuilder([...callbacks, { callback: this._suiteModifiers.get(prop), args }]); if (this._suiteAttributes.has(prop)) return this._suiteBuilder([...callbacks, { callback: this._suiteAttributes.get(prop), args: [] }]); return obj[prop]; }, }); } _testBuilder(callbacks) { return new Proxy((name, callback) => { const location = getCallerLocation(__filename); const test = new Test(this._currentSuite, name, callback, location); test.setTimeout(this._timeout); for (const { callback, args } of callbacks) callback(test, ...args); this._tests.push(test); return test; }, { get: (obj, prop) => { if (this._testModifiers.has(prop)) return (...args) => this._testBuilder([...callbacks, { callback: this._testModifiers.get(prop), args }]); if (this._testAttributes.has(prop)) return this._testBuilder([...callbacks, { callback: this._testAttributes.get(prop), args: [] }]); return obj[prop]; }, }); } testModifier(name, callback) { this._testModifiers.set(name, callback); } testAttribute(name, callback) { this._testAttributes.set(name, callback); } suiteModifier(name, callback) { this._suiteModifiers.set(name, callback); } suiteAttribute(name, callback) { this._suiteAttributes.set(name, callback); } async run(options = {}) { const { totalTimeout = 0 } = options; const testRuns = []; for (const test of this._testsToRun()) { let repeat = test.repeat(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) repeat *= suite.repeat(); for (let i = 0; i < repeat; i++) testRuns.push(new TestRun(test)); } this.emit(TestRunner.Events.Started, testRuns); let result; if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) { result = new Result(); result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI'); } else { this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure); let timeoutId; if (totalTimeout) { timeoutId = setTimeout(() => { this._runningPass._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */); }, totalTimeout); } try { result = await this._runningPass.run(testRuns).catch(e => { console.error(e); throw e; }); } finally { this._runningPass = null; clearTimeout(timeoutId); } } this.emit(TestRunner.Events.Finished, result); return result; } _testsToRun() { if (!this.hasFocusedTestsOrSuites()) return this._tests; const notFocusedSuites = new Set(); // Mark parent suites of focused tests as not focused. for (const test of this._tests) { if (test.focused()) { for (let suite = test.suite(); suite; suite = suite.parentSuite()) notFocusedSuites.add(suite); } } // Pick all tests that are focused or belong to focused suites. const tests = []; for (const test of this._tests) { let focused = test.focused(); for (let suite = test.suite(); suite; suite = suite.parentSuite()) focused = focused || (suite.focused() && !notFocusedSuites.has(suite)); if (focused) tests.push(test); } return tests; } async terminate() { if (!this._runningPass) return; await this._runningPass._terminate(TestResult.Terminated, 'Terminated with |TestRunner.terminate()| call', true /* force */, null /* error */); } timeout() { return this._timeout; } hasFocusedTestsOrSuites() { return this._tests.some(test => test.focused()) || this._suites.some(suite => suite.focused()); } focusMatchingTests(fullNameRegex) { for (const test of this._tests) { if (fullNameRegex.test(test.fullName())) test.setFocused(true); } } tests() { return this._tests.slice(); } suites() { return this._suites.slice(); } parallel() { return this._parallel; } } TestRunner.Events = { Started: 'started', Finished: 'finished', TestStarted: 'teststarted', TestFinished: 'testfinished', }; module.exports = TestRunner;