chore(testrunner): merge test spec with test, suite spec with suite (#1566)

utils/doclint/check_public_api/test/test.js

@@ -61,7 +61,7 @@ describe('checkPublicAPI', function() {
 runner.run();
 
 async function testLint(state, test) {
-  const dirPath = path.join(__dirname, test.name);
+  const dirPath = path.join(__dirname, test.name());
   const {expect} = new Matchers({
     toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
   });
@@ -75,7 +75,7 @@ async function testLint(state, test) {
 }
 
 async function testMDBuilder(state, test) {
-  const dirPath = path.join(__dirname, test.name);
+  const dirPath = path.join(__dirname, test.name());
   const {expect} = new Matchers({
     toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
   });
@@ -85,7 +85,7 @@ async function testMDBuilder(state, test) {
 }
 
 async function testJSBuilder(state, test) {
-  const dirPath = path.join(__dirname, test.name);
+  const dirPath = path.join(__dirname, test.name());
   const {expect} = new Matchers({
     toBeGolden: GoldenUtils.compare.bind(null, dirPath, dirPath)
   });

utils/testrunner/Reporter.js

@@ -49,14 +49,14 @@ class Reporter {
       console.log(`Running ${colors.yellow(runnableTests.length)} focused tests out of total ${colors.yellow(allTests.length)} on ${colors.yellow(this._runner.parallel())} worker${this._runner.parallel() > 1 ? 's' : ''}`);
       console.log('');
       const focusedSuites = this._runner.focusedSuites().map(suite => ({
-        id: suite.location.filePath + ':' + suite.location.lineNumber + ':' + suite.location.columnNumber,
+        id: suite.location().filePath + ':' + suite.location().lineNumber + ':' + suite.location().columnNumber,
-        fullName: suite.fullName,
+        fullName: suite.fullName(),
-        location: suite.location,
+        location: suite.location(),
       }));
       const focusedTests = this._runner.focusedTests().map(test => ({
-        id: test.location.filePath + ':' + test.location.lineNumber + ':' + test.location.columnNumber,
+        id: test.location().filePath + ':' + test.location().lineNumber + ':' + test.location().columnNumber,
-        fullName: test.fullName,
+        fullName: test.fullName(),
-        location: test.location,
+        location: test.location(),
       }));
       const focusedEntities = new Map([
         ...focusedSuites.map(suite => ([suite.id, suite])),
@@ -121,7 +121,7 @@ class Reporter {
       if (markedAsFailingTests.length > 0) {
         console.log('\nMarked as failing:');
         markedAsFailingTests.slice(0, this._showMarkedAsFailingTests).forEach((test, index) => {
-          console.log(`${index + 1}) ${test.fullName} (${formatLocation(test.location)})`);
+          console.log(`${index + 1}) ${test.fullName()} (${formatLocation(test.location())})`);
         });
       }
       if (this._showMarkedAsFailingTests < markedAsFailingTests.length) {
@@ -140,7 +140,7 @@ class Reporter {
       for (let i = 0; i < slowTests.length; ++i) {
         const test = slowTests[i];
         const duration = test.endTimestamp - test.startTimestamp;
-        console.log(`  (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName} (${formatLocation(test.location)})`);
+        console.log(`  (${i + 1}) ${colors.yellow((duration / 1000) + 's')} - ${test.fullName()} (${formatLocation(test.location())})`);
       }
     }
 
@@ -195,23 +195,23 @@ class Reporter {
     if (this._runner.parallel() > 1 && workerId !== undefined)
       prefix += ' ' + colors.gray(`[worker = ${workerId}]`);
     if (test.result === 'ok') {
-      console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName} (${formatLocation(test.location)})`);
+      console.log(`${prefix} ${colors.green('[OK]')} ${test.fullName()} (${formatLocation(test.location())})`);
     } else if (test.result === 'terminated') {
-      console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName} (${formatLocation(test.location)})`);
+      console.log(`${prefix} ${colors.magenta('[TERMINATED]')} ${test.fullName()} (${formatLocation(test.location())})`);
     } else if (test.result === 'crashed') {
-      console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName} (${formatLocation(test.location)})`);
+      console.log(`${prefix} ${colors.red('[CRASHED]')} ${test.fullName()} (${formatLocation(test.location())})`);
     } else if (test.result === 'skipped') {
     } else if (test.result === 'markedAsFailing') {
-      console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName} (${formatLocation(test.location)})`);
+      console.log(`${prefix} ${colors.yellow('[MARKED AS FAILING]')} ${test.fullName()} (${formatLocation(test.location())})`);
     } else if (test.result === 'timedout') {
-      console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout}ms]`)} ${test.fullName} (${formatLocation(test.location)})`);
+      console.log(`${prefix} ${colors.red(`[TIMEOUT ${test.timeout()}ms]`)} ${test.fullName()} (${formatLocation(test.location())})`);
       if (test.output) {
         console.log('  Output:');
         for (const line of test.output)
           console.log('  ' + line);
       }
     } else if (test.result === 'failed') {
-      console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName} (${formatLocation(test.location)})`);
+      console.log(`${prefix} ${colors.red('[FAIL]')} ${test.fullName()} (${formatLocation(test.location())})`);
       if (test.error instanceof MatchError) {
         let lines = this._filePathToLines.get(test.error.location.filePath);
         if (!lines) {
@@ -225,7 +225,7 @@ class Reporter {
         const lineNumber = test.error.location.lineNumber;
         if (lineNumber < lines.length) {
           const lineNumberLength = (lineNumber + 1 + '').length;
-          const FROM = Math.max(test.location.lineNumber - 1, lineNumber - 5);
+          const FROM = Math.max(test.location().lineNumber - 1, lineNumber - 5);
           const snippet = lines.slice(FROM, lineNumber).map((line, index) => `    ${(FROM + index + 1 + '').padStart(lineNumberLength, ' ')} | ${line}`).join('\n');
           const pointer = `    ` + ' '.repeat(lineNumberLength) + '   ' + '~'.repeat(test.error.location.columnNumber - 1) + '^';
           console.log('\n' + snippet + '\n' + colors.grey(pointer) + '\n');
@@ -243,10 +243,10 @@ class Reporter {
           console.log('  Stack:');
           let stack = test.error.stack;
           // Highlight first test location, if any.
-          const match = stack.match(new RegExp(test.location.filePath + ':(\\d+):(\\d+)'));
+          const match = stack.match(new RegExp(test.location().filePath + ':(\\d+):(\\d+)'));
           if (match) {
             const [, line, column] = match;
-            const fileName = `${test.location.fileName}:${line}:${column}`;
+            const fileName = `${test.location().fileName}:${line}:${column}`;
             stack = stack.substring(0, match.index) + stack.substring(match.index).replace(fileName, colors.yellow(fileName));
           }
           console.log(padLines(stack, 4));

utils/testrunner/TestRunner.js

@@ -73,39 +73,173 @@ function isTestFailure(testResult) {
 }
 
 class Test {
-  constructor(suite, name, callback, declaredMode, expectation, timeout) {
+  constructor(suite, name, callback, location) {
-    this.suite = suite;
+    this._suite = suite;
-    this.name = name;
+    this._name = name;
-    this.fullName = (suite.fullName + ' ' + name).trim();
+    this._fullName = (suite.fullName() + ' ' + name).trim();
-    this.declaredMode = declaredMode;
+    this._mode = TestMode.Run;
-    this.expectation = expectation;
+    this._expectation = TestExpectation.Ok;
-    this._callback = callback;
+    this._body = callback;
-    this.location = getCallerLocation(__filename);
+    this._location = location;
-    this.timeout = timeout;
+    this._timeout = INFINITE_TIMEOUT;
+    this._repeat = 1;
 
-    // Test results
+    // Test results. TODO: make these private.
     this.result = null;
     this.error = null;
     this.startTimestamp = 0;
     this.endTimestamp = 0;
+
+    this.Modes = { ...TestMode };
+    this.Expectations = { ...TestExpectation };
+  }
+
+  _clone() {
+    // TODO: introduce TestRun instead?
+    const test = new Test(this._suite, this._name, this._body, this._location);
+    test._timeout = this._timeout;
+    test._mode = this._mode;
+    test._expectation = this._expectation;
+    return test;
+  }
+
+  suite() {
+    return this._suite;
+  }
+
+  name() {
+    return this._name;
+  }
+
+  fullName() {
+    return this._fullName;
+  }
+
+  location() {
+    return this._location;
+  }
+
+  body() {
+    return this._body;
+  }
+
+  mode() {
+    return this._mode;
+  }
+
+  setMode(mode) {
+    if (this._mode !== TestMode.Focus)
+      this._mode = mode;
+  }
+
+  timeout() {
+    return this._timeout;
+  }
+
+  setTimeout(timeout) {
+    this._timeout = timeout;
+  }
+
+  expectation() {
+    return this._expectation;
+  }
+
+  setExpectation(expectation) {
+    this._expectation = expectation;
+  }
+
+  repeat() {
+    return this._repeat;
+  }
+
+  setRepeat(repeat) {
+    this._repeat = repeat;
+  }
+
+  effectiveMode() {
+    for (let suite = this._suite; suite; suite = suite.parentSuite()) {
+      if (suite.mode() === TestMode.Skip)
+        return TestMode.Skip;
+    }
+    return this._mode;
+  }
+
+  effectiveExpectation() {
+    for (let suite = this._suite; suite; suite = suite.parentSuite()) {
+      if (suite.expectation() === TestExpectation.Fail)
+        return TestExpectation.Fail;
+    }
+    return this._expectation;
   }
 }
 
 class Suite {
-  constructor(parentSuite, name, declaredMode, expectation) {
+  constructor(parentSuite, name, location) {
-    this.parentSuite = parentSuite;
+    this._parentSuite = parentSuite;
-    this.name = name;
+    this._name = name;
-    this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim();
+    this._fullName = (parentSuite ? parentSuite.fullName() + ' ' + name : name).trim();
-    this.declaredMode = declaredMode;
+    this._mode = TestMode.Run;
-    this.expectation = expectation;
+    this._expectation = TestExpectation.Ok;
-    /** @type {!Array<(!Test|!Suite)>} */
+    this._location = location;
-    this.children = [];
+    this._repeat = 1;
-    this.location = getCallerLocation(__filename);
 
+    // TODO: make these private.
     this.beforeAll = null;
     this.beforeEach = null;
     this.afterAll = null;
     this.afterEach = null;
+
+    this.Modes = { ...TestMode };
+    this.Expectations = { ...TestExpectation };
+  }
+
+  _clone() {
+    // TODO: introduce TestRun instead?
+    const suite = new Suite(this._parentSuite, this._name, this._location);
+    suite._mode = this._mode;
+    suite._expectation = this._expectation;
+    return suite;
+  }
+
+  parentSuite() {
+    return this._parentSuite;
+  }
+
+  name() {
+    return this._name;
+  }
+
+  fullName() {
+    return this._fullName;
+  }
+
+  mode() {
+    return this._mode;
+  }
+
+  setMode(mode) {
+    if (this._mode !== TestMode.Focus)
+      this._mode = mode;
+  }
+
+  location() {
+    return this._location;
+  }
+
+  expectation() {
+    return this._expectation;
+  }
+
+  setExpectation(expectation) {
+    this._expectation = expectation;
+  }
+
+  repeat() {
+    return this._repeat;
+  }
+
+  setRepeat(repeat) {
+    this._repeat = repeat;
   }
 }
 
@@ -179,14 +313,14 @@ class TestWorker {
     if (this._markTerminated(test))
       return;
 
-    if (test.declaredMode === TestMode.Skip) {
+    if (test.effectiveMode() === TestMode.Skip) {
       await this._testPass._willStartTest(this, test);
       test.result = TestResult.Skipped;
       await this._testPass._didFinishTest(this, test);
       return;
     }
 
-    if (test.expectation === TestExpectation.Fail && test.declaredMode !== TestMode.Focus) {
+    if (test.effectiveExpectation() === TestExpectation.Fail && test.effectiveMode() !== TestMode.Focus) {
       await this._testPass._willStartTest(this, test);
       test.result = TestResult.MarkedAsFailing;
       await this._testPass._didFinishTest(this, test);
@@ -194,7 +328,7 @@ class TestWorker {
     }
 
     const suiteStack = [];
-    for (let suite = test.suite; suite; suite = suite.parentSuite)
+    for (let suite = test.suite(); suite; suite = suite.parentSuite())
       suiteStack.push(suite);
     suiteStack.reverse();
 
@@ -230,7 +364,7 @@ class TestWorker {
 
     if (!test.error && !this._markTerminated(test)) {
       await this._testPass._willStartTestBody(this, test);
-      const { promise, terminate } = runUserCallback(test._callback, test.timeout, [this._state, test]);
+      const { promise, terminate } = runUserCallback(test.body(), test.timeout(), [this._state, test]);
       this._runningTestTerminate = terminate;
       test.error = await promise;
       this._runningTestTerminate = null;
@@ -259,7 +393,7 @@ class TestWorker {
 
     await this._testPass._willStartHook(this, suite, hook.location, hookName);
     const timeout = this._testPass._runner._timeout;
-    const { promise, terminate } = runUserCallback(hook.callback, timeout, [this._state, test]);
+    const { promise, terminate } = runUserCallback(hook.body, timeout, [this._state, test]);
     this._runningHookTerminate = terminate;
     let error = await promise;
     this._runningHookTerminate = null;
@@ -272,7 +406,7 @@ class TestWorker {
       }
       let message;
       if (error === TimeoutError) {
-        message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hookName}" in suite "${suite.fullName}"`;
+        message = `${locationString} - Timeout Exceeded ${timeout}ms while running "${hookName}" in suite "${suite.fullName()}"`;
         error = null;
       } else if (error === TerminatedError) {
         // Do not report termination details - it's just noise.
@@ -281,7 +415,7 @@ class TestWorker {
       } else {
         if (error.stack)
           await this._testPass._runner._sourceMapSupport.rewriteStackTraceWithSourceMaps(error);
-        message = `${locationString} - FAILED while running "${hookName}" in suite "${suite.fullName}": `;
+        message = `${locationString} - FAILED while running "${hookName}" in suite "${suite.fullName()}": `;
       }
       await this._testPass._didFailHook(this, suite, hook.location, hookName, message, error);
       test.error = error;
@@ -411,78 +545,29 @@ class TestPass {
   }
 
   async _willStartTestBody(worker, test) {
-    debug('testrunner:test')(`[${worker._workerId}] starting "${test.fullName}" (${test.location.fileName + ':' + test.location.lineNumber})`);
+    debug('testrunner:test')(`[${worker._workerId}] starting "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`);
   }
 
   async _didFinishTestBody(worker, test) {
-    debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName}" (${test.location.fileName + ':' + test.location.lineNumber})`);
+    debug('testrunner:test')(`[${worker._workerId}] ${test.result.toUpperCase()} "${test.fullName()}" (${test.location().fileName + ':' + test.location().lineNumber})`);
   }
 
   async _willStartHook(worker, suite, location, hookName) {
-    debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" started for "${suite.fullName}" (${location.fileName + ':' + location.lineNumber})`);
+    debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" started for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`);
   }
 
   async _didFailHook(worker, suite, location, hookName, message, error) {
-    debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" FAILED for "${suite.fullName}" (${location.fileName + ':' + location.lineNumber})`);
+    debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" FAILED for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`);
     if (message)
       this._result.addError(message, error, worker);
     this._result.setResult(TestResult.Crashed, message);
   }
 
   async _didCompleteHook(worker, suite, location, hookName) {
-    debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" OK for "${suite.fullName}" (${location.fileName + ':' + location.lineNumber})`);
+    debug('testrunner:hook')(`[${worker._workerId}] "${hookName}" OK for "${suite.fullName()}" (${location.fileName + ':' + location.lineNumber})`);
   }
 }
 
-// TODO: merge spec with Test/Suite.
-function createSpec(name, callback) {
-  let timeout = INFINITE_TIMEOUT;
-  let repeat = 1;
-  let expectation = TestExpectation.Ok;
-  let mode = TestMode.Run;
-  const spec = {
-    Modes: { ...TestMode },
-    Expectations: { ...TestExpectation },
-    name() {
-      return name;
-    },
-    callback() {
-      return callback;
-    },
-    mode() {
-      return mode;
-    },
-    setMode(m) {
-      if (mode !== TestMode.Focus)
-        mode = m;
-    },
-    expectations() {
-      return [expectation];
-    },
-    setExpectations(e) {
-      if (Array.isArray(e)) {
-        if (e.length > 1)
-          throw new Error('Only a single expectation is currently supported');
-        e = e[0];
-      }
-      expectation = e;
-    },
-    timeout() {
-      return timeout;
-    },
-    setTimeout(t) {
-      timeout = t;
-    },
-    repeat() {
-      return repeat;
-    },
-    setRepeat(r) {
-      repeat = r;
-    },
-  };
-  return spec;
-}
-
 class TestRunner extends EventEmitter {
   constructor(options = {}) {
     super();
@@ -495,15 +580,18 @@ class TestRunner extends EventEmitter {
     } = options;
     this._crashIfTestsAreFocusedOnCI = crashIfTestsAreFocusedOnCI;
     this._sourceMapSupport = new SourceMapSupport();
-    this._rootSuite = new Suite(null, '', TestMode.Run);
+    const dummyLocation = { fileName: '', filePath: '', lineNumber: 0, columnNumber: 0 };
+    this._rootSuite = new Suite(null, '', dummyLocation);
     this._currentSuite = this._rootSuite;
     this._tests = [];
     this._suites = [];
     this._timeout = timeout === 0 ? INFINITE_TIMEOUT : timeout;
     this._parallel = parallel;
     this._breakOnFailure = breakOnFailure;
-    this._modifiers = new Map();
+    this._suiteModifiers = new Map();
-    this._attributes = new Map();
+    this._suiteAttributes = new Map();
+    this._testModifiers = new Map();
+    this._testAttributes = new Map();
 
     if (MAJOR_NODEJS_VERSION >= 8 && disableTimeoutWhenInspectorIsEnabled) {
       if (inspector.url()) {
@@ -520,23 +608,27 @@ class TestRunner extends EventEmitter {
     this.afterAll = this._addHook.bind(this, 'afterAll');
     this.afterEach = this._addHook.bind(this, 'afterEach');
 
-    this.describe = this._specBuilder([], true);
+    this.describe = this._suiteBuilder([]);
-    this.it = this._specBuilder([], false);
+    this.it = this._testBuilder([]);
 
-    this._attributes.set('debug', t => {
+    this.testAttribute('debug', t => {
       t.setMode(t.Modes.Focus);
       t.setTimeout(INFINITE_TIMEOUT);
-      const N = t.callback().toString().split('\n').length;
+      const N = t.body().toString().split('\n').length;
-      const location = getCallerLocation(__filename);
+      const location = t.location();
       for (let line = 0; line < N; ++line)
         this._debuggerLogBreakpointLines.set(location.filePath, line + location.lineNumber);
     });
 
-    this._modifiers.set('skip', (t, condition) => condition && t.setMode(t.Modes.Skip));
+    this.testModifier('skip', (t, condition) => condition && t.setMode(t.Modes.Skip));
-    this._modifiers.set('fail', (t, condition) => condition && t.setExpectations(t.Expectations.Fail));
+    this.suiteModifier('skip', (s, condition) => condition && s.setMode(s.Modes.Skip));
-    this._modifiers.set('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
+    this.testModifier('fail', (t, condition) => condition && t.setExpectation(t.Expectations.Fail));
-    this._modifiers.set('repeat', (t, count) => t.setRepeat(count));
+    this.suiteModifier('fail', (s, condition) => condition && s.setExpectation(s.Expectations.Fail));
-    this._attributes.set('focus', t => t.setMode(t.Modes.Focus));
+    this.testModifier('slow', (t, condition) => condition && t.setTimeout(t.timeout() * 3));
+    this.testModifier('repeat', (t, count) => t.setRepeat(count));
+    this.suiteModifier('repeat', (s, count) => s.setRepeat(count));
+    this.testAttribute('focus', t => t.setMode(t.Modes.Focus));
+    this.suiteAttribute('focus', s => s.setMode(s.Modes.Focus));
     this.fdescribe = this.describe.focus;
     this.xdescribe = this.describe.skip(true);
     this.fit = this.it.focus;
@@ -544,86 +636,78 @@ class TestRunner extends EventEmitter {
     this.dit = this.it.debug;
   }
 
-  _specBuilder(callbacks, isSuite) {
+  _suiteBuilder(callbacks) {
-    return new Proxy(() => {}, {
+    return new Proxy((name, callback, ...suiteArgs) => {
-      apply: (target, thisArg, [name, callback]) => {
+      const location = getCallerLocation(__filename);
-        const spec = createSpec(name, callback);
+      const suite = new Suite(this._currentSuite, name, location);
-        spec.setTimeout(this._timeout);
+      for (const { callback, args } of callbacks)
-        for (const { callback, args } of callbacks)
+        callback(suite, ...args);
-          callback(spec, ...args);
+      for (let i = 0; i < suite.repeat(); i++) {
-        if (isSuite)
+        this._currentSuite = suite._clone();
-          this._addSuite(spec, []);
+        callback(...suiteArgs);
-        else
+        this._suites.push(this._currentSuite);
-          this._addTest(spec);
+        this._currentSuite = this._currentSuite.parentSuite();
-      },
+      }
+    }, {
       get: (obj, prop) => {
-        if (this._modifiers.has(prop))
+        if (this._suiteModifiers.has(prop))
-          return (...args) => this._specBuilder([...callbacks, { callback: this._modifiers.get(prop), args }], isSuite);
+          return (...args) => this._suiteBuilder([...callbacks, { callback: this._suiteModifiers.get(prop), args }]);
-        if (this._attributes.has(prop))
+        if (this._suiteAttributes.has(prop))
-          return this._specBuilder([...callbacks, { callback: this._attributes.get(prop), args: [] }], isSuite);
+          return this._suiteBuilder([...callbacks, { callback: this._suiteAttributes.get(prop), args: [] }]);
         return obj[prop];
       },
     });
   }
 
-  modifier(name, callback) {
+  _testBuilder(callbacks) {
-    this._modifiers.set(name, callback);
+    return new Proxy((name, callback) => {
+      const location = getCallerLocation(__filename);
+      const test = new Test(this._currentSuite, name, callback, location);
+      test.setTimeout(this._timeout);
+      for (const { callback, args } of callbacks)
+        callback(test, ...args);
+      for (let i = 0; i < test.repeat(); i++)
+        this._tests.push(test._clone());
+    }, {
+      get: (obj, prop) => {
+        if (this._testModifiers.has(prop))
+          return (...args) => this._testBuilder([...callbacks, { callback: this._testModifiers.get(prop), args }]);
+        if (this._testAttributes.has(prop))
+          return this._testBuilder([...callbacks, { callback: this._testAttributes.get(prop), args: [] }]);
+        return obj[prop];
+      },
+    });
   }
 
-  attribute(name, callback) {
+  testModifier(name, callback) {
-    this._attributes.set(name, callback);
+    this._testModifiers.set(name, callback);
+  }
+
+  testAttribute(name, callback) {
+    this._testAttributes.set(name, callback);
+  }
+
+  suiteModifier(name, callback) {
+    this._suiteModifiers.set(name, callback);
+  }
+
+  suiteAttribute(name, callback) {
+    this._suiteAttributes.set(name, callback);
   }
 
   loadTests(module, ...args) {
-    if (typeof module.describe === 'function') {
+    if (typeof module.describe === 'function')
-      const spec = createSpec('', module.describe);
+      this.describe('', module.describe, ...args);
-      spec.setMode(spec.Modes.Run);
+    if (typeof module.fdescribe === 'function')
-      this._addSuite(spec, args);
+      this.describe.focus('', module.fdescribe, ...args);
-    }
+    if (typeof module.xdescribe === 'function')
-    if (typeof module.fdescribe === 'function') {
+      this.describe.skip(true)('', module.xdescribe, ...args);
-      const spec = createSpec('', module.fdescribe);
-      spec.setMode(spec.Modes.Focus);
-      this._addSuite(spec, args);
-    }
-    if (typeof module.xdescribe === 'function') {
-      const spec = createSpec('', module.xdescribe);
-      spec.setMode(spec.Modes.Skip);
-      this._addSuite(spec, args);
-    }
-  }
-
-  _addTest(spec) {
-    for (let i = 0; i < spec.repeat(); i++) {
-      let expectation = spec.expectations()[0];
-      let mode = spec.mode();
-      for (let suite = this._currentSuite; suite; suite = suite.parentSuite) {
-        if (suite.expectation === TestExpectation.Fail)
-          expectation = TestExpectation.Fail;
-        if (suite.declaredMode === TestMode.Skip)
-          mode = TestMode.Skip;
-      }
-      const test = new Test(this._currentSuite, spec.name(), spec.callback(), mode, expectation, spec.timeout());
-      this._currentSuite.children.push(test);
-      this._tests.push(test);
-    }
-  }
-
-  _addSuite(spec, args) {
-    for (let i = 0; i < spec.repeat(); i++) {
-      const oldSuite = this._currentSuite;
-      const suite = new Suite(this._currentSuite, spec.name(), spec.mode(), spec.expectations()[0]);
-      this._suites.push(suite);
-      this._currentSuite.children.push(suite);
-      this._currentSuite = suite;
-      spec.callback()(...args);
-      this._currentSuite = oldSuite;
-    }
   }
 
   _addHook(hookName, callback) {
     assert(this._currentSuite[hookName] === null, `Only one ${hookName} hook available per suite`);
     const location = getCallerLocation(__filename);
-    this._currentSuite[hookName] = { callback, location };
+    this._currentSuite[hookName] = { body: callback, location };
   }
 
   async run(options = {}) {
@@ -636,21 +720,17 @@ class TestRunner extends EventEmitter {
     if (this._crashIfTestsAreFocusedOnCI && process.env.CI && this.hasFocusedTestsOrSuites()) {
       result.setResult(TestResult.Crashed, '"focused" tests or suites are probitted on CI');
     } else {
+      this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure);
       let timeoutId;
-      const timeoutPromise = new Promise(resolve => {
+      if (totalTimeout) {
-        const timeoutResult = new Result();
+        timeoutId = setTimeout(() => {
-        timeoutResult.setResult(TestResult.Crashed, `Total timeout of ${totalTimeout}ms reached.`);
+          this._runningPass._terminate(TestResult.Terminated, `Total timeout of ${totalTimeout}ms reached.`, true /* force */, null /* error */);
-        if (totalTimeout)
+        }, totalTimeout);
-          timeoutId = setTimeout(resolve.bind(null, timeoutResult), totalTimeout);
+      }
-      });
       try {
-        this._runningPass = new TestPass(this, this._parallel, this._breakOnFailure);
+        result = await this._runningPass.run(runnableTests).catch(e => { console.error(e); throw e; });
-        result = await Promise.race([
-          this._runningPass.run(runnableTests).catch(e => { console.error(e); throw e; }),
-          timeoutPromise,
-        ]);
-        this._runningPass = null;
       } finally {
+        this._runningPass = null;
         clearTimeout(timeoutId);
       }
     }
@@ -679,18 +759,18 @@ class TestRunner extends EventEmitter {
     // First pass: pick "fit" and blacklist parent suites
     for (let i = 0; i < this._tests.length; i++) {
       const test = this._tests[i];
-      if (test.declaredMode !== TestMode.Focus)
+      if (test.mode() !== TestMode.Focus)
         continue;
       tests.push({ i, test });
-      for (let suite = test.suite; suite; suite = suite.parentSuite)
+      for (let suite = test.suite(); suite; suite = suite.parentSuite())
         blacklistSuites.add(suite);
     }
     // Second pass: pick all tests that belong to non-blacklisted "fdescribe"
     for (let i = 0; i < this._tests.length; i++) {
       const test = this._tests[i];
       let insideFocusedSuite = false;
-      for (let suite = test.suite; suite; suite = suite.parentSuite) {
+      for (let suite = test.suite(); suite; suite = suite.parentSuite()) {
-        if (!blacklistSuites.has(suite) && suite.declaredMode === TestMode.Focus) {
+        if (!blacklistSuites.has(suite) && suite.mode() === TestMode.Focus) {
           insideFocusedSuite = true;
           break;
         }
@@ -703,11 +783,11 @@ class TestRunner extends EventEmitter {
   }
 
   focusedSuites() {
-    return this._suites.filter(suite => suite.declaredMode === TestMode.Focus);
+    return this._suites.filter(suite => suite.mode() === TestMode.Focus);
   }
 
   focusedTests() {
-    return this._tests.filter(test => test.declaredMode === TestMode.Focus);
+    return this._tests.filter(test => test.effectiveMode() === TestMode.Focus);
   }
 
   hasFocusedTestsOrSuites() {
@@ -716,8 +796,8 @@ class TestRunner extends EventEmitter {
 
   focusMatchingTests(fullNameRegex) {
     for (const test of this._tests) {
-      if (fullNameRegex.test(test.fullName))
+      if (fullNameRegex.test(test.fullName()))
-        test.declaredMode = TestMode.Focus;
+        test.setMode(TestMode.Focus);
     }
   }
 
@@ -726,7 +806,7 @@ class TestRunner extends EventEmitter {
   }
 
   failedTests() {
-    return this._tests.filter(test => test.result === TestResult.Failed || test.result === TestResult.TimedOut || test.result === TestResult.Crashed);
+    return this._tests.filter(test => isTestFailure(test.result));
   }
 
   passedTests() {

utils/testrunner/test/testrunner.spec.js

@@ -17,13 +17,13 @@ module.exports.addTests = function({testRunner, expect}) {
       t.it('uno', () => {});
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.name).toBe('uno');
+      expect(test.name()).toBe('uno');
-      expect(test.fullName).toBe('uno');
+      expect(test.fullName()).toBe('uno');
-      expect(test.declaredMode).toBe('run');
+      expect(test.effectiveMode()).toBe('run');
-      expect(test.location.filePath).toEqual(__filename);
+      expect(test.location().filePath).toEqual(__filename);
-      expect(test.location.fileName).toEqual('testrunner.spec.js');
+      expect(test.location().fileName).toEqual('testrunner.spec.js');
-      expect(test.location.lineNumber).toBeTruthy();
+      expect(test.location().lineNumber).toBeTruthy();
-      expect(test.location.columnNumber).toBeTruthy();
+      expect(test.location().columnNumber).toBeTruthy();
     });
   });
 
@@ -33,9 +33,9 @@ module.exports.addTests = function({testRunner, expect}) {
       t.xit('uno', () => {});
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.name).toBe('uno');
+      expect(test.name()).toBe('uno');
-      expect(test.fullName).toBe('uno');
+      expect(test.fullName()).toBe('uno');
-      expect(test.declaredMode).toBe('skip');
+      expect(test.effectiveMode()).toBe('skip');
     });
   });
 
@@ -45,9 +45,9 @@ module.exports.addTests = function({testRunner, expect}) {
       t.fit('uno', () => {});
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.name).toBe('uno');
+      expect(test.name()).toBe('uno');
-      expect(test.fullName).toBe('uno');
+      expect(test.fullName()).toBe('uno');
-      expect(test.declaredMode).toBe('focus');
+      expect(test.effectiveMode()).toBe('focus');
     });
     it('should run a failed focused test', async() => {
       const t = newTestRunner();
@@ -56,7 +56,7 @@ module.exports.addTests = function({testRunner, expect}) {
       expect(t.tests().length).toBe(1);
       await t.run();
       expect(run).toBe(true);
-      expect(t.failedTests()[0].name).toBe('uno');
+      expect(t.failedTests()[0].name()).toBe('uno');
     });
   });
 
@@ -68,12 +68,12 @@ module.exports.addTests = function({testRunner, expect}) {
       });
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.name).toBe('uno');
+      expect(test.name()).toBe('uno');
-      expect(test.fullName).toBe('suite uno');
+      expect(test.fullName()).toBe('suite uno');
-      expect(test.declaredMode).toBe('run');
+      expect(test.effectiveMode()).toBe('run');
-      expect(test.suite.name).toBe('suite');
+      expect(test.suite().name()).toBe('suite');
-      expect(test.suite.fullName).toBe('suite');
+      expect(test.suite().fullName()).toBe('suite');
-      expect(test.suite.declaredMode).toBe('run');
+      expect(test.suite().mode()).toBe('run');
     });
   });
 
@@ -85,8 +85,8 @@ module.exports.addTests = function({testRunner, expect}) {
       });
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.declaredMode).toBe('skip');
+      expect(test.effectiveMode()).toBe('skip');
-      expect(test.suite.declaredMode).toBe('skip');
+      expect(test.suite().mode()).toBe('skip');
     });
     it('focused tests inside a skipped suite are considered skipped', async() => {
       const t = newTestRunner();
@@ -95,8 +95,8 @@ module.exports.addTests = function({testRunner, expect}) {
       });
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.declaredMode).toBe('skip');
+      expect(test.effectiveMode()).toBe('skip');
-      expect(test.suite.declaredMode).toBe('skip');
+      expect(test.suite().mode()).toBe('skip');
     });
   });
 
@@ -108,8 +108,8 @@ module.exports.addTests = function({testRunner, expect}) {
       });
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.declaredMode).toBe('run');
+      expect(test.effectiveMode()).toBe('run');
-      expect(test.suite.declaredMode).toBe('focus');
+      expect(test.suite().mode()).toBe('focus');
     });
     it('skipped tests inside a focused suite should stay skipped', async() => {
       const t = newTestRunner();
@@ -118,8 +118,8 @@ module.exports.addTests = function({testRunner, expect}) {
       });
       expect(t.tests().length).toBe(1);
       const test = t.tests()[0];
-      expect(test.declaredMode).toBe('skip');
+      expect(test.effectiveMode()).toBe('skip');
-      expect(test.suite.declaredMode).toBe('focus');
+      expect(test.suite().mode()).toBe('focus');
     });
     it('should run all "run" tests inside a focused suite', async() => {
       const log = [];
@@ -164,7 +164,7 @@ module.exports.addTests = function({testRunner, expect}) {
       const t = newTestRunner({timeout: 123});
       const log = [];
 
-      t.modifier('foo', (t, ...args) => {
+      t.testModifier('foo', (t, ...args) => {
         log.push('foo');
 
         expect(t.Modes.Run).toBeTruthy();
@@ -173,7 +173,7 @@ module.exports.addTests = function({testRunner, expect}) {
         expect(t.mode()).toBe(t.Modes.Run);
         expect(t.Expectations.Ok).toBeTruthy();
         expect(t.Expectations.Fail).toBeTruthy();
-        expect(t.expectations()).toEqual([t.Expectations.Ok]);
+        expect(t.expectation()).toBe(t.Expectations.Ok);
         expect(t.timeout()).toBe(123);
         expect(t.repeat()).toBe(1);
 
@@ -182,17 +182,17 @@ module.exports.addTests = function({testRunner, expect}) {
         expect(args[1]).toBe('dos');
 
         t.setMode(t.Modes.Focus);
-        t.setExpectations([t.Expectations.Fail]);
+        t.setExpectation(t.Expectations.Fail);
         t.setTimeout(234);
         t.setRepeat(42);
       });
 
-      t.attribute('bar', t => {
+      t.testAttribute('bar', t => {
         log.push('bar');
         expect(t.mode()).toBe(t.Modes.Focus);
         t.setMode(t.Modes.Skip);
         expect(t.mode()).toBe(t.Modes.Focus);
-        expect(t.expectations()).toEqual([t.Expectations.Fail]);
+        expect(t.expectation()).toBe(t.Expectations.Fail);
         expect(t.timeout()).toBe(234);
         expect(t.repeat()).toBe(42);
       });
@@ -339,6 +339,24 @@ module.exports.addTests = function({testRunner, expect}) {
       await t.run();
       expect(ran).toBe(true);
     });
+    it('should handle repeat', async() => {
+      const t = newTestRunner();
+      let suite = 0;
+      let test = 0;
+      let beforeAll = 0;
+      let beforeEach = 0;
+      t.describe.repeat(2)('suite', () => {
+        suite++;
+        t.beforeAll(() => beforeAll++);
+        t.beforeEach(() => beforeEach++);
+        t.it.repeat(3)('uno', () => test++);
+      });
+      await t.run();
+      expect(suite).toBe(2);
+      expect(beforeAll).toBe(2);
+      expect(beforeEach).toBe(6);
+      expect(test).toBe(6);
+    });
     it('should run tests if some fail', async() => {
       const t = newTestRunner();
       const log = [];
@@ -376,7 +394,7 @@ module.exports.addTests = function({testRunner, expect}) {
       t.beforeEach(state => state.FOO = 42);
       t.it('uno', (state, test) => {
         log.push('state.FOO=' + state.FOO);
-        log.push('test=' + test.name);
+        log.push('test=' + test.name());
       });
       await t.run();
       expect(log.join()).toBe('state.FOO=42,test=uno');
@@ -454,6 +472,13 @@ module.exports.addTests = function({testRunner, expect}) {
       await t.run();
       expect(log.join()).toBe('1,2,3,4');
     });
+    it('should respect total timeout', async() => {
+      const t = newTestRunner({timeout: 10000});
+      t.it('uno', async () => { await new Promise(() => {}); });
+      const result = await t.run({totalTimeout: 1});
+      expect(t.tests()[0].result).toBe('terminated');
+      expect(result.message).toContain('Total timeout');
+    });
   });
 
   describe('TestRunner.run result', () => {