test: make fit run single test (#3394)

test/mocha/fixturesUI.js

@@ -29,7 +29,7 @@ process.env.JEST_WORKER_ID = 1;
 const fixturePool = new FixturePool();
 let revertBabelRequire;
 
-function fixturesUI(suite) {
+function fixturesUI(trialRun, suite) {
   const suites = [suite];
 
   suite.on(Suite.constants.EVENT_FILE_PRE_REQUIRE, function(context, file, mocha) {
@@ -37,8 +37,15 @@ function fixturesUI(suite) {
 
     context.beforeEach = common.beforeEach;
     context.afterEach = common.afterEach;
-    fixturePool.patchToEnableFixtures(context, 'beforeEach');
+    if (trialRun) {
-    fixturePool.patchToEnableFixtures(context, 'afterEach');
+      context.beforeEach = () => {};
+      context.afterEach = () => {};
+    } else {
+      context.beforeEach = common.beforeEach;
+      context.afterEach = common.afterEach;
+      fixturePool.patchToEnableFixtures(context, 'beforeEach');
+      fixturePool.patchToEnableFixtures(context, 'afterEach');
+    }
 
     context.run = mocha.options.delay && common.runWithSuite(suite);
 
@@ -74,14 +81,22 @@ function fixturesUI(suite) {
 
     context.it = context.specify = function(title, fn) {
       const suite = suites[0];
-      if (suite.isPending()) {
+      if (suite.isPending())
         fn = null;
+      let wrapper = fn;
+      if (trialRun) {
+        if (wrapper)
+          wrapper = () => {};
+      } else {
+        const wrapped = fixturePool.wrapTestCallback(wrapper);
+        wrapper = wrapped ? (done, ...args) => {
+          wrapped(...args).then(done).catch(done);
+        } : undefined;
+      }
+      if (wrapper) {
+        wrapper.toString = () => fn.toString();
+        wrapper.__original = fn;
       }
-      const wrapped = fixturePool.wrapTestCallback(fn);
-      const wrapper = wrapped ? (done, ...args) => {
-        wrapped(...args).then(done).catch(done);
-      } : undefined;
-
       const test = new Test(title, wrapper);
       test.file = file;
       suite.addTest(test);

test/mocha/index.js

@@ -14,42 +14,73 @@
  * limitations under the License.
  */
 
-const builtinReporters = require('mocha/lib/reporters');
 const fs = require('fs');
 const path = require('path');
 const program = require('commander');
 const { Runner } = require('./runner');
-const DotRunner = require('./dotReporter');
+const Mocha = require('mocha');
+const { fixturesUI } = require('./fixturesUI');
 
+class NullReporter {}
 
 program
   .version('Version ' + require('../../package.json').version)
+  .option('--timeout <timeout>', 'timeout', 10000)
   .option('--reporter <reporter>', 'reporter to use', '')
-  .option('--max-workers <maxWorkers>', 'reporter to use', '')
+  .option('--max-workers <maxWorkers>', 'max workers to use', Math.ceil(require('os').cpus().length / 2))
-  .action(async (command, args) => {
+  .option('--retries <retries>', 'number of times to retry a failing test', 1)
-    const testDir = path.join(process.cwd(), 'test');
+  .action(async (command) => {
-    const files = [];
+    // Collect files
-    for (const name of fs.readdirSync(testDir)) {
+    const files = collectFiles(command.args);
-      if (!name.includes('.spec.'))
+    const rootSuite = new Mocha.Suite('', new Mocha.Context(), true);
-        continue;
+
-      if (!command.args.length) {
+    // Build the test model, suite per file.
-        files.push(path.join(testDir, name));
+    for (const file of files) {
-        continue;
+      const mocha = new Mocha({
-      }
+        ui: fixturesUI.bind(null, true),
-      for (const filter of command.args) {
+        retries: command.retries,
-        if (name.includes(filter)) {
+        timeout: command.timeout,
-          files.push(path.join(testDir, name));
+        reporter: NullReporter
-          break;
+      });
-        }
+      mocha.addFile(file);
-      }
+      mocha.suite.title = path.basename(file);
+      mocha.suite.root = false;
+      rootSuite.suites.push(mocha.suite);
+      await new Promise(f => mocha.run(f));
     }
 
-    const runner = new Runner({
+    if (rootSuite.hasOnly())
-      reporter: command.reporter ? builtinReporters[command.reporter] : DotRunner,
+      rootSuite.filterOnly();
-      maxWorkers: command.maxWorkers || Math.ceil(require('os').cpus().length / 2)
+
+    console.log(`Running ${rootSuite.total()} tests`);
+    const runner = new Runner(rootSuite, {
+      maxWorkers: command.maxWorkers,
+      reporter: command.reporter,
+      retries: command.retries,
+      timeout: command.timeout,
     });
     await runner.run(files);
     await runner.stop();
   });
 
 program.parse(process.argv);
+
+function collectFiles(args) {
+  const testDir = path.join(process.cwd(), 'test');
+  const files = [];
+  for (const name of fs.readdirSync(testDir)) {
+    if (!name.includes('.spec.'))
+      continue;
+    if (!args.length) {
+      files.push(path.join(testDir, name));
+      continue;
+    }
+    for (const filter of args) {
+      if (name.includes(filter)) {
+        files.push(path.join(testDir, name));
+        break;
+      }
+    }
+  }
+  return files;
+}

test/mocha/runner.js

@@ -18,13 +18,16 @@ const child_process = require('child_process');
 const path = require('path');
 const { EventEmitter } = require('events');
 const Mocha = require('mocha');
-const { Serializer } = require('v8');
+const builtinReporters = require('mocha/lib/reporters');
+const DotRunner = require('./dotReporter');
 
 const constants = Mocha.Runner.constants;
 
 class Runner extends EventEmitter {
-  constructor(options) {
+  constructor(suite, options) {
     super();
+    this._suite = suite;
+    this._options = options;
     this._maxWorkers = options.maxWorkers;
     this._workers = new Set();
     this._freeWorkers = [];
@@ -37,15 +40,32 @@ class Runner extends EventEmitter {
       pending: 0,
       tests: 0,
     };
-    this._reporter = new options.reporter(this, {});
+    const reporterFactory = builtinReporters[options.reporter] || DotRunner;
+    this._reporter = new reporterFactory(this, {});
+
+    this._tests = new Map();
+    this._files = new Map();
+    this._traverse(suite);
   }
 
-  async run(files) {
+  _traverse(suite) {
+    for (const child of suite.suites)
+      this._traverse(child);
+    for (const test of suite.tests) {
+      if (!this._files.has(test.file))
+        this._files.set(test.file, 0);
+      const counter = this._files.get(test.file);
+      this._files.set(test.file, counter + 1);
+      this._tests.set(`${test.file}::${counter}`, test);
+    }
+  }
+
+  async run() {
     this.emit(constants.EVENT_RUN_BEGIN, {});
     const result = new Promise(f => this._runCallback = f);
-    for (const file of files) {
+    for (const file of this._files.keys()) {
       const worker = await this._obtainWorker();
-      worker.send({ method: 'run', params: file });
+      worker.send({ method: 'run', params: { file, options: this._options } });
     }
     await result;
     this.emit(constants.EVENT_RUN_END, {});
@@ -61,16 +81,16 @@ class Runner extends EventEmitter {
       });
       let readyCallback;
       const result = new Promise(f => readyCallback = f);
-      worker.send({ method: 'init', params: ++this._workerId });
+      worker.send({ method: 'init', params: { workerId: ++this._workerId } });
       worker.on('message', message => {
         if (message.method === 'ready')
-            readyCallback();
+          readyCallback();
         this._messageFromWorker(worker, message);
       });
       worker.on('exit', () => {
         this._workers.delete(worker);
         if (!this._workers.size)
-          this._runCallback();
+          this._stopCallback();
       });
       this._workers.add(worker);
       await result;
@@ -89,25 +109,24 @@ class Runner extends EventEmitter {
           callback(worker);
         } else {
           this._freeWorkers.push(worker);
-          if (this._freeWorkers.length === this._workers.size) {
+          if (this._freeWorkers.length === this._workers.size)
             this._runCallback();
-          }
         }
         break;
       }
       case 'start':
         break;
       case 'test':
-        this.emit(constants.EVENT_TEST_BEGIN, this._parse(params.test));
+        this.emit(constants.EVENT_TEST_BEGIN, this._updateTest(params.test));
         break;
       case 'pending':
-        this.emit(constants.EVENT_TEST_PENDING, this._parse(params.test));
+        this.emit(constants.EVENT_TEST_PENDING, this._updateTest(params.test));
         break;
       case 'pass':
-        this.emit(constants.EVENT_TEST_PASS, this._parse(params.test));
+        this.emit(constants.EVENT_TEST_PASS, this._updateTest(params.test));
         break;
       case 'fail':
-        const test = this._parse(params.test);
+        const test = this._updateTest(params.test);
         this.emit(constants.EVENT_TEST_FAIL, test, params.error);
         break;
       case 'end':
@@ -120,19 +139,11 @@ class Runner extends EventEmitter {
     }
   }
 
-  _parse(serialized) {
+  _updateTest(serialized) {
-    return {
+    const test = this._tests.get(serialized.id);
-      ...serialized,
+    test._currentRetry = serialized.currentRetry;
-      currentRetry: () => serialized.currentRetry,
+    this.duration = serialized.duration;
-      fullTitle: () => serialized.fullTitle,
+    return test;
-      slow: () => serialized.slow,
-      timeout: () => serialized.timeout,
-      titlePath: () => serialized.titlePath,
-      isPending: () => serialized.isPending,
-      parent: {
-        fullTitle: () => ''
-      }
-    };
   }
 
   async stop() {

test/mocha/worker.js

@@ -16,7 +16,7 @@
 
 const path = require('path');
 const Mocha = require('mocha');
-const { fixturesUI } = require('./fixturesUI');
+const { fixturesUI, fixturePool } = require('./fixturesUI');
 const { gracefullyCloseAll } = require('../../lib/server/processLauncher');
 const GoldenUtils = require('../../utils/testrunner/GoldenUtils');
 
@@ -32,11 +32,11 @@ let closed = false;
 
 process.on('message', async message => {
   if (message.method === 'init')
-    process.env.JEST_WORKER_ID = message.params;
+    process.env.JEST_WORKER_ID = message.params.workerId;
   if (message.method === 'stop')
-    gracefullyCloseAndExit();
+    await gracefullyCloseAndExit();
   if (message.method === 'run')
-    await runSingleTest(message.params);
+    await runSingleTest(message.params.file, message.params.options);
 });
 
 process.on('disconnect', gracefullyCloseAndExit);
@@ -55,10 +55,12 @@ async function gracefullyCloseAndExit() {
 
 class NullReporter {}
 
-async function runSingleTest(file) {
+async function runSingleTest(file, options) {
+  let nextOrdinal = 0;
   const mocha = new Mocha({
-    ui: fixturesUI,
+    ui: fixturesUI.bind(null, false),
-    timeout: 10000,
+    retries: options.retries === 1 ? undefined : options.retries,
+    timeout: options.timeout,
     reporter: NullReporter
   });
   mocha.addFile(file);
@@ -71,22 +73,27 @@ async function runSingleTest(file) {
   });
 
   runner.on(constants.EVENT_TEST_BEGIN, test => {
-    sendMessageToParent('test', { test: sanitizeTest(test) });
+    // Retries will produce new test instances, store ordinal on the original function.
+    let ordinal = nextOrdinal++;
+    if (typeof test.fn.__original.__ordinal !== 'number')
+      test.fn.__original.__ordinal = ordinal;
+    sendMessageToParent('test', { test: serializeTest(test, ordinal) });
   });
 
   runner.on(constants.EVENT_TEST_PENDING, test => {
-    sendMessageToParent('pending', { test: sanitizeTest(test) });
+    // Pending does not get test begin signal, so increment ordinal.
+    sendMessageToParent('pending', { test: serializeTest(test, nextOrdinal++) });
   });
 
   runner.on(constants.EVENT_TEST_PASS, test => {
-    sendMessageToParent('pass', { test: sanitizeTest(test) });
+    sendMessageToParent('pass', { test: serializeTest(test, test.fn.__original.__ordinal) });
   });
 
   runner.on(constants.EVENT_TEST_FAIL, (test, error) => {
     sendMessageToParent('fail', {
-      test: sanitizeTest(test),
+      test: serializeTest(test, test.fn.__original.__ordinal),
       error: serializeError(error),
-     });
+    });
   });
 
   runner.once(constants.EVENT_RUN_END, async () => {
@@ -105,17 +112,12 @@ function sendMessageToParent(method, params = {}) {
   }
 }
 
-function sanitizeTest(test) {
+function serializeTest(test, origin) {
   return {
+    id: `${test.file}::${origin}`,
     currentRetry: test.currentRetry(),
     duration: test.duration,
-    file: test.file,
-    fullTitle: test.fullTitle(),
-    isPending: test.isPending(),
-    slow: test.slow(),
-    timeout: test.timeout(),
     title: test.title,
-    titlePath: test.titlePath(),
   };
 }