test: introduce test collector (#3515)

test/base.fixture.ts

@@ -25,14 +25,13 @@ import { Transport } from '../lib/rpc/transport';
 import { setUnderTest } from '../lib/helper';
 import { installCoverageHooks } from './runner/coverage';
 import { valueFromEnv } from './runner/utils';
-import { registerFixture, registerWorkerFixture } from './runner/fixtures';
+import { registerFixture, registerWorkerFixture, registerWorkerGenerator } from './runner/fixtures';
 import './runner/builtin.fixtures';
 
 import {mkdtempAsync, removeFolderAsync} from './utils';
 
 setUnderTest(); // Note: we must call setUnderTest before requiring Playwright
 
-const browserName = process.env.BROWSER || 'chromium';
 const platform = os.platform();
 
 declare global {
@@ -58,6 +57,8 @@ declare global {
   }
 }
 
+const browserName = process.env.BROWSER;
+
 (global as any).MAC = platform === 'darwin';
 (global as any).LINUX = platform === 'linux';
 (global as any).WIN = platform === 'win32';
@@ -92,7 +93,6 @@ const getExecutablePath = (browserName) => {
     return process.env.FFPATH;
   if (browserName === 'webkit' && process.env.WKPATH)
     return process.env.WKPATH;
-  return
 }
 
 registerWorkerFixture('defaultBrowserOptions', async({browserName}, test) => {
@@ -151,7 +151,7 @@ registerFixture('toImpl', async ({playwright}, test) => {
 });
 
 registerWorkerFixture('browserType', async ({playwright, browserName}, test) => {
-  const browserType = playwright[process.env.BROWSER || 'chromium']
+  const browserType = playwright[browserName];
   const executablePath = getExecutablePath(browserName)
   if (executablePath)
     browserType._executablePath = executablePath
@@ -184,8 +184,10 @@ registerFixture('server', async ({httpService}, test) => {
   await test(httpService.server);
 });
 
-registerWorkerFixture('browserName', async ({}, test) => {
+registerWorkerGenerator('browserName', () => {
-  await test(browserName);
+  if (process.env.BROWSER)
+    return [process.env.BROWSER];
+  return ['chromium', 'webkit', 'firefox'];
 });
 
 registerWorkerFixture('isChromium', async ({browserName}, test) => {
@@ -216,5 +218,5 @@ registerWorkerFixture('asset', async ({}, test) => {
 });
 
 registerWorkerFixture('golden', async ({browserName}, test) => {
-  await test(p => path.join(`${browserName}`, p));
+  await test(p => path.join(browserName, p));
 });

test/runner/fixtures.js

@@ -19,6 +19,7 @@ const debug = require('debug');
 
 const registrations = new Map();
 const registrationsByFile = new Map();
+const generatorRegistrations = new Map();
 
 class Fixture {
   constructor(pool, name, scope, fn) {
@@ -28,10 +29,13 @@ class Fixture {
     this.fn = fn;
     this.deps = fixtureParameterNames(this.fn);
     this.usages = new Set();
-    this.value = null;
+    this.generatorValue = this.pool.generators.get(name);
+    this.value = this.generatorValue || null;
   }
 
   async setup() {
+    if (this.generatorValue)
+      return;
     for (const name of this.deps) {
       await this.pool.setupFixture(name);
       this.pool.instances.get(name).usages.add(this.name);
@@ -55,6 +59,8 @@ class Fixture {
   }
 
   async teardown() {
+    if (this.generatorValue)
+      return;
     if (this._teardown)
       return;
     this._teardown = true;
@@ -76,6 +82,7 @@ class Fixture {
 class FixturePool {
   constructor() {
     this.instances = new Map();
+    this.generators = new Map();
   }
 
   async setupFixture(name) {
@@ -120,21 +127,23 @@ class FixturePool {
       }
     };
   }
+}
 
-  fixtures(callback) {
+function fixturesForCallback(callback) {
-    const result = new Set();
+  const names = new Set();
   const visit  = (callback) => {
     for (const name of fixtureParameterNames(callback)) {
-        if (name in result)
+      if (name in names)
         continue;
-        result.add(name);
+        names.add(name);
       const { fn } = registrations.get(name)
       visit(fn);
     }
   };
   visit(callback);
+  const result = [...names];
+  result.sort();
   return result;
-  }
 }
 
 function fixtureParameterNames(fn) {
@@ -165,6 +174,11 @@ function registerWorkerFixture(name, fn) {
   innerRegisterFixture(name, 'worker', fn);
 };
 
+function registerWorkerGenerator(name, fn) {
+  innerRegisterFixture(name, 'worker', () => {});
+  generatorRegistrations.set(name, fn);
+}
+
 function collectRequires(file, result) {
   if (result.has(file))
     return;
@@ -179,12 +193,16 @@ function lookupRegistrations(file, scope) {
   const deps = new Set();
   collectRequires(file, deps);
   const allDeps = [...deps].reverse();
-  let result = [];
+  let result = new Map();
   for (const dep of allDeps) {
     const registrationList = registrationsByFile.get(dep);
     if (!registrationList)
       continue;
-    result = result.concat(registrationList.filter(r => r.scope === scope));
+    for (const r of registrationList) {
+      if (scope && r.scope !== scope)
+        continue;
+        result.set(r.name, r);
+    }
   }
   return result;
 }
@@ -192,7 +210,7 @@ function lookupRegistrations(file, scope) {
 function rerunRegistrations(file, scope) {
   // When we are running several tests in the same worker, we should re-run registrations before
   // each file. That way we erase potential fixture overrides from the previous test runs.
-  for (const registration of lookupRegistrations(file, scope))
+  for (const registration of lookupRegistrations(file, scope).values())
     registrations.set(registration.name, registration);
 }
 
@@ -202,9 +220,9 @@ function computeWorkerHash(file) {
   // This collection of fixtures is the fingerprint of the worker setup, a "worker hash".
   // Tests with the matching "worker hash" will reuse the same worker.
   const hash = crypto.createHash('sha1');
-  for (const registration of lookupRegistrations(file, 'worker'))
+  for (const registration of lookupRegistrations(file, 'worker').values())
     hash.update(registration.location);
   return hash.digest('hex');
 }
 
-module.exports = { FixturePool, registerFixture, registerWorkerFixture, computeWorkerHash, rerunRegistrations };
+module.exports = { FixturePool, registerFixture, registerWorkerFixture, computeWorkerHash, rerunRegistrations, lookupRegistrations, fixturesForCallback, registerWorkerGenerator, generatorRegistrations };

test/runner/fixturesUI.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-const { FixturePool, registerFixture, registerWorkerFixture, rerunRegistrations } = require('./fixtures');
+const { registerFixture, registerWorkerFixture, registerWorkerGenerator } = require('./fixtures');
 const { Test, Suite } = require('mocha');
 const { installTransform } = require('./transform');
 const commonSuite = require('mocha/lib/interfaces/common');
@@ -23,8 +23,8 @@ Error.stackTraceLimit = 15;
 global.testOptions = require('./testOptions');
 global.registerFixture = registerFixture;
 global.registerWorkerFixture = registerWorkerFixture;
+global.registerWorkerGenerator = registerWorkerGenerator;
 
-const fixturePool = new FixturePool();
 let revertBabelRequire;
 
 function specBuilder(modifiers, specCallback) {
@@ -57,7 +57,7 @@ function specBuilder(modifiers, specCallback) {
   return builder({}, null);
 }
 
-function fixturesUI(testRunner, suite) {
+function fixturesUI(wrappers, suite) {
   const suites = [suite];
 
   suite.on(Suite.constants.EVENT_FILE_PRE_REQUIRE, function(context, file, mocha) {
@@ -65,26 +65,18 @@ function fixturesUI(testRunner, suite) {
 
     const it = specBuilder(['skip', 'fail', 'slow', 'only'], (specs, title, fn) => {
       const suite = suites[0];
+
       if (suite.isPending())
         fn = null;
-      let wrapper;
+      const wrapper = fn ? wrappers.testWrapper(fn) : undefined;
-      const wrapped = fixturePool.wrapTestCallback(fn);
-      wrapper = wrapped ? (done, ...args) => {
-        if (!testRunner.shouldRunTest()) {
-          done();
-          return;
-        }
-        wrapped(...args).then(done).catch(done);
-      } : undefined;
       if (wrapper) {
         wrapper.toString = () => fn.toString();
         wrapper.__original = fn;
       }
       const test = new Test(title, wrapper);
-      test.__fixtures = fixturePool.fixtures(fn);
       test.file = file;
       suite.addTest(test);
-      const only = specs.only && specs.only[0];
+      const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0];
       if (specs.slow && specs.slow[0])
         test.timeout(90000);
       if (only)
@@ -102,7 +94,7 @@ function fixturesUI(testRunner, suite) {
         file: file,
         fn: fn
       });
-      const only = specs.only && specs.only[0];
+      const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0];
       if (only)
         suite.markOnly();
       if (!only && specs.skip && specs.skip[0])
@@ -112,23 +104,9 @@ function fixturesUI(testRunner, suite) {
       return suite;
     });
 
-    context.beforeEach = (fn) => {
+    context.beforeEach = fn => wrappers.hookWrapper(common.beforeEach.bind(common), fn);
-      if (!testRunner.shouldRunTest(true))
+    context.afterEach = fn => wrappers.hookWrapper(common.afterEach.bind(common), fn);
-        return;
-      return common.beforeEach(async () => {
-        return await fixturePool.resolveParametersAndRun(fn);
-      });
-    };
-    context.afterEach = (fn) => {
-      if (!testRunner.shouldRunTest(true))
-        return;
-      return common.afterEach(async () => {
-        return await fixturePool.resolveParametersAndRun(fn);
-      });
-    };
-
     context.run = mocha.options.delay && common.runWithSuite(suite);
-
     context.describe = describe;
     context.fdescribe = describe.only(true);
     context.xdescribe = describe.skip(true);
@@ -141,8 +119,7 @@ function fixturesUI(testRunner, suite) {
 
   suite.on(Suite.constants.EVENT_FILE_POST_REQUIRE, function(context, file, mocha) {
     revertBabelRequire();
-    rerunRegistrations(file, 'test');
   });
 };
 
-module.exports = { fixturesUI, fixturePool, registerFixture, registerWorkerFixture };
+module.exports = { fixturesUI };

test/runner/index.js

@@ -18,9 +18,7 @@ const fs = require('fs');
 const path = require('path');
 const program = require('commander');
 const { Runner } = require('./runner');
-const { TestRunner, createTestSuite } = require('./testRunner');
+const { TestCollector } = require('./testCollector');
-
-class NullReporter {}
 
 program
   .version('Version ' + require('../../package.json').version)
@@ -38,42 +36,31 @@ program
     // Collect files]
     const testDir = path.join(process.cwd(), command.args[0]);
     const files = collectFiles(testDir, '', command.args.slice(1));
-    const rootSuite = new createTestSuite();
 
-    let total = 0;
+    const testCollector = new TestCollector({
-    // Build the test model, suite per file.
-    for (const file of files) {
-      const testRunner = new TestRunner(file, [], {
       forbidOnly: command.forbidOnly || undefined,
       grep: command.grep,
-        reporter: NullReporter,
-        testDir,
       timeout: command.timeout,
-        trialRun: true,
     });
-      total += testRunner.grepTotal();
+    for (const file of files)
-      rootSuite.addSuite(testRunner.suite);
+      testCollector.addFile(file);
-      testRunner.suite.title = path.basename(file);
-    }
 
+    const rootSuite = testCollector.suite;
+    const total = rootSuite.total();
     if (!total) {
-      console.error('No tests found.');
+      console.error('=================');
+      console.error(' No tests found.');
+      console.error('=================');
       process.exit(1);
     }
 
     // Filter tests.
     if (rootSuite.hasOnly())
       rootSuite.filterOnly();
-    if (!command.reporter) {
-      console.log();
-      total = Math.min(total, rootSuite.total()); // First accounts for grep, second for only.
-      const workers = Math.min(command.jobs, files.length);
-      console.log(`Running ${total} test${ total > 1 ? 's' : '' } using ${workers} worker${ workers > 1 ? 's' : ''}`);
-    }
 
     // Trial run does not need many workers, use one.
     const jobs = (command.trialRun || command.debug) ? 1 : command.jobs;
-    const runner = new Runner(rootSuite, {
+    const runner = new Runner(rootSuite, total, {
       debug: command.debug,
       quiet: command.quiet,
       grep: command.grep,

test/runner/runner.js

@@ -27,7 +27,7 @@ const constants = Mocha.Runner.constants;
 process.setMaxListeners(0);
 
 class Runner extends EventEmitter {
-  constructor(suite, options) {
+  constructor(suite, total, options) {
     super();
     this._suite = suite;
     this._options = options;
@@ -45,30 +45,36 @@ class Runner extends EventEmitter {
     const reporterFactory = builtinReporters[options.reporter] || DotRunner;
     this._reporter = new reporterFactory(this, {});
 
-    this._tests = new Map();
+    this._testById = new Map();
-    this._files = new Map();
+    this._testsByConfiguredFile = new Map();
-
-    let grep;
-    if (options.grep) {
-      const match = options.grep.match(/^\/(.*)\/(g|i|)$|.*/);
-      grep = new RegExp(match[1] || match[0], match[2]);
-    }
 
     suite.eachTest(test => {
-      if (grep && !grep.test(test.fullTitle()))
+      const configuredFile = `${test.file}::[${test.__configurationString}]`;
-        return;
+      if (!this._testsByConfiguredFile.has(configuredFile)) {
-      if (!this._files.has(test.file))
+        this._testsByConfiguredFile.set(configuredFile, {
-        this._files.set(test.file, 0);
+          file: test.file,
-      const counter = this._files.get(test.file);
+          configuredFile,
-      this._files.set(test.file, counter + 1);
+          ordinals: [],
-      this._tests.set(`${test.file}::${counter}`, test);
+          configurationObject: test.__configurationObject,
+          configurationString: test.__configurationString
         });
       }
+      const { ordinals } = this._testsByConfiguredFile.get(configuredFile);
+      ordinals.push(test.__ordinal);
+      this._testById.set(`${test.__ordinal}@${configuredFile}`, test);
+    });
+
+    if (process.stdout.isTTY) {
+      console.log();
+      const jobs = Math.min(options.jobs, this._testsByConfiguredFile.size);
+      console.log(`Running ${total} test${ total > 1 ? 's' : '' } using ${jobs} worker${ jobs > 1 ? 's' : ''}`);
+    }
+  }
 
   _filesSortedByWorkerHash() {
     const result = [];
-    for (const [file, count] of this._files.entries())
+    for (const entry of this._testsByConfiguredFile.values())
-      result.push({ file, hash: computeWorkerHash(file), ordinals: new Array(count).fill(0).map((_, i) => i) });
+      result.push({ ...entry, hash: entry.configurationString + '@' + computeWorkerHash(entry.file) });
     result.sort((a, b) => a.hash < b.hash ? -1 : (a.hash === b.hash ? 0 : 1));
     return result;
   }
@@ -170,7 +176,7 @@ class Runner extends EventEmitter {
   }
 
   _updateTest(serialized) {
-    const test = this._tests.get(serialized.id);
+    const test = this._testById.get(serialized.id);
     test.duration = serialized.duration;
     return test;
   }
@@ -228,7 +234,7 @@ class OopWorker extends EventEmitter {
 
   run(entry) {
     this.hash = entry.hash;
-    this.process.send({ method: 'run', params: { file: entry.file, ordinals: entry.ordinals, options: this.runner._options } });
+    this.process.send({ method: 'run', params: { entry, options: this.runner._options } });
   }
 
   stop() {
@@ -252,7 +258,7 @@ class InProcessWorker extends EventEmitter {
   constructor(runner) {
     super();
     this.runner = runner;
-    this.fixturePool = require('./fixturesUI').fixturePool;
+    this.fixturePool = require('./testRunner').fixturePool;
   }
 
   async init() {
@@ -265,7 +271,7 @@ class InProcessWorker extends EventEmitter {
   async run(entry) {
     delete require.cache[entry.file];
     const { TestRunner } = require('./testRunner');
-    const testRunner = new TestRunner(entry.file, entry.ordinals, this.runner._options);
+    const testRunner = new TestRunner(entry, this.runner._options);
     for (const event of ['test', 'pending', 'pass', 'fail', 'done'])
       testRunner.on(event, this.emit.bind(this, event));
     testRunner.run();

test/runner/testCollector.js

@@ -0,0 +1,135 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const path = require('path');
+const Mocha = require('mocha');
+const { fixturesForCallback, generatorRegistrations } = require('./fixtures');
+const { fixturesUI } = require('./fixturesUI');
+
+global.testOptions = require('./testOptions');
+
+class NullReporter {}
+
+class TestCollector {
+  constructor(options) {
+    this._options = options;
+    this.suite = new Mocha.Suite('', new Mocha.Context(), true);
+    this._total = 0;
+    if (options.grep) {
+      const match = options.grep.match(/^\/(.*)\/(g|i|)$|.*/);
+      this._grep = new RegExp(match[1] || match[0], match[2]);
+    }
+  }
+
+  addFile(file) {
+    const mocha = new Mocha({
+      forbidOnly: this._options.forbidOnly,
+      reporter: NullReporter,
+      timeout: this._options.timeout,
+      ui: fixturesUI.bind(null, {
+        testWrapper: (fn) => done => done(),
+        hookWrapper: (hook, fn) => {},
+        ignoreOnly: false,
+      }),
+    });
+    mocha.addFile(file);
+    mocha.loadFiles();
+
+    const workerGeneratorConfigurations = new Map();
+
+    let ordinal = 0;
+    mocha.suite.eachTest(test => {
+      // All tests are identified with their ordinals.
+      test.__ordinal = ordinal++;
+
+      // Get all the fixtures that the test needs.
+      const fixtures = fixturesForCallback(test.fn.__original);
+
+      // For generator fixtures, collect all variants of the fixture values
+      // to build different workers for them.
+      const generatorConfigurations = [];
+      for (const name of fixtures) {
+        if (!generatorRegistrations.has(name))
+          continue;
+        const values = generatorRegistrations.get(name)();
+        let state = generatorConfigurations.length ? generatorConfigurations.slice() : [[]];
+        generatorConfigurations.length = 0;
+        for (const gen of state) {
+          for (const value of values)
+            generatorConfigurations.push([...gen, { name, value }]);
+        }
+      }
+
+      // No generator fixtures for test, include empty set.
+      if (!generatorConfigurations.length)
+        generatorConfigurations.push([]);
+
+      for (const configurationObject of generatorConfigurations) {
+        // Serialize configuration as readable string, we will use it as a hash.
+        const tokens = [];
+        for (const { name, value } of configurationObject)
+          tokens.push(`${name}=${value}`);
+        const configurationString = tokens.join(', ');
+        // Allocate worker for this configuration, add test into it.
+        if (!workerGeneratorConfigurations.has(configurationString))
+          workerGeneratorConfigurations.set(configurationString, { configurationObject, configurationString, tests: new Set() });
+        workerGeneratorConfigurations.get(configurationString).tests.add(test);
+      }
+    });
+
+    if (mocha.suite.hasOnly())
+      mocha.suite.filterOnly();
+
+    // Clone the suite as many times as there are worker hashes.
+    // Only include the tests that requested these generations.
+    for (const [hash, {configurationObject, configurationString, tests}] of workerGeneratorConfigurations.entries()) {
+      const clone = this._cloneSuite(mocha.suite, configurationObject, configurationString, tests);
+      this.suite.addSuite(clone);
+      clone.title = path.basename(file) + (hash.length ? `::[${hash}]` : '');
+    }
+  }
+
+  _cloneSuite(suite, configurationObject, configurationString, tests) {
+    const copy = suite.clone();
+    copy.__configurationObject = configurationObject;
+    for (const child of suite.suites)
+      copy.addSuite(this._cloneSuite(child, configurationObject, configurationString, tests));
+    for (const test of suite.tests) {
+      if (!tests.has(test))
+        continue;
+      if (this._grep && !this._grep.test(test.fullTitle()))
+        continue;
+      const testCopy = test.clone();
+      testCopy.__ordinal = test.__ordinal;
+      testCopy.__configurationObject = configurationObject;
+      testCopy.__configurationString = configurationString;
+      copy.addTest(testCopy);
+    }
+    return copy;
+  } 
+}
+
+
+function grepTotal(mocha, suite) {
+  let total = 0;
+  suite.eachTest(test => {
+    if (mocha.options.grep.test(test.fullTitle()))
+      total++;
+  });
+  return total;
+}
+
+module.exports = { TestCollector };

test/runner/testRunner.js

@@ -16,9 +16,11 @@
 
 const path = require('path');
 const Mocha = require('mocha');
+const { FixturePool, rerunRegistrations, fixturesForCallback } = require('./fixtures');
 const { fixturesUI } = require('./fixturesUI');
 const { EventEmitter } = require('events');
 
+const fixturePool = new FixturePool();
 global.expect = require('expect');
 global.testOptions = require('./testOptions');
 const GoldenUtils = require('./GoldenUtils');
@@ -26,27 +28,34 @@ const GoldenUtils = require('./GoldenUtils');
 class NullReporter {}
 
 class TestRunner extends EventEmitter {
-  constructor(file, ordinals, options) {
+  constructor(entry, options) {
     super();
     this.mocha = new Mocha({
-      forbidOnly: options.forbidOnly,
       reporter: NullReporter,
       timeout: options.timeout,
-      ui: fixturesUI.bind(null, this),
+      ui: fixturesUI.bind(null, {
+        testWrapper: fn => this._testWrapper(fn),
+        hookWrapper: (hook, fn) => this._hookWrapper(hook, fn),
+        ignoreOnly: true
+      }),
     });
-    if (options.grep)
-      this.mocha.grep(options.grep);
     this._currentOrdinal = -1;
     this._failedWithError = false;
-    this._ordinals = new Set(ordinals);
+    this._file = entry.file;
-    this._remaining = new Set(ordinals);
+    this._ordinals = new Set(entry.ordinals);
+    this._remaining = new Set(entry.ordinals);
     this._trialRun = options.trialRun;
     this._passes = 0;
     this._failures = 0;
     this._pending = 0;
-    this._relativeTestFile = path.relative(options.testDir, file);
+    this._configuredFile = entry.configuredFile;
-    this.mocha.addFile(file);
+    this._configurationObject = entry.configurationObject;
-    this.mocha.suite.filterOnly();
+    this._configurationString = entry.configurationString;
+    this._parsedGeneratorConfiguration = new Map();
+    for (const {name, value} of this._configurationObject)
+      this._parsedGeneratorConfiguration.set(name, value);
+    this._relativeTestFile = path.relative(options.testDir, this._file);
+    this.mocha.addFile(this._file);
     this.mocha.loadFiles();
     this.suite = this.mocha.suite;
   }
@@ -54,6 +63,9 @@ class TestRunner extends EventEmitter {
   async run() {
     let callback;
     const result = new Promise(f => callback = f);
+    rerunRegistrations(this._file, 'test');
+    for (const [name, value] of this._parsedGeneratorConfiguration)
+      fixturePool.generators.set(name, value);
     const runner = this.mocha.run(callback);
 
     const constants = Mocha.Runner.constants;
@@ -65,7 +77,7 @@ class TestRunner extends EventEmitter {
       if (this._ordinals.size && !this._ordinals.has(ordinal))
         return;
       this._remaining.delete(ordinal);
-      this.emit('test', { test: serializeTest(test, ordinal) });
+      this.emit('test', { test: this._serializeTest(test, ordinal) });
     });
 
     runner.on(constants.EVENT_TEST_PENDING, test => {
@@ -76,7 +88,7 @@ class TestRunner extends EventEmitter {
         return;
       this._remaining.delete(ordinal);
       ++this._pending;
-      this.emit('pending', { test: serializeTest(test, ordinal) });
+      this.emit('pending', { test: this._serializeTest(test, ordinal) });
     });
 
     runner.on(constants.EVENT_TEST_PASS, test => {
@@ -87,7 +99,7 @@ class TestRunner extends EventEmitter {
       if (this._ordinals.size && !this._ordinals.has(ordinal))
         return;
       ++this._passes;
-      this.emit('pass', { test: serializeTest(test, ordinal) });
+      this.emit('pass', { test: this._serializeTest(test, ordinal) });
     });
 
     runner.on(constants.EVENT_TEST_FAIL, (test, error) => {
@@ -96,7 +108,7 @@ class TestRunner extends EventEmitter {
       ++this._failures;
       this._failedWithError = error;
       this.emit('fail', {
-        test: serializeTest(test, this._currentOrdinal),
+        test: this._serializeTest(test, this._currentOrdinal),
         error: serializeError(error),
       });
     });
@@ -112,7 +124,7 @@ class TestRunner extends EventEmitter {
     await result;
   }
 
-  shouldRunTest(hook) {
+  _shouldRunTest(hook) {
     if (this._trialRun || this._failedWithError)
       return false;
     if (hook) {
@@ -126,13 +138,30 @@ class TestRunner extends EventEmitter {
     return true;
   }
 
-  grepTotal() {
+  _testWrapper(fn) {
-    let total = 0;
+    const wrapped = fixturePool.wrapTestCallback(fn);
-    this.suite.eachTest(test => {
+    return wrapped ? (done, ...args) => {
-      if (this.mocha.options.grep.test(test.fullTitle()))
+      if (!this._shouldRunTest()) {
-        total++;
+        done();
+        return;
+      }
+      wrapped(...args).then(done).catch(done);
+    } : undefined;
+  }
+
+  _hookWrapper(hook, fn) {
+    if (!this._shouldRunTest(true))
+      return;
+    return hook(async () => {
+      return await fixturePool.resolveParametersAndRun(fn);
     });
-    return total;
+  }
+
+  _serializeTest(test, ordinal) {
+    return {
+      id: `${ordinal}@${this._configuredFile}`,
+      duration: test.duration,
+    };
   }
   
   _serializeStats(stats) {
@@ -145,17 +174,6 @@ class TestRunner extends EventEmitter {
   }  
 }
 
-function createTestSuite() {
-  return new Mocha.Suite('', new Mocha.Context(), true);
-}
-
-function serializeTest(test, origin) {
-  return {
-    id: `${test.file}::${origin}`,
-    duration: test.duration,
-  };
-}
-
 function trimCycles(obj) {
   const cache = new Set();
   return JSON.parse(
@@ -190,4 +208,4 @@ function initializeImageMatcher(options) {
   global.expect.extend({ toMatchImage });
 }
 
-module.exports = { TestRunner, createTestSuite, initializeImageMatcher };
+module.exports = { TestRunner, initializeImageMatcher, fixturePool };

test/runner/worker.js

@@ -14,9 +14,8 @@
  * limitations under the License.
  */
 
-const { fixturePool } = require('./fixturesUI');
 const { gracefullyCloseAll } = require('../../lib/server/processLauncher');
-const { TestRunner, initializeImageMatcher } = require('./testRunner');
+const { TestRunner, initializeImageMatcher, fixturePool } = require('./testRunner');
 const { initializeWorker } = require('./builtin.fixtures');
 
 const util = require('util');
@@ -57,7 +56,7 @@ process.on('message', async message => {
     return;
   }
   if (message.method === 'run') {
-    const testRunner = new TestRunner(message.params.file, message.params.ordinals, message.params.options);
+    const testRunner = new TestRunner(message.params.entry, message.params.options);
     for (const event of ['test', 'pending', 'pass', 'fail', 'done'])
       testRunner.on(event, sendMessageToParent.bind(null, event));
     await testRunner.run();