test: support --grep, --forbid-only (#3417)

.github/workflows/tests.yml

@@ -38,7 +38,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --max-workers=1 --timeout=30000 && npm run coverage"
+    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --jobs=1 --forbid-only --timeout=30000 && npm run coverage"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
@@ -69,7 +69,7 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test/runner --max-workers=1 --timeout=30000
+    - run: node test/runner --jobs=1 --forbid-only --timeout=30000
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"
@@ -103,7 +103,7 @@ jobs:
     - uses: microsoft/playwright-github-action@v1
     - run: npm ci
     - run: npm run build
-    - run: node test/runner --max-workers=1 --timeout=30000
+    - run: node test/runner --jobs=1 --forbid-only --timeout=30000
       shell: bash
       env:
         BROWSER: ${{ matrix.browser }}
@@ -160,7 +160,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --max-workers=1 --timeout=30000"
+    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --jobs=1 --forbid-only --timeout=30000"
       if: ${{ always() }}
       env:
         BROWSER: ${{ matrix.browser }}
@@ -194,7 +194,7 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --max-workers=1 --timeout=30000"
+    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --jobs=1 --forbid-only --timeout=30000"
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "pw:*,-pw:wrapped*,-pw:test*"

test/runner/fixtures.js

@@ -1,152 +0,0 @@
-/**
- * Copyright Microsoft Corporation. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-const path = require('path');
-const childProcess = require('child_process');
-
-const { TestServer } = require('../../utils/testserver');
-const { Connection } = require('../../lib/rpc/client/connection');
-const { Transport } = require('../../lib/rpc/transport');
-const { setUnderTest } = require('../../lib/helper');
-const { valueFromEnv } = require('./utils');
-const { registerFixture, registerWorkerFixture } = require('./fixturePool');
-
-setUnderTest();  // Note: we must call setUnderTest before requiring Playwright.
-
-const browserName = process.env.BROWSER || 'chromium';
-
-module.exports = function registerFixtures(global) {
-  registerWorkerFixture('parallelIndex', async ({}, test) => {
-    await test(process.env.JEST_WORKER_ID - 1);
-  });
-
-  registerWorkerFixture('http_server', async ({parallelIndex}, test) => {
-    const assetsPath = path.join(__dirname, '..', 'assets');
-    const cachedPath = path.join(__dirname, '..', 'assets', 'cached');
-
-    const port = 8907 + parallelIndex * 2;
-    const server = await TestServer.create(assetsPath, port);
-    server.enableHTTPCache(cachedPath);
-    server.PORT = port;
-    server.PREFIX = `http://localhost:${port}`;
-    server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`;
-    server.EMPTY_PAGE = `http://localhost:${port}/empty.html`;
-
-    const httpsPort = port + 1;
-    const httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort);
-    httpsServer.enableHTTPCache(cachedPath);
-    httpsServer.PORT = httpsPort;
-    httpsServer.PREFIX = `https://localhost:${httpsPort}`;
-    httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`;
-    httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`;
-
-    await test({server, httpsServer});
-
-    await Promise.all([
-      server.stop(),
-      httpsServer.stop(),
-    ]);
-  });
-
-  registerWorkerFixture('defaultBrowserOptions', async({}, test) => {
-    let executablePath = undefined;
-    if (browserName === 'chromium' && process.env.CRPATH)
-      executablePath = process.env.CRPATH;
-    if (browserName === 'firefox' && process.env.FFPATH)
-      executablePath = process.env.FFPATH;
-    if (browserName === 'webkit' && process.env.WKPATH)
-      executablePath = process.env.WKPATH;
-    if (executablePath)
-      console.error(`Using executable at ${executablePath}`);
-    await test({
-      handleSIGINT: false,
-      slowMo: valueFromEnv('SLOW_MO', 0),
-      headless: !!valueFromEnv('HEADLESS', true),
-      executablePath
-    });
-  });
-
-  registerWorkerFixture('playwright', async({}, test) => {
-    if (process.env.PWCHANNEL === 'wire') {
-      const connection = new Connection();
-      const spawnedProcess = childProcess.fork(path.join(__dirname, '..', '..', 'lib', 'rpc', 'server'), [], {
-        stdio: 'pipe',
-        detached: true,
-      });
-      spawnedProcess.unref();
-      const onExit = (exitCode, signal) => {
-        throw new Error(`Server closed with exitCode=${exitCode} signal=${signal}`);
-      };
-      spawnedProcess.on('exit', onExit);
-      const transport = new Transport(spawnedProcess.stdin, spawnedProcess.stdout);
-      connection.onmessage = message => transport.send(JSON.stringify(message));
-      transport.onmessage = message => connection.dispatch(JSON.parse(message));
-      const playwrightObject = await connection.waitForObjectWithKnownName('Playwright');
-      await test(playwrightObject);
-      spawnedProcess.removeListener('exit', onExit);
-      spawnedProcess.stdin.destroy();
-      spawnedProcess.stdout.destroy();
-      spawnedProcess.stderr.destroy();
-    } else {
-      await test(require('../../index'));
-    }
-  });
-
-  registerFixture('toImpl', async ({playwright}, test) => {
-    await test(playwright._toImpl);
-  });
-
-  registerWorkerFixture('browserType', async ({playwright}, test) => {
-    await test(playwright[process.env.BROWSER || 'chromium']);
-  });
-
-  registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => {
-    const browser = await browserType.launch(defaultBrowserOptions);
-    try {
-      await test(browser);
-      if (browser.contexts().length !== 0) {
-        console.warn(`\nWARNING: test did not close all created contexts! ${new Error().stack}\n`);
-        await Promise.all(browser.contexts().map(context => context.close()));
-      }
-    } finally {
-      await browser.close();
-    }
-  });
-
-  registerFixture('context', async ({browser}, test) => {
-    const context = await browser.newContext();
-    try {
-      await test(context);
-    } finally {
-      await context.close();
-    }
-  });
-
-  registerFixture('page', async ({context}, test) => {
-    const page = await context.newPage();
-    await test(page);
-  });
-
-  registerFixture('server', async ({http_server}, test) => {
-    http_server.server.reset();
-    await test(http_server.server);
-  });
-
-  registerFixture('httpsServer', async ({http_server}, test) => {
-    http_server.httpsServer.reset();
-    await test(http_server.httpsServer);
-  });
-}

test/runner/index.js

@@ -27,40 +27,53 @@ class NullReporter {}
 
 program
   .version('Version ' + require('../../package.json').version)
-  .option('--timeout <timeout>', 'timeout', 10000)
-  .option('--reporter <reporter>', 'reporter to use', '')
-  .option('--max-workers <maxWorkers>', 'max workers to use', Math.ceil(require('os').cpus().length / 2))
-  .option('--retries <retries>', 'number of times to retry a failing test', 1)
+  .option('--forbid-only', 'Fail if exclusive test(s) encountered', false)
+  .option('-g, --grep <grep>', 'Only run tests matching this string or regexp', '.*')
+  .option('-j, --jobs <jobs>', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2))
+  .option('--reporter <reporter>', 'Specify reporter to use', '')
+  .option('--timeout <timeout>', 'Specify test timeout threshold (in milliseconds), default: 10000', 10000)
   .action(async (command) => {
     // Collect files
     const files = [];
     collectFiles(path.join(process.cwd(), 'test'), command.args, files);
     const rootSuite = new Mocha.Suite('', new Mocha.Context(), true);
 
-    console.log(`Transpiling ${files.length} test files`);
+    console.log(`Parsing ${files.length} test files`);
+    let total = 0;
     // Build the test model, suite per file.
     for (const file of files) {
       const mocha = new Mocha({
-        ui: fixturesUI.bind(null, true),
+        forbidOnly: command.forbidOnly || undefined,
+        reporter: NullReporter,
         retries: command.retries,
         timeout: command.timeout,
-        reporter: NullReporter
+        ui: fixturesUI.bind(null, true),
       });
+      if (command.grep)
+        mocha.grep(command.grep);
       mocha.addFile(file);
-      mocha.suite.title = path.basename(file);
-      mocha.suite.root = false;
-      rootSuite.suites.push(mocha.suite);
+      let runner;
       await new Promise(f => {
-        const runner = mocha.run(f);
+        runner = mocha.run(f);
         runner.on(constants.EVENT_RUN_BEGIN, () => {
           process.stdout.write(colors.yellow('\u00B7'));
         });
       });
+      total += runner.grepTotal(mocha.suite);
+
+      rootSuite.addSuite(mocha.suite);
+      mocha.suite.title = path.basename(file);
     }
+
+    if (rootSuite.hasOnly())
+      rootSuite.filterOnly();
     console.log();
+    total = Math.min(total, rootSuite.total()); // First accounts for grep, second for only.
+    console.log(`Running ${total} tests using ${Math.min(command.jobs, total)} workers`);
 
     const runner = new Runner(rootSuite, {
-      maxWorkers: command.maxWorkers,
+      grep: command.grep,
+      jobs: command.jobs,
       reporter: command.reporter,
       retries: command.retries,
       timeout: command.timeout,

test/runner/runner.js

@@ -30,7 +30,7 @@ class Runner extends EventEmitter {
     super();
     this._suite = suite;
     this._options = options;
-    this._maxWorkers = options.maxWorkers;
+    this._jobs = options.jobs;
     this._workers = new Set();
     this._freeWorkers = [];
     this._workerClaimers = [];
@@ -49,9 +49,6 @@ class Runner extends EventEmitter {
     this._tests = new Map();
     this._files = new Map();
 
-    if (suite.hasOnly())
-      suite.filterOnly();
-    console.log(`Running ${suite.total()} tests using ${this._maxWorkers} workers`);
     this._traverse(suite);
   }
 
@@ -109,7 +106,7 @@ class Runner extends EventEmitter {
     if (this._freeWorkers.length)
       return this._freeWorkers.pop();
     // If we can create worker, create it.
-    if (this._workers.size < this._maxWorkers)
+    if (this._workers.size < this._jobs)
       this._createWorker();
     // Wait for the next available worker.
     await new Promise(f => this._workerClaimers.push(f));

test/runner/worker.js

@@ -67,6 +67,8 @@ async function runSingleTest(file, options) {
     timeout: options.timeout,
     reporter: NullReporter
   });
+  if (options.grep)
+    mocha.grep(options.grep);
   mocha.addFile(file);
   mocha.suite.filterOnly();