diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c804a75bde..16bdb483a6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --max-workers=1 --timeout=30000 && npm run coverage" + - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --jobs=1 --forbid-only --timeout=30000 && npm run coverage" env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" @@ -69,7 +69,7 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test/runner --max-workers=1 --timeout=30000 + - run: node test/runner --jobs=1 --forbid-only --timeout=30000 env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" @@ -103,7 +103,7 @@ jobs: - uses: microsoft/playwright-github-action@v1 - run: npm ci - run: npm run build - - run: node test/runner --max-workers=1 --timeout=30000 + - run: node test/runner --jobs=1 --forbid-only --timeout=30000 shell: bash env: BROWSER: ${{ matrix.browser }} @@ -160,7 +160,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --max-workers=1 --timeout=30000" + - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --jobs=1 --forbid-only --timeout=30000" if: ${{ always() }} env: BROWSER: ${{ matrix.browser }} @@ -194,7 +194,7 @@ jobs: # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR # Wrap `npm run` in a subshell to redirect STDERR to file. # Enable core dumps in the subshell. - - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --max-workers=1 --timeout=30000" + - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && node test/runner --jobs=1 --forbid-only --timeout=30000" env: BROWSER: ${{ matrix.browser }} DEBUG: "pw:*,-pw:wrapped*,-pw:test*" diff --git a/test/runner/fixtures.js b/test/runner/fixtures.js deleted file mode 100644 index a5e2c5867a..0000000000 --- a/test/runner/fixtures.js +++ /dev/null @@ -1,152 +0,0 @@ -/** - * Copyright Microsoft Corporation. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const path = require('path'); -const childProcess = require('child_process'); - -const { TestServer } = require('../../utils/testserver'); -const { Connection } = require('../../lib/rpc/client/connection'); -const { Transport } = require('../../lib/rpc/transport'); -const { setUnderTest } = require('../../lib/helper'); -const { valueFromEnv } = require('./utils'); -const { registerFixture, registerWorkerFixture } = require('./fixturePool'); - -setUnderTest(); // Note: we must call setUnderTest before requiring Playwright. - -const browserName = process.env.BROWSER || 'chromium'; - -module.exports = function registerFixtures(global) { - registerWorkerFixture('parallelIndex', async ({}, test) => { - await test(process.env.JEST_WORKER_ID - 1); - }); - - registerWorkerFixture('http_server', async ({parallelIndex}, test) => { - const assetsPath = path.join(__dirname, '..', 'assets'); - const cachedPath = path.join(__dirname, '..', 'assets', 'cached'); - - const port = 8907 + parallelIndex * 2; - const server = await TestServer.create(assetsPath, port); - server.enableHTTPCache(cachedPath); - server.PORT = port; - server.PREFIX = `http://localhost:${port}`; - server.CROSS_PROCESS_PREFIX = `http://127.0.0.1:${port}`; - server.EMPTY_PAGE = `http://localhost:${port}/empty.html`; - - const httpsPort = port + 1; - const httpsServer = await TestServer.createHTTPS(assetsPath, httpsPort); - httpsServer.enableHTTPCache(cachedPath); - httpsServer.PORT = httpsPort; - httpsServer.PREFIX = `https://localhost:${httpsPort}`; - httpsServer.CROSS_PROCESS_PREFIX = `https://127.0.0.1:${httpsPort}`; - httpsServer.EMPTY_PAGE = `https://localhost:${httpsPort}/empty.html`; - - await test({server, httpsServer}); - - await Promise.all([ - server.stop(), - httpsServer.stop(), - ]); - }); - - registerWorkerFixture('defaultBrowserOptions', async({}, test) => { - let executablePath = undefined; - if (browserName === 'chromium' && process.env.CRPATH) - executablePath = process.env.CRPATH; - if (browserName === 'firefox' && process.env.FFPATH) - executablePath = process.env.FFPATH; - if (browserName === 'webkit' && process.env.WKPATH) - executablePath = process.env.WKPATH; - if (executablePath) - console.error(`Using executable at ${executablePath}`); - await test({ - handleSIGINT: false, - slowMo: valueFromEnv('SLOW_MO', 0), - headless: !!valueFromEnv('HEADLESS', true), - executablePath - }); - }); - - registerWorkerFixture('playwright', async({}, test) => { - if (process.env.PWCHANNEL === 'wire') { - const connection = new Connection(); - const spawnedProcess = childProcess.fork(path.join(__dirname, '..', '..', 'lib', 'rpc', 'server'), [], { - stdio: 'pipe', - detached: true, - }); - spawnedProcess.unref(); - const onExit = (exitCode, signal) => { - throw new Error(`Server closed with exitCode=${exitCode} signal=${signal}`); - }; - spawnedProcess.on('exit', onExit); - const transport = new Transport(spawnedProcess.stdin, spawnedProcess.stdout); - connection.onmessage = message => transport.send(JSON.stringify(message)); - transport.onmessage = message => connection.dispatch(JSON.parse(message)); - const playwrightObject = await connection.waitForObjectWithKnownName('Playwright'); - await test(playwrightObject); - spawnedProcess.removeListener('exit', onExit); - spawnedProcess.stdin.destroy(); - spawnedProcess.stdout.destroy(); - spawnedProcess.stderr.destroy(); - } else { - await test(require('../../index')); - } - }); - - registerFixture('toImpl', async ({playwright}, test) => { - await test(playwright._toImpl); - }); - - registerWorkerFixture('browserType', async ({playwright}, test) => { - await test(playwright[process.env.BROWSER || 'chromium']); - }); - - registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, test) => { - const browser = await browserType.launch(defaultBrowserOptions); - try { - await test(browser); - if (browser.contexts().length !== 0) { - console.warn(`\nWARNING: test did not close all created contexts! ${new Error().stack}\n`); - await Promise.all(browser.contexts().map(context => context.close())); - } - } finally { - await browser.close(); - } - }); - - registerFixture('context', async ({browser}, test) => { - const context = await browser.newContext(); - try { - await test(context); - } finally { - await context.close(); - } - }); - - registerFixture('page', async ({context}, test) => { - const page = await context.newPage(); - await test(page); - }); - - registerFixture('server', async ({http_server}, test) => { - http_server.server.reset(); - await test(http_server.server); - }); - - registerFixture('httpsServer', async ({http_server}, test) => { - http_server.httpsServer.reset(); - await test(http_server.httpsServer); - }); -} diff --git a/test/runner/index.js b/test/runner/index.js index cea63f3e23..716412191b 100644 --- a/test/runner/index.js +++ b/test/runner/index.js @@ -27,40 +27,53 @@ class NullReporter {} program .version('Version ' + require('../../package.json').version) - .option('--timeout ', 'timeout', 10000) - .option('--reporter ', 'reporter to use', '') - .option('--max-workers ', 'max workers to use', Math.ceil(require('os').cpus().length / 2)) - .option('--retries ', 'number of times to retry a failing test', 1) + .option('--forbid-only', 'Fail if exclusive test(s) encountered', false) + .option('-g, --grep ', 'Only run tests matching this string or regexp', '.*') + .option('-j, --jobs ', 'Number of concurrent jobs for --parallel; use 1 to run in serial, default: (number of CPU cores / 2)', Math.ceil(require('os').cpus().length / 2)) + .option('--reporter ', 'Specify reporter to use', '') + .option('--timeout ', 'Specify test timeout threshold (in milliseconds), default: 10000', 10000) .action(async (command) => { // Collect files const files = []; collectFiles(path.join(process.cwd(), 'test'), command.args, files); const rootSuite = new Mocha.Suite('', new Mocha.Context(), true); - console.log(`Transpiling ${files.length} test files`); + console.log(`Parsing ${files.length} test files`); + let total = 0; // Build the test model, suite per file. for (const file of files) { const mocha = new Mocha({ - ui: fixturesUI.bind(null, true), + forbidOnly: command.forbidOnly || undefined, + reporter: NullReporter, retries: command.retries, timeout: command.timeout, - reporter: NullReporter + ui: fixturesUI.bind(null, true), }); + if (command.grep) + mocha.grep(command.grep); mocha.addFile(file); - mocha.suite.title = path.basename(file); - mocha.suite.root = false; - rootSuite.suites.push(mocha.suite); + let runner; await new Promise(f => { - const runner = mocha.run(f); + runner = mocha.run(f); runner.on(constants.EVENT_RUN_BEGIN, () => { process.stdout.write(colors.yellow('\u00B7')); }); }); + total += runner.grepTotal(mocha.suite); + + rootSuite.addSuite(mocha.suite); + mocha.suite.title = path.basename(file); } + + if (rootSuite.hasOnly()) + rootSuite.filterOnly(); console.log(); + total = Math.min(total, rootSuite.total()); // First accounts for grep, second for only. + console.log(`Running ${total} tests using ${Math.min(command.jobs, total)} workers`); const runner = new Runner(rootSuite, { - maxWorkers: command.maxWorkers, + grep: command.grep, + jobs: command.jobs, reporter: command.reporter, retries: command.retries, timeout: command.timeout, diff --git a/test/runner/runner.js b/test/runner/runner.js index 5c33051812..dfde544d5a 100644 --- a/test/runner/runner.js +++ b/test/runner/runner.js @@ -30,7 +30,7 @@ class Runner extends EventEmitter { super(); this._suite = suite; this._options = options; - this._maxWorkers = options.maxWorkers; + this._jobs = options.jobs; this._workers = new Set(); this._freeWorkers = []; this._workerClaimers = []; @@ -49,9 +49,6 @@ class Runner extends EventEmitter { this._tests = new Map(); this._files = new Map(); - if (suite.hasOnly()) - suite.filterOnly(); - console.log(`Running ${suite.total()} tests using ${this._maxWorkers} workers`); this._traverse(suite); } @@ -109,7 +106,7 @@ class Runner extends EventEmitter { if (this._freeWorkers.length) return this._freeWorkers.pop(); // If we can create worker, create it. - if (this._workers.size < this._maxWorkers) + if (this._workers.size < this._jobs) this._createWorker(); // Wait for the next available worker. await new Promise(f => this._workerClaimers.push(f)); diff --git a/test/runner/worker.js b/test/runner/worker.js index c1868a7935..848543e2e9 100644 --- a/test/runner/worker.js +++ b/test/runner/worker.js @@ -67,6 +67,8 @@ async function runSingleTest(file, options) { timeout: options.timeout, reporter: NullReporter }); + if (options.grep) + mocha.grep(options.grep); mocha.addFile(file); mocha.suite.filterOnly();