diff --git a/package-lock.json b/package-lock.json index 1c0c24155e..a40989b0b4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3154,9 +3154,9 @@ } }, "commander": { - "version": "2.20.3", - "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", - "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-6.0.0.tgz", + "integrity": "sha512-s7EA+hDtTYNhuXkTlhqew4txMZVdszBmKWSPEMxGr8ru8JXR7bLUFIAtPhcSuFdJQ0ILMxnJi8GkQL0yvDy/YA==", "dev": true }, "commondir": { @@ -10056,6 +10056,12 @@ "source-map-support": "~0.5.12" }, "dependencies": { + "commander": { + "version": "2.20.3", + "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", + "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", + "dev": true + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index 9340b84941..83112fd0c9 100644 --- a/package.json +++ b/package.json @@ -70,6 +70,7 @@ "@typescript-eslint/eslint-plugin": "^2.6.1", "@typescript-eslint/parser": "^2.6.1", "colors": "^1.4.0", + "commander": "^6.0.0", "commonmark": "^0.28.1", "cross-env": "^5.0.5", "electron": "^9.0.0-beta.24", diff --git a/test/jest/playwrightEnvironment.js b/test/jest/playwrightEnvironment.js index 56efc314d8..1a6ae1b41c 100644 --- a/test/jest/playwrightEnvironment.js +++ b/test/jest/playwrightEnvironment.js @@ -15,7 +15,6 @@ */ const { FixturePool, registerFixture, registerWorkerFixture } = require('../harness/fixturePool'); -const os = require('os'); const path = require('path'); const fs = require('fs'); const debug = require('debug'); diff --git a/test/mocha/dot.js b/test/mocha/dotReporter.js similarity index 95% rename from test/mocha/dot.js rename to test/mocha/dotReporter.js index cc3e6b2160..eff3829415 100644 --- a/test/mocha/dot.js +++ b/test/mocha/dotReporter.js @@ -18,7 +18,7 @@ const Base = require('mocha/lib/reporters/base'); const constants = require('mocha/lib/runner').constants; const colors = require('colors/safe'); -class Dot extends Base { +class DotReporter extends Base { constructor(runner, options) { super(runner, options); @@ -48,4 +48,4 @@ class Dot extends Base { } } -module.exports = Dot; +module.exports = DotReporter; diff --git a/test/mocha/index.js b/test/mocha/index.js index 4beeab00e4..68654486ae 100644 --- a/test/mocha/index.js +++ b/test/mocha/index.js @@ -14,38 +14,42 @@ * limitations under the License. */ +const builtinReporters = require('mocha/lib/reporters'); const fs = require('fs'); const path = require('path'); -const Mocha = require('mocha'); -const { fixturesUI, fixturePool } = require('./fixturesUI'); -const dot = require('./dot'); -const { Matchers } = require('../../utils/testrunner/Matchers'); +const program = require('commander'); +const { Runner } = require('./runner'); +const DotRunner = require('./dotReporter'); -const browserName = process.env.BROWSER || 'chromium'; -const goldenPath = path.join(__dirname, '..', 'golden-' + browserName); -const outputPath = path.join(__dirname, '..', 'output-' + browserName); -global.expect = new Matchers({ goldenPath, outputPath }).expect; -global.testOptions = require('../harness/testOptions'); -const mocha = new Mocha({ - ui: fixturesUI, - reporter: dot, - timeout: 10000, -}); -const testDir = path.join(process.cwd(), 'test'); +program + .version('Version ' + require('../../package.json').version) + .option('--reporter ', 'reporter to use', '') + .option('--max-workers ', 'reporter to use', '') + .action(async (command, args) => { + const testDir = path.join(process.cwd(), 'test'); + const files = []; + for (const name of fs.readdirSync(testDir)) { + if (!name.includes('.spec.')) + continue; + if (!command.args.length) { + files.push(path.join(testDir, name)); + continue; + } + for (const filter of command.args) { + if (name.includes(filter)) { + files.push(path.join(testDir, name)); + break; + } + } + } -const filter = process.argv[2]; + const runner = new Runner({ + reporter: command.reporter ? builtinReporters[command.reporter] : DotRunner, + maxWorkers: command.maxWorkers || Math.ceil(require('os').cpus().length / 2) + }); + await runner.run(files); + await runner.stop(); + }); -fs.readdirSync(testDir).filter(function(file) { - return file.includes('.spec.') && (!filter || file.includes(filter)); -}).forEach(function(file) { - mocha.addFile(path.join(testDir, file)); -}); - -const runner = mocha.run((failures) => { - process.exitCode = failures ? 1 : 0; -}); -const constants = Mocha.Runner.constants; -runner.on(constants.EVENT_RUN_END, test => { - fixturePool.teardownScope('worker'); -}); +program.parse(process.argv); diff --git a/test/mocha/runner.js b/test/mocha/runner.js new file mode 100644 index 0000000000..84288ccf77 --- /dev/null +++ b/test/mocha/runner.js @@ -0,0 +1,146 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const child_process = require('child_process'); +const path = require('path'); +const { EventEmitter } = require('events'); +const Mocha = require('mocha'); +const { Serializer } = require('v8'); + +const constants = Mocha.Runner.constants; + +class Runner extends EventEmitter { + constructor(options) { + super(); + this._maxWorkers = options.maxWorkers; + this._workers = new Set(); + this._freeWorkers = []; + this._callbacks = []; + this._workerId = 0; + this.stats = { + duration: 0, + failures: 0, + passes: 0, + pending: 0, + tests: 0, + }; + this._reporter = new options.reporter(this, {}); + } + + async run(files) { + this.emit(constants.EVENT_RUN_BEGIN, {}); + const result = new Promise(f => this._runCallback = f); + for (const file of files) { + const worker = await this._obtainWorker(); + worker.send({ method: 'run', params: file }); + } + await result; + this.emit(constants.EVENT_RUN_END, {}); + } + + async _obtainWorker() { + if (this._freeWorkers.length) + return this._freeWorkers.pop(); + + if (this._workers.size < this._maxWorkers) { + const worker = child_process.fork(path.join(__dirname, 'worker.js'), { + detached: false + }); + let readyCallback; + const result = new Promise(f => readyCallback = f); + worker.send({ method: 'init', params: ++this._workerId }); + worker.on('message', message => { + if (message.method === 'ready') + readyCallback(); + this._messageFromWorker(worker, message); + }); + worker.on('exit', () => { + this._workers.delete(worker); + if (!this._workers.size) + this._runCallback(); + }); + this._workers.add(worker); + await result; + return worker; + } + + return new Promise(f => this._callbacks.push(f)); + } + + _messageFromWorker(worker, message) { + const { method, params } = message; + switch (method) { + case 'done': { + if (this._callbacks.length) { + const callback = this._callbacks.shift(); + callback(worker); + } else { + this._freeWorkers.push(worker); + if (this._freeWorkers.length === this._workers.size) { + this._runCallback(); + } + } + break; + } + case 'start': + break; + case 'test': + this.emit(constants.EVENT_TEST_BEGIN, this._parse(params.test)); + break; + case 'pending': + this.emit(constants.EVENT_TEST_PENDING, this._parse(params.test)); + break; + case 'pass': + this.emit(constants.EVENT_TEST_PASS, this._parse(params.test)); + break; + case 'fail': + const test = this._parse(params.test); + this.emit(constants.EVENT_TEST_FAIL, test, params.error); + break; + case 'end': + this.stats.duration += params.stats.duration; + this.stats.failures += params.stats.failures; + this.stats.passes += params.stats.passes; + this.stats.pending += params.stats.pending; + this.stats.tests += params.stats.tests; + break; + } + } + + _parse(serialized) { + return { + ...serialized, + currentRetry: () => serialized.currentRetry, + fullTitle: () => serialized.fullTitle, + slow: () => serialized.slow, + timeout: () => serialized.timeout, + titlePath: () => serialized.titlePath, + isPending: () => serialized.isPending, + parent: { + fullTitle: () => '' + } + }; + } + + async stop() { + const result = new Promise(f => this._stopCallback = f); + for (const worker of this._workers) + worker.send({ method: 'stop' }); + await result; + } +} + +module.exports = { Runner }; diff --git a/test/mocha/worker.js b/test/mocha/worker.js new file mode 100644 index 0000000000..4d89d7b887 --- /dev/null +++ b/test/mocha/worker.js @@ -0,0 +1,166 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); +const Mocha = require('mocha'); +const { fixturesUI } = require('./fixturesUI'); +const { gracefullyCloseAll } = require('../../lib/server/processLauncher'); +const GoldenUtils = require('../../utils/testrunner/GoldenUtils'); + +const browserName = process.env.BROWSER || 'chromium'; +const goldenPath = path.join(__dirname, '..', 'golden-' + browserName); +const outputPath = path.join(__dirname, '..', 'output-' + browserName); +global.expect = require('expect'); +global.testOptions = require('../harness/testOptions'); + +extendExpects(); + +let closed = false; + +process.on('message', async message => { + if (message.method === 'init') + process.env.JEST_WORKER_ID = message.params; + if (message.method === 'stop') + gracefullyCloseAndExit(); + if (message.method === 'run') + await runSingleTest(message.params); +}); + +process.on('disconnect', gracefullyCloseAndExit); +process.on('SIGINT',() => {}); +process.on('SIGTERM',() => {}); +sendMessageToParent('ready'); + +async function gracefullyCloseAndExit() { + closed = true; + // Force exit after 30 seconds. + setTimeout(() => process.exit(0), 30000); + // Meanwhile, try to gracefully close all browsers. + await gracefullyCloseAll(); + process.exit(0); +} + +class NullReporter {} + +async function runSingleTest(file) { + const mocha = new Mocha({ + ui: fixturesUI, + timeout: 10000, + reporter: NullReporter + }); + mocha.addFile(file); + + const runner = mocha.run(); + + const constants = Mocha.Runner.constants; + runner.on(constants.EVENT_RUN_BEGIN, () => { + sendMessageToParent('start'); + }); + + runner.on(constants.EVENT_TEST_BEGIN, test => { + sendMessageToParent('test', { test: sanitizeTest(test) }); + }); + + runner.on(constants.EVENT_TEST_PENDING, test => { + sendMessageToParent('pending', { test: sanitizeTest(test) }); + }); + + runner.on(constants.EVENT_TEST_PASS, test => { + sendMessageToParent('pass', { test: sanitizeTest(test) }); + }); + + runner.on(constants.EVENT_TEST_FAIL, (test, error) => { + sendMessageToParent('fail', { + test: sanitizeTest(test), + error: serializeError(error), + }); + }); + + runner.once(constants.EVENT_RUN_END, async () => { + sendMessageToParent('end', { stats: serializeStats(runner.stats) }); + sendMessageToParent('done'); + }); +} + +function sendMessageToParent(method, params = {}) { + if (closed) + return; + try { + process.send({ method, params }); + } catch (e) { + // Can throw when closing. + } +} + +function sanitizeTest(test) { + return { + currentRetry: test.currentRetry(), + duration: test.duration, + file: test.file, + fullTitle: test.fullTitle(), + isPending: test.isPending(), + slow: test.slow(), + timeout: test.timeout(), + title: test.title, + titlePath: test.titlePath(), + }; +} + +function serializeStats(stats) { + return { + tests: stats.tests, + passes: stats.passes, + duration: stats.duration, + failures: stats.failures, + pending: stats.pending, + } +} + +function trimCycles(obj) { + const cache = new Set(); + return JSON.parse( + JSON.stringify(obj, function(key, value) { + if (typeof value === 'object' && value !== null) { + if (cache.has(value)) + return '' + value; + cache.add(value); + } + return value; + }) + ); +} + +function serializeError(error) { + if (error instanceof Error) { + return { + message: error.message, + stack: error.stack + } + } + return trimCycles(error); +} + +function extendExpects() { + function toBeGolden(received, goldenName) { + const {pass, message} = GoldenUtils.compare(received, { + goldenPath, + outputPath, + goldenName + }); + return {pass, message: () => message}; + }; + global.expect.extend({ toBeGolden }); +}