diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index f376e71352..0847c9597b 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -203,7 +203,7 @@ export class Runner { for (const [project, files] of filesByProject) { report.projects.push({ ...sanitizeConfigForJSON(project, new Set()), - files: files, + files }); } return report; diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index b32d5063d8..e925da62c2 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -30,6 +30,11 @@ import { test as base } from './stable-test-runner'; const removeFolderAsync = promisify(rimraf); +export type CliRunResult = { + exitCode: number, + output: string, +}; + export type RunResult = { exitCode: number, output: string, @@ -113,7 +118,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b } const outputDir = path.join(baseDir, 'test-results'); const reportFile = path.join(outputDir, 'report.json'); - const args = ['node', cliEntrypoint, 'test']; + const args = ['test']; if (!options.usesCustomOutputDir) args.push('--output=' + outputDir); if (!options.usesCustomReporters) @@ -124,12 +129,71 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b ); if (options.additionalArgs) args.push(...options.additionalArgs); - const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'playwright-test-cache-')); + + const cwd = options.cwd ? path.resolve(baseDir, options.cwd) : baseDir; + // eslint-disable-next-line prefer-const + let { exitCode, output } = await runPlaywrightCommand(childProcess, cwd, args, { + PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile, + ...env, + }, options.sendSIGINTAfter); + + const summary = (re: RegExp) => { + let result = 0; + let match = re.exec(output); + while (match) { + result += (+match[1]); + match = re.exec(output); + } + return result; + }; + const passed = summary(/(\d+) passed/g); + const failed = summary(/(\d+) failed/g); + const flaky = summary(/(\d+) flaky/g); + const skipped = summary(/(\d+) skipped/g); + const interrupted = summary(/(\d+) interrupted/g); + let report; + try { + report = JSON.parse(fs.readFileSync(reportFile).toString()); + } catch (e) { + output += '\n' + e.toString(); + } + + const results: JSONReportTestResult[] = []; + function visitSuites(suites?: JSONReportSuite[]) { + if (!suites) + return; + for (const suite of suites) { + for (const spec of suite.specs) { + for (const test of spec.tests) + results.push(...test.results); + } + visitSuites(suite.suites); + } + } + if (report) + visitSuites(report.suites); + + return { + exitCode, + output, + passed, + failed, + flaky, + skipped, + interrupted, + report, + results, + }; +} + +async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess'], cwd: string, commandWithArguments: string[], env: Env, sendSIGINTAfter?: number): Promise { + const command = ['node', cliEntrypoint]; + command.push(...commandWithArguments); + const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-')); const testProcess = childProcess({ - command: args, + command, env: { ...process.env, - PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile, PWTEST_CACHE_DIR: cacheDir, // BEGIN: Reserved CI CI: undefined, @@ -151,11 +215,11 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b NODE_OPTIONS: undefined, ...env, }, - cwd: options.cwd ? path.resolve(baseDir, options.cwd) : baseDir, + cwd, }); let didSendSigint = false; testProcess.onOutput = () => { - if (options.sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= options.sendSIGINTAfter) { + if (sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= sendSIGINTAfter) { didSendSigint = true; process.kill(testProcess.process.pid!, 'SIGINT'); } @@ -163,56 +227,10 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b const { exitCode } = await testProcess.exited; await removeFolderAsync(cacheDir); - const outputString = testProcess.output.toString(); - const summary = (re: RegExp) => { - let result = 0; - let match = re.exec(outputString); - while (match) { - result += (+match[1]); - match = re.exec(outputString); - } - return result; - }; - const passed = summary(/(\d+) passed/g); - const failed = summary(/(\d+) failed/g); - const flaky = summary(/(\d+) flaky/g); - const skipped = summary(/(\d+) skipped/g); - const interrupted = summary(/(\d+) interrupted/g); - let report; - try { - report = JSON.parse(fs.readFileSync(reportFile).toString()); - } catch (e) { - testProcess.output += '\n' + e.toString(); - } - - const results: JSONReportTestResult[] = []; - function visitSuites(suites?: JSONReportSuite[]) { - if (!suites) - return; - for (const suite of suites) { - for (const spec of suite.specs) { - for (const test of spec.tests) - results.push(...test.results); - } - visitSuites(suite.suites); - } - } - if (report) - visitSuites(report.suites); - - return { - exitCode, - output: testProcess.output, - passed, - failed, - flaky, - skipped, - interrupted, - report, - results, - }; + return { exitCode, output: testProcess.output.toString() }; } + type RunOptions = { sendSIGINTAfter?: number; usesCustomOutputDir?: boolean; @@ -226,6 +244,7 @@ type Fixtures = { runTSC: (files: Files) => Promise; nodeVersion: { major: number, minor: number, patch: number }; runGroups: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>; + runCommand: (files: Files, args: string[]) => Promise; }; export const test = base @@ -245,6 +264,13 @@ export const test = base }); }, + runCommand: async ({ childProcess }, use, testInfo: TestInfo) => { + await use(async (files: Files, args: string[]) => { + const baseDir = await writeFiles(testInfo, files); + return await runPlaywrightCommand(childProcess, baseDir, args, { }); + }); + }, + runTSC: async ({ childProcess }, use, testInfo) => { await use(async files => { const baseDir = await writeFiles(testInfo, { 'tsconfig.json': JSON.stringify(TSCONFIG), ...files }); diff --git a/tests/playwright-test/project-setup.spec.ts b/tests/playwright-test/project-setup.spec.ts index 55044ed13d..fe5c818afa 100644 --- a/tests/playwright-test/project-setup.spec.ts +++ b/tests/playwright-test/project-setup.spec.ts @@ -328,3 +328,100 @@ test('same file cannot be a setup and a test in different projects', async ({ ru expect(exitCode).toBe(1); expect(output).toContain(`a.test.ts" matches 'setup' filter in project "p1" and 'testMatch' filter in project "p2"`); }); + +test('list-files should enumerate setup files in same group', async ({ runCommand }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { + name: 'p1', + setup: /.*a..setup.ts$/, + testMatch: /.*a.test.ts$/, + }, + { + name: 'p2', + setup: /.*b.setup.ts$/, + testMatch: /.*b.test.ts$/ + }, + ] + };`, + 'a1.setup.ts': ` + const { test } = pwt; + test('test1', async () => { }); + `, + 'a2.setup.ts': ` + const { test } = pwt; + test('test1', async () => { }); + `, + 'a.test.ts': ` + const { test } = pwt; + test('test2', async () => { }); + `, + 'b.setup.ts': ` + const { test } = pwt; + test('test3', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test4', async () => { }); + `, + }; + + const { exitCode, output } = await runCommand(files, ['list-files']); + expect(exitCode).toBe(0); + const json = JSON.parse(output); + expect(json.projects.map(p => p.name)).toEqual(['p1', 'p2']); + expect(json.projects[0].files.map(f => path.basename(f))).toEqual(['a.test.ts', 'a1.setup.ts', 'a2.setup.ts']); + expect(json.projects[1].files.map(f => path.basename(f))).toEqual(['b.setup.ts', 'b.test.ts']); +}); + +test('test --list should enumerate setup tests as regular ones', async ({ runCommand }, testInfo) => { + const files = { + 'playwright.config.ts': ` + module.exports = { + projects: [ + { + name: 'p1', + setup: /.*a..setup.ts$/, + testMatch: /.*a.test.ts$/, + }, + { + name: 'p2', + setup: /.*b.setup.ts$/, + testMatch: /.*b.test.ts$/ + }, + ] + };`, + 'a1.setup.ts': ` + const { test } = pwt; + test('test1', async () => { }); + `, + 'a2.setup.ts': ` + const { test } = pwt; + test('test1', async () => { }); + `, + 'a.test.ts': ` + const { test } = pwt; + test('test2', async () => { }); + `, + 'b.setup.ts': ` + const { test } = pwt; + test('test3', async () => { }); + `, + 'b.test.ts': ` + const { test } = pwt; + test('test4', async () => { }); + `, + }; + + const { exitCode, output } = await runCommand(files, ['test', '--list']); + expect(exitCode).toBe(0); + expect(output).toContain(`Listing tests: + [p1] › a.test.ts:6:7 › test2 + [p1] › a1.setup.ts:5:7 › test1 + [p1] › a2.setup.ts:5:7 › test1 + [p2] › b.setup.ts:5:7 › test3 + [p2] › b.test.ts:6:7 › test4 +Total: 5 tests in 5 files`); +});