diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index ec173143cb..a3df355b1a 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -31,6 +31,7 @@ import type { TraceMode } from '../types/test'; import { builtInReporters, defaultReporter, defaultTimeout } from './common/config'; import type { FullConfigInternal } from './common/config'; import program from 'playwright-core/lib/cli/program'; +import type { ReporterDescription } from '..'; function addTestCommand(program: Command) { const command = program.command('test [test-filter...]'); @@ -98,7 +99,7 @@ function addMergeReportsCommand(program: Command) { } }); command.option('-c, --config ', `Configuration file. Can be used to specify additional configuration for the output report.`); - command.option('--reporter ', 'Output report type', 'list'); + command.option('--reporter ', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`); command.addHelpText('afterAll', ` Arguments [dir]: Directory containing blob reports. @@ -181,7 +182,12 @@ async function mergeReports(reportDir: string | undefined, opts: { [key: string] const dir = path.resolve(process.cwd(), reportDir || ''); if (!(await fs.promises.stat(dir)).isDirectory()) throw new Error('Directory does not exist: ' + dir); - await createMergedReport(config, dir, opts.reporter || ['list']); + let reporterDescriptions: ReporterDescription[] | undefined = resolveReporterOption(opts.reporter); + if (!reporterDescriptions && configFile) + reporterDescriptions = config.config.reporter; + if (!reporterDescriptions) + reporterDescriptions = [[defaultReporter]]; + await createMergedReport(config, dir, reporterDescriptions!); } function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides { @@ -195,7 +201,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid quiet: options.quiet ? options.quiet : undefined, repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined, retries: options.retries ? parseInt(options.retries, 10) : undefined, - reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [resolveReporter(r)]) : undefined, + reporter: resolveReporterOption(options.reporter), shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined, timeout: options.timeout ? parseInt(options.timeout, 10) : undefined, ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined, @@ -233,6 +239,12 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid return overrides; } +function resolveReporterOption(reporter?: string): ReporterDescription[] | undefined { + if (!reporter || !reporter.length) + return undefined; + return reporter.split(',').map((r: string) => [resolveReporter(r)]); +} + function resolveReporter(id: string) { if (builtInReporters.includes(id as any)) return id; diff --git a/packages/playwright-test/src/common/config.ts b/packages/playwright-test/src/common/config.ts index 3571c37afe..f91891e82e 100644 --- a/packages/playwright-test/src/common/config.ts +++ b/packages/playwright-test/src/common/config.ts @@ -84,7 +84,7 @@ export class FullConfigInternal { maxFailures: takeFirst(configCLIOverrides.maxFailures, config.maxFailures, 0), metadata: takeFirst(config.metadata, {}), preserveOutput: takeFirst(config.preserveOutput, 'always'), - reporter: takeFirst(configCLIOverrides.reporter ? toReporters(configCLIOverrides.reporter as any) : undefined, resolveReporters(config.reporter, configDir), [[defaultReporter]]), + reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(config.reporter, configDir), [[defaultReporter]]), reportSlowTests: takeFirst(config.reportSlowTests, { max: 5, threshold: 15000 }), quiet: takeFirst(configCLIOverrides.quiet, config.quiet, false), projects: [], diff --git a/packages/playwright-test/src/common/ipc.ts b/packages/playwright-test/src/common/ipc.ts index 9b3982d445..1be9668bdb 100644 --- a/packages/playwright-test/src/common/ipc.ts +++ b/packages/playwright-test/src/common/ipc.ts @@ -16,7 +16,7 @@ import { serializeCompilationCache } from './compilationCache'; import type { FullConfigInternal } from './config'; -import type { TestInfoError, TestStatus } from '../../types/test'; +import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test'; export type ConfigCLIOverrides = { forbidOnly?: boolean; @@ -27,7 +27,7 @@ export type ConfigCLIOverrides = { quiet?: boolean; repeatEach?: number; retries?: number; - reporter?: string; + reporter?: ReporterDescription[]; shard?: { current: number, total: number }; timeout?: number; ignoreSnapshots?: boolean; diff --git a/packages/playwright-test/src/reporters/blob.ts b/packages/playwright-test/src/reporters/blob.ts index 721356d748..02ef397cfe 100644 --- a/packages/playwright-test/src/reporters/blob.ts +++ b/packages/playwright-test/src/reporters/blob.ts @@ -30,6 +30,7 @@ import { createReporters } from '../runner/reporters'; import { defaultReportFolder } from './html'; import { TeleReporterEmitter } from './teleEmitter'; import { Multiplexer } from './multiplexer'; +import type { ReporterDescription } from '../../types/test'; type BlobReporterOptions = { @@ -105,7 +106,7 @@ export class BlobReporter extends TeleReporterEmitter { } } -export async function createMergedReport(config: FullConfigInternal, dir: string, reporterNames: string[]) { +export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[]) { const shardFiles = await sortedShardFiles(dir); const resourceDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-report-')); await fs.promises.mkdir(resourceDir, { recursive: true }); @@ -114,7 +115,7 @@ export async function createMergedReport(config: FullConfigInternal, dir: string const events = mergeEvents(shardReports); patchAttachmentPaths(events, resourceDir); - const reporters = await createReporters(config, 'merge', reporterNames); + const reporters = await createReporters(config, 'merge', reporterDescriptions); const receiver = new TeleReporterReceiver(path.sep, new Multiplexer(reporters)); for (const event of events) await receiver.dispatch(event); diff --git a/packages/playwright-test/src/reporters/teleEmitter.ts b/packages/playwright-test/src/reporters/teleEmitter.ts index 33945e5dd2..5ba554bf0d 100644 --- a/packages/playwright-test/src/reporters/teleEmitter.ts +++ b/packages/playwright-test/src/reporters/teleEmitter.ts @@ -126,7 +126,7 @@ export class TeleReporterEmitter implements Reporter { return { rootDir: config.rootDir, configFile: this._relativePath(config.configFile), - listOnly: FullConfigInternal.from(config).cliListOnly, + listOnly: FullConfigInternal.from(config)?.cliListOnly, workers: config.workers, }; } diff --git a/packages/playwright-test/src/runner/reporters.ts b/packages/playwright-test/src/runner/reporters.ts index a4bb24e59e..c1a09a38ef 100644 --- a/packages/playwright-test/src/runner/reporters.ts +++ b/packages/playwright-test/src/runner/reporters.ts @@ -31,7 +31,7 @@ import { loadReporter } from './loadUtils'; import { BlobReporter } from '../reporters/blob'; import type { ReporterDescription } from '../../types/test'; -export async function createReporters(config: FullConfigInternal, mode: 'list' | 'run' | 'ui' | 'merge', reporterNames?: string[]): Promise { +export async function createReporters(config: FullConfigInternal, mode: 'list' | 'run' | 'ui' | 'merge', descriptions?: ReporterDescription[]): Promise { const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { dot: mode === 'list' ? ListModeReporter : DotReporter, line: mode === 'list' ? ListModeReporter : LineReporter, @@ -44,9 +44,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' | blob: BlobReporter, }; const reporters: Reporter[] = []; - const descriptions: ReporterDescription[] = reporterNames ? - reporterNames.map(name => [name, config.config.reporter.find(([reporterName]) => reporterName === name)]) : - config.config.reporter; + descriptions ??= config.config.reporter; for (const r of descriptions) { const [name, arg] = r; const options = { ...arg, configDir: config.configDir }; diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index d8a2efc100..e0c5fb9040 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -354,7 +354,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); expect(reportFiles).toEqual(['report-1-of-2.zip']); - const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html', '--reporter', 'line'] }); + const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html,line'] }); expect(exitCode).toBe(0); // Check that line reporter was called. @@ -366,4 +366,69 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport // Check html report presence. await showReport(); await expect(page.getByText('first')).toBeVisible(); +}); + +test('multiple output reports based on config', async ({ runInlineTest, mergeReports }) => { + test.slow(); + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'merged/playwright.config.ts': ` + module.exports = { + reporter: [['blob', { outputDir: 'merged-blob' }], ['html', { outputFolder: 'html' }], ['line']] + }; + `, + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + import fs from 'fs'; + + test('first', async ({}) => { + const attachmentPath = test.info().outputPath('foo.txt'); + fs.writeFileSync(attachmentPath, 'hello!'); + await test.info().attach('file-attachment', {path: attachmentPath}); + + console.log('console info'); + console.error('console error'); + }); + test('failing 1', async ({}) => { + await test.info().attach('text-attachment', { body: 'hi!' }); + expect(1).toBe(2); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + ` + }; + await runInlineTest(files, { shard: `1/2` }); + await runInlineTest(files, { shard: `2/2` }); + + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual(['report-1-of-2.zip', 'report-2-of-2.zip']); + const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] }); + expect(exitCode).toBe(0); + + // Check that line reporter was called. + const text = stripAnsi(output); + expect(text).toContain('Running 6 tests using 2 workers'); + expect(text).toContain('[1/6] a.test.js:5:11 › first'); + expect(text).toContain('a.test.js:13:11 › failing 1 (retry #1)'); + + // Check html report presence. + expect((await fs.promises.stat(test.info().outputPath('merged/html/index.html'))).isFile).toBeTruthy(); + + // Check report presence. + expect((await fs.promises.stat(test.info().outputPath('merged/merged-blob/report.zip'))).isFile).toBeTruthy(); + }); \ No newline at end of file