chore: comma separated reporter names, reporter arg from config (#22693)

packages/playwright-test/src/cli.ts

@@ -31,6 +31,7 @@ import type { TraceMode } from '../types/test';
 import { builtInReporters, defaultReporter, defaultTimeout } from './common/config';
 import type { FullConfigInternal } from './common/config';
 import program from 'playwright-core/lib/cli/program';
+import type { ReporterDescription } from '..';
 
 function addTestCommand(program: Command) {
   const command = program.command('test [test-filter...]');
@@ -98,7 +99,7 @@ function addMergeReportsCommand(program: Command) {
     }
   });
   command.option('-c, --config <file>', `Configuration file. Can be used to specify additional configuration for the output report.`);
-  command.option('--reporter <reporter...>', 'Output report type', 'list');
+  command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`);
   command.addHelpText('afterAll', `
 Arguments [dir]:
   Directory containing blob reports.
@@ -181,7 +182,12 @@ async function mergeReports(reportDir: string | undefined, opts: { [key: string]
   const dir = path.resolve(process.cwd(), reportDir || '');
   if (!(await fs.promises.stat(dir)).isDirectory())
     throw new Error('Directory does not exist: ' + dir);
-  await createMergedReport(config, dir, opts.reporter || ['list']);
+  let reporterDescriptions: ReporterDescription[] | undefined = resolveReporterOption(opts.reporter);
+  if (!reporterDescriptions && configFile)
+    reporterDescriptions = config.config.reporter;
+  if (!reporterDescriptions)
+    reporterDescriptions = [[defaultReporter]];
+  await createMergedReport(config, dir, reporterDescriptions!);
 }
 
 function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrides {
@@ -195,7 +201,7 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
     quiet: options.quiet ? options.quiet : undefined,
     repeatEach: options.repeatEach ? parseInt(options.repeatEach, 10) : undefined,
     retries: options.retries ? parseInt(options.retries, 10) : undefined,
-    reporter: (options.reporter && options.reporter.length) ? options.reporter.split(',').map((r: string) => [resolveReporter(r)]) : undefined,
+    reporter: resolveReporterOption(options.reporter),
     shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
     timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
     ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
@@ -233,6 +239,12 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
   return overrides;
 }
 
+function resolveReporterOption(reporter?: string): ReporterDescription[] | undefined {
+  if (!reporter || !reporter.length)
+    return undefined;
+  return reporter.split(',').map((r: string) => [resolveReporter(r)]);
+}
+
 function resolveReporter(id: string) {
   if (builtInReporters.includes(id as any))
     return id;

packages/playwright-test/src/common/config.ts

@@ -84,7 +84,7 @@ export class FullConfigInternal {
       maxFailures: takeFirst(configCLIOverrides.maxFailures, config.maxFailures, 0),
       metadata: takeFirst(config.metadata, {}),
       preserveOutput: takeFirst(config.preserveOutput, 'always'),
-      reporter: takeFirst(configCLIOverrides.reporter ? toReporters(configCLIOverrides.reporter as any) : undefined, resolveReporters(config.reporter, configDir), [[defaultReporter]]),
+      reporter: takeFirst(configCLIOverrides.reporter, resolveReporters(config.reporter, configDir), [[defaultReporter]]),
       reportSlowTests: takeFirst(config.reportSlowTests, { max: 5, threshold: 15000 }),
       quiet: takeFirst(configCLIOverrides.quiet, config.quiet, false),
       projects: [],

packages/playwright-test/src/common/ipc.ts

@@ -16,7 +16,7 @@
 
 import { serializeCompilationCache } from './compilationCache';
 import type { FullConfigInternal } from './config';
-import type { TestInfoError, TestStatus } from '../../types/test';
+import type { ReporterDescription, TestInfoError, TestStatus } from '../../types/test';
 
 export type ConfigCLIOverrides = {
   forbidOnly?: boolean;
@@ -27,7 +27,7 @@ export type ConfigCLIOverrides = {
   quiet?: boolean;
   repeatEach?: number;
   retries?: number;
-  reporter?: string;
+  reporter?: ReporterDescription[];
   shard?: { current: number, total: number };
   timeout?: number;
   ignoreSnapshots?: boolean;

packages/playwright-test/src/reporters/blob.ts

@@ -30,6 +30,7 @@ import { createReporters } from '../runner/reporters';
 import { defaultReportFolder } from './html';
 import { TeleReporterEmitter } from './teleEmitter';
 import { Multiplexer } from './multiplexer';
+import type { ReporterDescription } from '../../types/test';
 
 
 type BlobReporterOptions = {
@@ -105,7 +106,7 @@ export class BlobReporter extends TeleReporterEmitter {
   }
 }
 
-export async function createMergedReport(config: FullConfigInternal, dir: string, reporterNames: string[]) {
+export async function createMergedReport(config: FullConfigInternal, dir: string, reporterDescriptions: ReporterDescription[]) {
   const shardFiles = await sortedShardFiles(dir);
   const resourceDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-report-'));
   await fs.promises.mkdir(resourceDir, { recursive: true });
@@ -114,7 +115,7 @@ export async function createMergedReport(config: FullConfigInternal, dir: string
     const events = mergeEvents(shardReports);
     patchAttachmentPaths(events, resourceDir);
 
-    const reporters = await createReporters(config, 'merge', reporterNames);
+    const reporters = await createReporters(config, 'merge', reporterDescriptions);
     const receiver = new TeleReporterReceiver(path.sep, new Multiplexer(reporters));
     for (const event of events)
       await receiver.dispatch(event);

packages/playwright-test/src/reporters/teleEmitter.ts

@@ -126,7 +126,7 @@ export class TeleReporterEmitter implements Reporter {
     return {
       rootDir: config.rootDir,
       configFile: this._relativePath(config.configFile),
-      listOnly: FullConfigInternal.from(config).cliListOnly,
+      listOnly: FullConfigInternal.from(config)?.cliListOnly,
       workers: config.workers,
     };
   }

packages/playwright-test/src/runner/reporters.ts

@@ -31,7 +31,7 @@ import { loadReporter } from './loadUtils';
 import { BlobReporter } from '../reporters/blob';
 import type { ReporterDescription } from '../../types/test';
 
-export async function createReporters(config: FullConfigInternal, mode: 'list' | 'run' | 'ui' | 'merge', reporterNames?: string[]): Promise<Reporter[]> {
+export async function createReporters(config: FullConfigInternal, mode: 'list' | 'run' | 'ui' | 'merge', descriptions?: ReporterDescription[]): Promise<Reporter[]> {
   const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = {
     dot: mode === 'list' ? ListModeReporter : DotReporter,
     line: mode === 'list' ? ListModeReporter : LineReporter,
@@ -44,9 +44,7 @@ export async function createReporters(config: FullConfigInternal, mode: 'list' |
     blob: BlobReporter,
   };
   const reporters: Reporter[] = [];
-  const descriptions: ReporterDescription[] = reporterNames ?
+  descriptions ??= config.config.reporter;
-    reporterNames.map(name => [name, config.config.reporter.find(([reporterName]) => reporterName === name)]) :
-    config.config.reporter;
   for (const r of descriptions) {
     const [name, arg] = r;
     const options = { ...arg, configDir: config.configDir };

tests/playwright-test/reporter-blob.spec.ts

@@ -354,7 +354,7 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport
   const reportFiles = await fs.promises.readdir(reportDir);
   reportFiles.sort();
   expect(reportFiles).toEqual(['report-1-of-2.zip']);
-  const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html', '--reporter', 'line'] });
+  const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--reporter', 'html,line'] });
   expect(exitCode).toBe(0);
 
   // Check that line reporter was called.
@@ -367,3 +367,68 @@ test('multiple output reports', async ({ runInlineTest, mergeReports, showReport
   await showReport();
   await expect(page.getByText('first')).toBeVisible();
 });
+
+test('multiple output reports based on config', async ({ runInlineTest, mergeReports }) => {
+  test.slow();
+  const reportDir = test.info().outputPath('blob-report');
+  const files = {
+    'merged/playwright.config.ts': `
+      module.exports = {
+        reporter: [['blob', { outputDir: 'merged-blob' }], ['html', { outputFolder: 'html' }], ['line']]
+      };
+    `,
+    'playwright.config.ts': `
+      module.exports = {
+        retries: 1,
+        reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]]
+      };
+    `,
+    'a.test.js': `
+      import { test, expect } from '@playwright/test';
+      import fs from 'fs';
+
+      test('first', async ({}) => {
+        const attachmentPath = test.info().outputPath('foo.txt');
+        fs.writeFileSync(attachmentPath, 'hello!');
+        await test.info().attach('file-attachment', {path: attachmentPath});
+
+        console.log('console info');
+        console.error('console error');
+      });
+      test('failing 1', async ({}) => {
+        await test.info().attach('text-attachment', { body: 'hi!' });
+        expect(1).toBe(2);
+      });
+      test.skip('skipped 1', async ({}) => {});
+    `,
+    'b.test.js': `
+      import { test, expect } from '@playwright/test';
+      test('math 2', async ({}) => { });
+      test('failing 2', async ({}) => {
+        expect(1).toBe(2);
+      });
+      test.skip('skipped 2', async ({}) => {});
+    `
+  };
+  await runInlineTest(files, { shard: `1/2` });
+  await runInlineTest(files, { shard: `2/2` });
+
+  const reportFiles = await fs.promises.readdir(reportDir);
+  reportFiles.sort();
+  expect(reportFiles).toEqual(['report-1-of-2.zip', 'report-2-of-2.zip']);
+  const { exitCode, output } = await mergeReports(reportDir, { 'PW_TEST_DEBUG_REPORTERS': '1' }, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] });
+  expect(exitCode).toBe(0);
+
+  // Check that line reporter was called.
+  const text = stripAnsi(output);
+  expect(text).toContain('Running 6 tests using 2 workers');
+  expect(text).toContain('[1/6] a.test.js:5:11 › first');
+  expect(text).toContain('a.test.js:13:11 › failing 1 (retry #1)');
+
+  // Check html report presence.
+  expect((await fs.promises.stat(test.info().outputPath('merged/html/index.html'))).isFile).toBeTruthy();
+
+  // Check report presence.
+  expect((await fs.promises.stat(test.info().outputPath('merged/merged-blob/report.zip'))).isFile).toBeTruthy();
+
+});