feat: html reporter sharded option (#20737)

Make sharded report feature an opt-in:

```ts
{
   reporter: [['html', { sharded: true }]]
};
```

#10437

docs/src/test-reporters-js.md

@@ -310,8 +310,37 @@ Or if there is a custom folder name:
 npx playwright show-report my-report
 ```
 
-> The `html` reporter currently does not support merging reports generated across multiple [`--shards`](./test-parallel.md#shard-tests-between-multiple-machines) into a single report. See [this](https://github.com/microsoft/playwright/issues/10437) issue for available third party solutions.
+#### Sharded report
 
+When running tests on [multiple shards](./test-parallel.md#shard-tests-between-multiple-machines), the `html` reporter can automatically show test results from all shards in one page when configured with `sharded: true`.
+
+```js tab=js-js
+// playwright.config.js
+// @ts-check
+
+const { defineConfig } = require('@playwright/test');
+
+module.exports = defineConfig({
+  reporter: [['html', { sharded: true }]],
+});
+```
+
+```js tab=js-ts
+// playwright.config.ts
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  reporter: [['html', { sharded: true }]],
+});
+```
+
+You can use sharded html report combined with a file hosting that allows serving html files.
+
+In your CI recipe, after running tests in each shard, upload all files from `playwright-report` directory to the **same location**. After that you can open `index.html` from the uploaded location directly in the browser.
+
+:::note
+The `html` report for each shard consists of `index.html` and a data file named like `report-003-of-100.zip`. It's ok to overwrite  `index.html` with one another when copying sharded reports to a single directory.
+:::
 
 ### JSON reporter
 

packages/playwright-test/src/reporters/html.ts

@@ -43,6 +43,7 @@ type HtmlReportOpenOption = 'always' | 'never' | 'on-failure';
 type HtmlReporterOptions = {
   outputFolder?: string,
   open?: HtmlReportOpenOption,
+  sharded?: boolean,
   host?: string,
   port?: number,
 };
@@ -53,6 +54,7 @@ class HtmlReporter implements Reporter {
   private _montonicStartTime: number = 0;
   private _options: HtmlReporterOptions;
   private _outputFolder!: string;
+  private _sharded!: boolean;
   private _open: string | undefined;
   private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;
 
@@ -67,8 +69,9 @@ class HtmlReporter implements Reporter {
   onBegin(config: FullConfig, suite: Suite) {
     this._montonicStartTime = monotonicTime();
     this.config = config as FullConfigInternal;
-    const { outputFolder, open } = this._resolveOptions();
+    const { outputFolder, open, sharded } = this._resolveOptions();
     this._outputFolder = outputFolder;
+    this._sharded = sharded;
     this._open = open;
     const reportedWarnings = new Set<string>();
     for (const project of config.projects) {
@@ -89,19 +92,20 @@ class HtmlReporter implements Reporter {
     this.suite = suite;
   }
 
-  _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption } {
+  _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption, sharded: boolean } {
     let { outputFolder } = this._options;
     if (outputFolder)
       outputFolder = path.resolve(this.config._internal.configDir, outputFolder);
     return {
       outputFolder: reportFolderFromEnv() ?? outputFolder ?? defaultReportFolder(this.config._internal.configDir),
       open: process.env.PW_TEST_HTML_REPORT_OPEN as any || this._options.open || 'on-failure',
+      sharded: !!this._options.sharded
     };
   }
 
   async onEnd() {
     const duration = monotonicTime() - this._montonicStartTime;
-    const shard = this.config.shard;
+    const shard = this._sharded ? this.config.shard : null;
     const projectSuites = this.suite.suites;
     const reports = projectSuites.map(suite => {
       const rawReporter = new RawReporter();

packages/playwright-test/types/test.d.ts

@@ -25,7 +25,7 @@ export type ReporterDescription =
   ['github'] |
   ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] |
   ['json'] | ['json', { outputFile?: string }] |
-  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] |
+  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] |
   ['null'] |
   [string] | [string, any];
 

tests/playwright-test/reporter-html.spec.ts

@@ -979,7 +979,11 @@ test.describe('report location', () => {
 test('should shard report', async ({ runInlineTest, showReport, page }, testInfo) => {
   const totalShards = 3;
 
-  const testFiles = {};
+  const testFiles = {
+    'playwright.config.ts': `
+      module.exports = { reporter: [['html', { sharded: true }]] };
+    `,
+  };
   for (let i = 0; i < totalShards; i++) {
     testFiles[`a-${i}.spec.ts`] = `
       const { test } = pwt;
@@ -995,7 +999,7 @@ test('should shard report', async ({ runInlineTest, showReport, page }, testInfo
 
   for (let i = 1; i <= totalShards; i++) {
     const result = await runInlineTest(testFiles,
-        { 'reporter': 'dot,html', 'retries': 1, 'shard': `${i}/${totalShards}` },
+        { 'retries': 1, 'shard': `${i}/${totalShards}` },
         { PW_TEST_HTML_REPORT_OPEN: 'never' },
         { usesCustomReporters: true });
 
@@ -1024,14 +1028,19 @@ test('should shard report', async ({ runInlineTest, showReport, page }, testInfo
   await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(totalShards);
 });
 
-test('should pad report numbers with zeros', async ({ runInlineTest, showReport, page }, testInfo) => {
+test('should pad report numbers with zeros', async ({ runInlineTest }, testInfo) => {
-  const result = await runInlineTest({
+  const testFiles = {
-    'a.test.js': `
+    'playwright.config.ts': `
-      const { test } = pwt;
+      module.exports = { reporter: [['html', { sharded: true }]] };
-      test('passes', async ({}) => {});
     `,
-  }, { reporter: 'dot,html', shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });
+  };
-
+  for (let i = 0; i < 100; i++) {
+    testFiles[`a-${i}.spec.ts`] = `
+      const { test } = pwt;
+      test('passes', async ({}) => { });
+    `;
+  }
+  const result = await runInlineTest(testFiles, { shard: '3/100' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { usesCustomReporters: true });
   expect(result.exitCode).toBe(0);
   const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
   expect(new Set(files)).toEqual(new Set([
@@ -1043,7 +1052,11 @@ test('should pad report numbers with zeros', async ({ runInlineTest, showReport,
 test('should show report with missing shards', async ({ runInlineTest, showReport, page }, testInfo) => {
   const totalShards = 15;
 
-  const testFiles = {};
+  const testFiles = {
+    'playwright.config.ts': `
+      module.exports = { reporter: [['html', { sharded: true }]] };
+    `,
+  };
   for (let i = 0; i < totalShards; i++) {
     testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = `
       const { test } = pwt;
@@ -1060,7 +1073,7 @@ test('should show report with missing shards', async ({ runInlineTest, showRepor
   // Run tests in 2 out of 15 shards.
   for (const i of [10, 13]) {
     const result = await runInlineTest(testFiles,
-        { 'reporter': 'dot,html', 'retries': 1, 'shard': `${i}/${totalShards}` },
+        { 'retries': 1, 'shard': `${i}/${totalShards}` },
         { PW_TEST_HTML_REPORT_OPEN: 'never' },
         { usesCustomReporters: true });
 
@@ -1090,3 +1103,44 @@ test('should show report with missing shards', async ({ runInlineTest, showRepor
   await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(2);
   await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(2);
 });
+
+
+test('should produce single file report when shard: false', async ({ runInlineTest, showReport, page }, testInfo) => {
+  const totalShards = 5;
+
+  const testFiles = {};
+  for (let i = 0; i < totalShards; i++) {
+    testFiles[`a-${String(i).padStart(2, '0')}.spec.ts`] = `
+      const { test } = pwt;
+      test('passes', async ({}) => { expect(2).toBe(2); });
+      test('fails', async ({}) => { expect(1).toBe(2); });
+      test('skipped', async ({}) => { test.skip('Does not work') });
+      test('flaky', async ({}, testInfo) => { expect(testInfo.retry).toBe(1); });
+    `;
+  }
+
+  // Run single shard.
+  const currentShard = 3;
+  const result = await runInlineTest(testFiles,
+      { 'reporter': 'dot,html', 'retries': 1, 'shard': `${currentShard}/${totalShards}` },
+      { PW_TEST_HTML_REPORT_OPEN: 'never' },
+      { usesCustomReporters: true });
+
+
+  expect(result.exitCode).toBe(1);
+  const files = await fs.promises.readdir(testInfo.outputPath(`playwright-report`));
+  expect(files).toEqual(['index.html']);
+
+  await showReport();
+
+  await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('4');
+  await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('1');
+  await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('1');
+  await expect(page.locator('.subnav-item:has-text("Flaky") .counter')).toHaveText('1');
+  await expect(page.locator('.subnav-item:has-text("Skipped") .counter')).toHaveText('1');
+
+  await expect(page.locator('.test-file-test-outcome-unexpected >> text=fails')).toHaveCount(1);
+  await expect(page.locator('.test-file-test-outcome-flaky >> text=flaky')).toHaveCount(1);
+  await expect(page.locator('.test-file-test-outcome-expected >> text=passes')).toHaveCount(1);
+  await expect(page.locator('.test-file-test-outcome-skipped >> text=skipped')).toHaveCount(1);
+});

utils/generate_types/overrides-test.d.ts

@@ -24,7 +24,7 @@ export type ReporterDescription =
   ['github'] |
   ['junit'] | ['junit', { outputFile?: string, stripANSIControlSequences?: boolean }] |
   ['json'] | ['json', { outputFile?: string }] |
-  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure' }] |
+  ['html'] | ['html', { outputFolder?: string, open?: 'always' | 'never' | 'on-failure', sharded?: boolean }] |
   ['null'] |
   [string] | [string, any];