feat(html): show top-level errors (#27763)

Drive-by:
- extract `TestErrorView`;
- replace `data-test-id` with `data-testid` and `getByTestId()`.

---

<img width="1001" alt="top-level errors in html report"
src="https://github.com/microsoft/playwright/assets/9881434/2d6c0c52-8df1-46a9-b3fd-06ddc6f16796">

packages/html-reporter/src/chip.tsx

@@ -29,7 +29,7 @@ export const Chip: React.FC<{
   dataTestId?: string,
   targetRef?: React.RefObject<HTMLDivElement>,
 }> = ({ header, expanded, setExpanded, children, noInsets, dataTestId, targetRef }) => {
-  return <div className='chip' data-test-id={dataTestId} ref={targetRef}>
+  return <div className='chip' data-testid={dataTestId} ref={targetRef}>
     <div
       className={'chip-header' + (setExpanded ? ' expanded-' + expanded : '')}
       onClick={() => setExpanded?.(!expanded)}

packages/html-reporter/src/metadataView.tsx

@@ -125,7 +125,7 @@ const InnerMetadataView: React.FC<Metainfo> = metadata => {
 
 const MetadataViewItem: React.FC<{ content: JSX.Element | string; icon?: keyof typeof icons, href?: string, testId?: string }> = ({ content, icon, href, testId }) => {
   return (
-    <div className='my-1 hbox' data-test-id={testId} >
+    <div className='my-1 hbox' data-testid={testId} >
       <div className='mr-2'>
         {icons[icon || 'blank']()}
       </div>

packages/html-reporter/src/testErrorView.css

@@ -0,0 +1,28 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+.test-error-message {
+  white-space: pre;
+  font-family: monospace;
+  overflow: auto;
+  flex: none;
+  padding: 0;
+  background-color: var(--color-canvas-subtle);
+  border-radius: 6px;
+  padding: 16px;
+  line-height: initial;
+  margin-bottom: 6px;
+}

packages/html-reporter/src/testErrorView.tsx

@@ -0,0 +1,56 @@
+/*
+  Copyright (c) Microsoft Corporation.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/
+
+import ansi2html from 'ansi-to-html';
+import * as React from 'react';
+import './testErrorView.css';
+
+export const TestErrorView: React.FC<{
+  error: string;
+}> = ({ error }) => {
+  const html = React.useMemo(() => {
+    const config: any = {
+      bg: 'var(--color-canvas-subtle)',
+      fg: 'var(--color-fg-default)',
+    };
+    config.colors = ansiColors;
+    return new ansi2html(config).toHtml(escapeHTML(error));
+  }, [error]);
+  return <div className='test-error-message' dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
+};
+
+const ansiColors = {
+  0: '#000',
+  1: '#C00',
+  2: '#0C0',
+  3: '#C50',
+  4: '#00C',
+  5: '#C0C',
+  6: '#0CC',
+  7: '#CCC',
+  8: '#555',
+  9: '#F55',
+  10: '#5F5',
+  11: '#FF5',
+  12: '#55F',
+  13: '#F5F',
+  14: '#5FF',
+  15: '#FFF'
+};
+
+function escapeHTML(text: string): string {
+  return text.replace(/[&"<>]/g, c => ({ '&': '&amp;', '"': '&quot;', '<': '&lt;', '>': '&gt;' }[c]!));
+}

packages/html-reporter/src/testFilesView.tsx

@@ -20,6 +20,8 @@ import type { Filter } from './filter';
 import { TestFileView } from './testFileView';
 import './testFileView.css';
 import { msToString } from './uiUtils';
+import { AutoChip } from './chip';
+import { TestErrorView } from './testErrorView';
 
 export const TestFilesView: React.FC<{
   report?: HTMLReport,
@@ -48,6 +50,9 @@ export const TestFilesView: React.FC<{
       <div data-testid="overall-time" style={{ color: 'var(--color-fg-subtle)', marginRight: '10px' }}>{report ? new Date(report.startTime).toLocaleString() : ''}</div>
       <div data-testid="overall-duration" style={{ color: 'var(--color-fg-subtle)' }}>Total time: {msToString(filteredStats.duration)}</div>
     </div>
+    {report && report.errors.length && <AutoChip header='Errors' dataTestId='report-errors'>
+      {report.errors.map((error, index) => <TestErrorView key={'test-report-error-message-' + index} error={error}></TestErrorView>)}
+    </AutoChip>}
     {report && filteredFiles.map(({ file, defaultExpanded }) => {
       return <TestFileView
         key={`file-${file.fileId}`}

packages/html-reporter/src/testResultView.css

@@ -39,19 +39,6 @@
   color: var(--color-fg-muted);
 }
 
-.test-result-error-message {
-  white-space: pre;
-  font-family: monospace;
-  overflow: auto;
-  flex: none;
-  padding: 0;
-  background-color: var(--color-canvas-subtle);
-  border-radius: 6px;
-  padding: 16px;
-  line-height: initial;
-  margin-bottom: 6px;
-}
-
 .test-result-counter {
   border-radius: 12px;
   color: var(--color-canvas-default);

packages/html-reporter/src/testResultView.tsx

@@ -15,7 +15,6 @@
 */
 
 import type { TestAttachment, TestCase, TestResult, TestStep } from './types';
-import ansi2html from 'ansi-to-html';
 import * as React from 'react';
 import { TreeItem } from './treeItem';
 import { msToString } from './uiUtils';
@@ -25,6 +24,7 @@ import { AttachmentLink, generateTraceUrl } from './links';
 import { statusIcon } from './statusIcon';
 import type { ImageDiff } from './imageDiffView';
 import { ImageDiffView } from './imageDiffView';
+import { TestErrorView } from './testErrorView';
 import './testResultView.css';
 
 function groupImageDiffs(screenshots: Set<TestAttachment>): ImageDiff[] {
@@ -94,7 +94,7 @@ export const TestResultView: React.FC<{
 
   return <div className='test-result'>
     {!!result.errors.length && <AutoChip header='Errors'>
-      {result.errors.map((error, index) => <ErrorMessage key={'test-result-error-message-' + index} error={error}></ErrorMessage>)}
+      {result.errors.map((error, index) => <TestErrorView key={'test-result-error-message-' + index} error={error}></TestErrorView>)}
     </AutoChip>}
     {!!result.steps.length && <AutoChip header='Test Steps'>
       {result.steps.map((step, i) => <StepTreeItem key={`step-${i}`} step={step} depth={0}></StepTreeItem>)}
@@ -154,44 +154,7 @@ const StepTreeItem: React.FC<{
   </span>} loadChildren={step.steps.length + (step.snippet ? 1 : 0) ? () => {
     const children = step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1}></StepTreeItem>);
     if (step.snippet)
-      children.unshift(<ErrorMessage key='line' error={step.snippet}></ErrorMessage>);
+      children.unshift(<TestErrorView key='line' error={step.snippet}></TestErrorView>);
     return children;
   } : undefined} depth={depth}></TreeItem>;
 };
-
-const ErrorMessage: React.FC<{
-  error: string;
-}> = ({ error }) => {
-  const html = React.useMemo(() => {
-    const config: any = {
-      bg: 'var(--color-canvas-subtle)',
-      fg: 'var(--color-fg-default)',
-    };
-    config.colors = ansiColors;
-    return new ansi2html(config).toHtml(escapeHTML(error));
-  }, [error]);
-  return <div className='test-result-error-message' dangerouslySetInnerHTML={{ __html: html || '' }}></div>;
-};
-
-const ansiColors = {
-  0: '#000',
-  1: '#C00',
-  2: '#0C0',
-  3: '#C50',
-  4: '#00C',
-  5: '#C0C',
-  6: '#0CC',
-  7: '#CCC',
-  8: '#555',
-  9: '#F55',
-  10: '#5F5',
-  11: '#FF5',
-  12: '#55F',
-  13: '#F5F',
-  14: '#5FF',
-  15: '#FFF'
-};
-
-function escapeHTML(text: string): string {
-  return text.replace(/[&"<>]/g, c => ({ '&': '&amp;', '"': '&quot;', '<': '&lt;', '>': '&gt;' }[c]!));
-}

packages/html-reporter/src/types.ts

@@ -43,6 +43,7 @@ export type HTMLReport = {
   projectNames: string[];
   startTime: number;
   duration: number;
+  errors: string[];  // Top-level errors that are not attributed to any test.
 };
 
 export type TestFile = {

packages/playwright/src/reporters/html.ts

@@ -22,10 +22,10 @@ import type { TransformCallback } from 'stream';
 import { Transform } from 'stream';
 import { toPosixPath } from './json';
 import { codeFrameColumns } from '../transform/babelBundle';
-import type { FullResult, FullConfig, Location, Suite, TestCase as TestCasePublic, TestResult as TestResultPublic, TestStep as TestStepPublic } from '../../types/testReporter';
+import type { FullResult, FullConfig, Location, Suite, TestCase as TestCasePublic, TestResult as TestResultPublic, TestStep as TestStepPublic, TestError } from '../../types/testReporter';
 import type { SuitePrivate } from '../../types/reporterPrivate';
 import { HttpServer, assert, calculateSha1, copyFileAndMakeWritable, gracefullyProcessExitDoNotHang, removeFolders, sanitizeForFilePath } from 'playwright-core/lib/utils';
-import { formatResultFailure, stripAnsiEscapes } from './base';
+import { formatError, formatResultFailure, stripAnsiEscapes } from './base';
 import { resolveReporterOutputPath } from '../util';
 import type { Metadata } from '../../types/test';
 import type { ZipFile } from 'playwright-core/lib/zipBundle';
@@ -64,6 +64,7 @@ class HtmlReporter extends EmptyReporter {
   private _attachmentsBaseURL!: string;
   private _open: string | undefined;
   private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined;
+  private _topLevelErrors: TestError[] = [];
 
   constructor(options: HtmlReporterOptions) {
     super();
@@ -111,11 +112,15 @@ class HtmlReporter extends EmptyReporter {
     };
   }
 
+  override onError(error: TestError): void {
+    this._topLevelErrors.push(error);
+  }
+
   override async onEnd(result: FullResult) {
     const projectSuites = this.suite.suites;
     await removeFolders([this._outputFolder]);
     const builder = new HtmlBuilder(this.config, this._outputFolder, this._attachmentsBaseURL);
-    this._buildResult = await builder.build(this.config.metadata, projectSuites, result);
+    this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors);
   }
 
   override async onExit() {
@@ -217,8 +222,7 @@ class HtmlBuilder {
     this._attachmentsBaseURL = attachmentsBaseURL;
   }
 
-  async build(metadata: Metadata, projectSuites: Suite[], result: FullResult): Promise<{ ok: boolean, singleTestId: string | undefined }> {
+  async build(metadata: Metadata, projectSuites: Suite[], result: FullResult, topLevelErrors: TestError[]): Promise<{ ok: boolean, singleTestId: string | undefined }> {
-
     const data = new Map<string, { testFile: TestFile, testFileSummary: TestFileSummary }>();
     for (const projectSuite of projectSuites) {
       for (const fileSuite of projectSuite.suites) {
@@ -276,7 +280,8 @@ class HtmlBuilder {
       duration: result.duration,
       files: [...data.values()].map(e => e.testFileSummary),
       projectNames: projectSuites.map(r => r.project()!.name),
-      stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) }
+      stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) },
+      errors: topLevelErrors.map(error => formatError(error, true).message),
     };
     htmlReport.files.sort((f1, f2) => {
       const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky;

tests/playwright-test/reporter-html.spec.ts

@@ -425,7 +425,7 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
       await showReport();
       await page.click('text=fails');
-      await expect(page.locator('.test-result-error-message span:has-text("received")').nth(1)).toHaveCSS('color', 'rgb(204, 0, 0)');
+      await expect(page.locator('.test-error-message span:has-text("received")').nth(1)).toHaveCSS('color', 'rgb(204, 0, 0)');
     });
 
     test('should show trace source', async ({ runInlineTest, page, showReport }) => {
@@ -910,16 +910,16 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
         expect(result.exitCode).toBe(0);
         await page.click('text=awesome commit message');
-        await expect.soft(page.locator('data-test-id=revision.id')).toContainText(/^[a-f\d]+$/i);
+        await expect.soft(page.getByTestId('revision.id')).toContainText(/^[a-f\d]+$/i);
-        await expect.soft(page.locator('data-test-id=revision.id >> a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha');
+        await expect.soft(page.getByTestId('revision.id').locator('a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha');
-        await expect.soft(page.locator('data-test-id=revision.timestamp')).toContainText(/AM|PM/);
+        await expect.soft(page.getByTestId('revision.timestamp')).toContainText(/AM|PM/);
         await expect.soft(page.locator('text=awesome commit message')).toHaveCount(2);
         await expect.soft(page.locator('text=William')).toBeVisible();
         await expect.soft(page.locator('text=shakespeare@example.local')).toBeVisible();
         await expect.soft(page.locator('text=CI/CD Logs')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/actions/runs/example-run-id');
         await expect.soft(page.locator('text=Report generated on')).toContainText(/AM|PM/);
-        await expect.soft(page.locator('data-test-id=metadata-chip')).toBeVisible();
+        await expect.soft(page.getByTestId('metadata-chip')).toBeVisible();
-        await expect.soft(page.locator('data-test-id=metadata-error')).not.toBeVisible();
+        await expect.soft(page.getByTestId('metadata-error')).not.toBeVisible();
       });
 
 
@@ -951,16 +951,16 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
         expect(result.exitCode).toBe(0);
         await page.click('text=a better subject');
-        await expect.soft(page.locator('data-test-id=revision.id')).toContainText(/^[a-f\d]+$/i);
+        await expect.soft(page.getByTestId('revision.id')).toContainText(/^[a-f\d]+$/i);
-        await expect.soft(page.locator('data-test-id=revision.id >> a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha');
+        await expect.soft(page.getByTestId('revision.id').locator('a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha');
-        await expect.soft(page.locator('data-test-id=revision.timestamp')).toContainText(/AM|PM/);
+        await expect.soft(page.getByTestId('revision.timestamp')).toContainText(/AM|PM/);
         await expect.soft(page.locator('text=a better subject')).toHaveCount(2);
         await expect.soft(page.locator('text=William')).toBeVisible();
         await expect.soft(page.locator('text=shakespeare@example.local')).toBeVisible();
         await expect.soft(page.locator('text=CI/CD Logs')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/actions/runs/example-run-id');
         await expect.soft(page.locator('text=Report generated on')).toContainText(/AM|PM/);
-        await expect.soft(page.locator('data-test-id=metadata-chip')).toBeVisible();
+        await expect.soft(page.getByTestId('metadata-chip')).toBeVisible();
-        await expect.soft(page.locator('data-test-id=metadata-error')).not.toBeVisible();
+        await expect.soft(page.getByTestId('metadata-error')).not.toBeVisible();
       });
 
       test('should not have metadata by default', async ({ runInlineTest, showReport, page }) => {
@@ -979,8 +979,8 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
         expect(result.exitCode).toBe(0);
         await expect.soft(page.locator('text="my sample test"')).toBeVisible();
-        await expect.soft(page.locator('data-test-id=metadata-error')).not.toBeVisible();
+        await expect.soft(page.getByTestId('metadata-error')).not.toBeVisible();
-        await expect.soft(page.locator('data-test-id=metadata-chip')).not.toBeVisible();
+        await expect.soft(page.getByTestId('metadata-chip')).not.toBeVisible();
       });
 
       test('should not include metadata if user supplies invalid values via metadata field', async ({ runInlineTest, showReport, page }) => {
@@ -1003,8 +1003,8 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
         expect(result.exitCode).toBe(0);
         await expect.soft(page.locator('text="my sample test"')).toBeVisible();
-        await expect.soft(page.locator('data-test-id=metadata-error')).toBeVisible();
+        await expect.soft(page.getByTestId('metadata-error')).toBeVisible();
-        await expect.soft(page.locator('data-test-id=metadata-chip')).not.toBeVisible();
+        await expect.soft(page.getByTestId('metadata-chip')).not.toBeVisible();
       });
     });
 
@@ -2225,6 +2225,30 @@ for (const useIntermediateMergeReport of [false, true] as const) {
         /afterAll hook/,
       ]);
     });
+
+    test('should display top-level errors', async ({ runInlineTest, showReport, page }) => {
+      const result = await runInlineTest({
+        'a.test.js': `
+          const { test, expect } = require('@playwright/test');
+          test('passes', async ({}) => {
+          });
+        `,
+        'globalTeardown.ts': `
+          export default async function globalTeardown() {
+            throw new Error('From teardown');
+          }
+        `,
+        'playwright.config.ts': `
+          export default { globalTeardown: './globalTeardown.ts' };
+        `,
+      }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });
+
+      expect(result.exitCode).toBe(1);
+      expect(result.passed).toBe(1);
+
+      await showReport();
+      await expect(page.getByTestId('report-errors')).toHaveText(/Error: From teardown.*at globalTeardown.ts:3.*export default async function globalTeardown/s);
+    });
   });
 }