diff --git a/packages/html-reporter/src/chip.tsx b/packages/html-reporter/src/chip.tsx index e98883d492..d236141475 100644 --- a/packages/html-reporter/src/chip.tsx +++ b/packages/html-reporter/src/chip.tsx @@ -29,7 +29,7 @@ export const Chip: React.FC<{ dataTestId?: string, targetRef?: React.RefObject, }> = ({ header, expanded, setExpanded, children, noInsets, dataTestId, targetRef }) => { - return
+ return
setExpanded?.(!expanded)} diff --git a/packages/html-reporter/src/metadataView.tsx b/packages/html-reporter/src/metadataView.tsx index d5f3a912df..0f4fe6e73c 100644 --- a/packages/html-reporter/src/metadataView.tsx +++ b/packages/html-reporter/src/metadataView.tsx @@ -125,7 +125,7 @@ const InnerMetadataView: React.FC = metadata => { const MetadataViewItem: React.FC<{ content: JSX.Element | string; icon?: keyof typeof icons, href?: string, testId?: string }> = ({ content, icon, href, testId }) => { return ( -
+
{icons[icon || 'blank']()}
diff --git a/packages/html-reporter/src/testErrorView.css b/packages/html-reporter/src/testErrorView.css new file mode 100644 index 0000000000..afb543a0c2 --- /dev/null +++ b/packages/html-reporter/src/testErrorView.css @@ -0,0 +1,28 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +.test-error-message { + white-space: pre; + font-family: monospace; + overflow: auto; + flex: none; + padding: 0; + background-color: var(--color-canvas-subtle); + border-radius: 6px; + padding: 16px; + line-height: initial; + margin-bottom: 6px; +} diff --git a/packages/html-reporter/src/testErrorView.tsx b/packages/html-reporter/src/testErrorView.tsx new file mode 100644 index 0000000000..5208158b1c --- /dev/null +++ b/packages/html-reporter/src/testErrorView.tsx @@ -0,0 +1,56 @@ +/* + Copyright (c) Microsoft Corporation. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +import ansi2html from 'ansi-to-html'; +import * as React from 'react'; +import './testErrorView.css'; + +export const TestErrorView: React.FC<{ + error: string; +}> = ({ error }) => { + const html = React.useMemo(() => { + const config: any = { + bg: 'var(--color-canvas-subtle)', + fg: 'var(--color-fg-default)', + }; + config.colors = ansiColors; + return new ansi2html(config).toHtml(escapeHTML(error)); + }, [error]); + return
; +}; + +const ansiColors = { + 0: '#000', + 1: '#C00', + 2: '#0C0', + 3: '#C50', + 4: '#00C', + 5: '#C0C', + 6: '#0CC', + 7: '#CCC', + 8: '#555', + 9: '#F55', + 10: '#5F5', + 11: '#FF5', + 12: '#55F', + 13: '#F5F', + 14: '#5FF', + 15: '#FFF' +}; + +function escapeHTML(text: string): string { + return text.replace(/[&"<>]/g, c => ({ '&': '&', '"': '"', '<': '<', '>': '>' }[c]!)); +} diff --git a/packages/html-reporter/src/testFilesView.tsx b/packages/html-reporter/src/testFilesView.tsx index c0154af4d6..710beaf3ad 100644 --- a/packages/html-reporter/src/testFilesView.tsx +++ b/packages/html-reporter/src/testFilesView.tsx @@ -20,6 +20,8 @@ import type { Filter } from './filter'; import { TestFileView } from './testFileView'; import './testFileView.css'; import { msToString } from './uiUtils'; +import { AutoChip } from './chip'; +import { TestErrorView } from './testErrorView'; export const TestFilesView: React.FC<{ report?: HTMLReport, @@ -48,6 +50,9 @@ export const TestFilesView: React.FC<{
{report ? new Date(report.startTime).toLocaleString() : ''}
Total time: {msToString(filteredStats.duration)}
+ {report && report.errors.length && + {report.errors.map((error, index) => )} + } {report && filteredFiles.map(({ file, defaultExpanded }) => { return ): ImageDiff[] { @@ -94,7 +94,7 @@ export const TestResultView: React.FC<{ return
{!!result.errors.length && - {result.errors.map((error, index) => )} + {result.errors.map((error, index) => )} } {!!result.steps.length && {result.steps.map((step, i) => )} @@ -154,44 +154,7 @@ const StepTreeItem: React.FC<{ } loadChildren={step.steps.length + (step.snippet ? 1 : 0) ? () => { const children = step.steps.map((s, i) => ); if (step.snippet) - children.unshift(); + children.unshift(); return children; } : undefined} depth={depth}>; }; - -const ErrorMessage: React.FC<{ - error: string; -}> = ({ error }) => { - const html = React.useMemo(() => { - const config: any = { - bg: 'var(--color-canvas-subtle)', - fg: 'var(--color-fg-default)', - }; - config.colors = ansiColors; - return new ansi2html(config).toHtml(escapeHTML(error)); - }, [error]); - return
; -}; - -const ansiColors = { - 0: '#000', - 1: '#C00', - 2: '#0C0', - 3: '#C50', - 4: '#00C', - 5: '#C0C', - 6: '#0CC', - 7: '#CCC', - 8: '#555', - 9: '#F55', - 10: '#5F5', - 11: '#FF5', - 12: '#55F', - 13: '#F5F', - 14: '#5FF', - 15: '#FFF' -}; - -function escapeHTML(text: string): string { - return text.replace(/[&"<>]/g, c => ({ '&': '&', '"': '"', '<': '<', '>': '>' }[c]!)); -} diff --git a/packages/html-reporter/src/types.ts b/packages/html-reporter/src/types.ts index 021218abc5..5e71eb0b84 100644 --- a/packages/html-reporter/src/types.ts +++ b/packages/html-reporter/src/types.ts @@ -43,6 +43,7 @@ export type HTMLReport = { projectNames: string[]; startTime: number; duration: number; + errors: string[]; // Top-level errors that are not attributed to any test. }; export type TestFile = { diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 396518da83..731ed0be3c 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -22,10 +22,10 @@ import type { TransformCallback } from 'stream'; import { Transform } from 'stream'; import { toPosixPath } from './json'; import { codeFrameColumns } from '../transform/babelBundle'; -import type { FullResult, FullConfig, Location, Suite, TestCase as TestCasePublic, TestResult as TestResultPublic, TestStep as TestStepPublic } from '../../types/testReporter'; +import type { FullResult, FullConfig, Location, Suite, TestCase as TestCasePublic, TestResult as TestResultPublic, TestStep as TestStepPublic, TestError } from '../../types/testReporter'; import type { SuitePrivate } from '../../types/reporterPrivate'; import { HttpServer, assert, calculateSha1, copyFileAndMakeWritable, gracefullyProcessExitDoNotHang, removeFolders, sanitizeForFilePath } from 'playwright-core/lib/utils'; -import { formatResultFailure, stripAnsiEscapes } from './base'; +import { formatError, formatResultFailure, stripAnsiEscapes } from './base'; import { resolveReporterOutputPath } from '../util'; import type { Metadata } from '../../types/test'; import type { ZipFile } from 'playwright-core/lib/zipBundle'; @@ -64,6 +64,7 @@ class HtmlReporter extends EmptyReporter { private _attachmentsBaseURL!: string; private _open: string | undefined; private _buildResult: { ok: boolean, singleTestId: string | undefined } | undefined; + private _topLevelErrors: TestError[] = []; constructor(options: HtmlReporterOptions) { super(); @@ -111,11 +112,15 @@ class HtmlReporter extends EmptyReporter { }; } + override onError(error: TestError): void { + this._topLevelErrors.push(error); + } + override async onEnd(result: FullResult) { const projectSuites = this.suite.suites; await removeFolders([this._outputFolder]); const builder = new HtmlBuilder(this.config, this._outputFolder, this._attachmentsBaseURL); - this._buildResult = await builder.build(this.config.metadata, projectSuites, result); + this._buildResult = await builder.build(this.config.metadata, projectSuites, result, this._topLevelErrors); } override async onExit() { @@ -217,8 +222,7 @@ class HtmlBuilder { this._attachmentsBaseURL = attachmentsBaseURL; } - async build(metadata: Metadata, projectSuites: Suite[], result: FullResult): Promise<{ ok: boolean, singleTestId: string | undefined }> { - + async build(metadata: Metadata, projectSuites: Suite[], result: FullResult, topLevelErrors: TestError[]): Promise<{ ok: boolean, singleTestId: string | undefined }> { const data = new Map(); for (const projectSuite of projectSuites) { for (const fileSuite of projectSuite.suites) { @@ -276,7 +280,8 @@ class HtmlBuilder { duration: result.duration, files: [...data.values()].map(e => e.testFileSummary), projectNames: projectSuites.map(r => r.project()!.name), - stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) } + stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) }, + errors: topLevelErrors.map(error => formatError(error, true).message), }; htmlReport.files.sort((f1, f2) => { const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 10b77559a1..9a82a0f87d 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -425,7 +425,7 @@ for (const useIntermediateMergeReport of [false, true] as const) { await showReport(); await page.click('text=fails'); - await expect(page.locator('.test-result-error-message span:has-text("received")').nth(1)).toHaveCSS('color', 'rgb(204, 0, 0)'); + await expect(page.locator('.test-error-message span:has-text("received")').nth(1)).toHaveCSS('color', 'rgb(204, 0, 0)'); }); test('should show trace source', async ({ runInlineTest, page, showReport }) => { @@ -910,16 +910,16 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(result.exitCode).toBe(0); await page.click('text=awesome commit message'); - await expect.soft(page.locator('data-test-id=revision.id')).toContainText(/^[a-f\d]+$/i); - await expect.soft(page.locator('data-test-id=revision.id >> a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha'); - await expect.soft(page.locator('data-test-id=revision.timestamp')).toContainText(/AM|PM/); + await expect.soft(page.getByTestId('revision.id')).toContainText(/^[a-f\d]+$/i); + await expect.soft(page.getByTestId('revision.id').locator('a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha'); + await expect.soft(page.getByTestId('revision.timestamp')).toContainText(/AM|PM/); await expect.soft(page.locator('text=awesome commit message')).toHaveCount(2); await expect.soft(page.locator('text=William')).toBeVisible(); await expect.soft(page.locator('text=shakespeare@example.local')).toBeVisible(); await expect.soft(page.locator('text=CI/CD Logs')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/actions/runs/example-run-id'); await expect.soft(page.locator('text=Report generated on')).toContainText(/AM|PM/); - await expect.soft(page.locator('data-test-id=metadata-chip')).toBeVisible(); - await expect.soft(page.locator('data-test-id=metadata-error')).not.toBeVisible(); + await expect.soft(page.getByTestId('metadata-chip')).toBeVisible(); + await expect.soft(page.getByTestId('metadata-error')).not.toBeVisible(); }); @@ -951,16 +951,16 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(result.exitCode).toBe(0); await page.click('text=a better subject'); - await expect.soft(page.locator('data-test-id=revision.id')).toContainText(/^[a-f\d]+$/i); - await expect.soft(page.locator('data-test-id=revision.id >> a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha'); - await expect.soft(page.locator('data-test-id=revision.timestamp')).toContainText(/AM|PM/); + await expect.soft(page.getByTestId('revision.id')).toContainText(/^[a-f\d]+$/i); + await expect.soft(page.getByTestId('revision.id').locator('a')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/commit/example-sha'); + await expect.soft(page.getByTestId('revision.timestamp')).toContainText(/AM|PM/); await expect.soft(page.locator('text=a better subject')).toHaveCount(2); await expect.soft(page.locator('text=William')).toBeVisible(); await expect.soft(page.locator('text=shakespeare@example.local')).toBeVisible(); await expect.soft(page.locator('text=CI/CD Logs')).toHaveAttribute('href', 'https://playwright.dev/microsoft/playwright-example-for-test/actions/runs/example-run-id'); await expect.soft(page.locator('text=Report generated on')).toContainText(/AM|PM/); - await expect.soft(page.locator('data-test-id=metadata-chip')).toBeVisible(); - await expect.soft(page.locator('data-test-id=metadata-error')).not.toBeVisible(); + await expect.soft(page.getByTestId('metadata-chip')).toBeVisible(); + await expect.soft(page.getByTestId('metadata-error')).not.toBeVisible(); }); test('should not have metadata by default', async ({ runInlineTest, showReport, page }) => { @@ -979,8 +979,8 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(result.exitCode).toBe(0); await expect.soft(page.locator('text="my sample test"')).toBeVisible(); - await expect.soft(page.locator('data-test-id=metadata-error')).not.toBeVisible(); - await expect.soft(page.locator('data-test-id=metadata-chip')).not.toBeVisible(); + await expect.soft(page.getByTestId('metadata-error')).not.toBeVisible(); + await expect.soft(page.getByTestId('metadata-chip')).not.toBeVisible(); }); test('should not include metadata if user supplies invalid values via metadata field', async ({ runInlineTest, showReport, page }) => { @@ -1003,8 +1003,8 @@ for (const useIntermediateMergeReport of [false, true] as const) { expect(result.exitCode).toBe(0); await expect.soft(page.locator('text="my sample test"')).toBeVisible(); - await expect.soft(page.locator('data-test-id=metadata-error')).toBeVisible(); - await expect.soft(page.locator('data-test-id=metadata-chip')).not.toBeVisible(); + await expect.soft(page.getByTestId('metadata-error')).toBeVisible(); + await expect.soft(page.getByTestId('metadata-chip')).not.toBeVisible(); }); }); @@ -2225,6 +2225,30 @@ for (const useIntermediateMergeReport of [false, true] as const) { /afterAll hook/, ]); }); + + test('should display top-level errors', async ({ runInlineTest, showReport, page }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test, expect } = require('@playwright/test'); + test('passes', async ({}) => { + }); + `, + 'globalTeardown.ts': ` + export default async function globalTeardown() { + throw new Error('From teardown'); + } + `, + 'playwright.config.ts': ` + export default { globalTeardown: './globalTeardown.ts' }; + `, + }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(1); + + await showReport(); + await expect(page.getByTestId('report-errors')).toHaveText(/Error: From teardown.*at globalTeardown.ts:3.*export default async function globalTeardown/s); + }); }); }