From 4e8c83055fa1d50d5900cd6ae3d98fcc12c949af Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Mon, 6 Jan 2025 11:03:35 -0800 Subject: [PATCH] chore: split output clients by capabilities and base dir (#34135) --- packages/playwright/src/reporters/base.ts | 209 ++++++++++++------ packages/playwright/src/reporters/dot.ts | 16 +- packages/playwright/src/reporters/github.ts | 19 +- packages/playwright/src/reporters/html.ts | 8 +- .../src/reporters/internalReporter.ts | 4 +- packages/playwright/src/reporters/json.ts | 4 +- packages/playwright/src/reporters/junit.ts | 4 +- packages/playwright/src/reporters/line.ts | 14 +- packages/playwright/src/reporters/list.ts | 50 ++--- packages/playwright/src/reporters/markdown.ts | 7 +- packages/playwright/src/runner/reporters.ts | 9 +- packages/playwright/src/runner/runner.ts | 7 +- packages/playwright/src/runner/testServer.ts | 3 +- packages/playwright/src/runner/watchMode.ts | 10 +- tests/playwright-test/reporter-blob.spec.ts | 5 +- 15 files changed, 231 insertions(+), 138 deletions(-) diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 38b69d9d65..ec11413391 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -24,6 +24,8 @@ import { resolveReporterOutputPath } from '../util'; export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; export const kOutputSymbol = Symbol('output'); +type Colors = typeof realColors; + type ErrorDetails = { message: string; location?: Location; @@ -40,7 +42,57 @@ type TestSummary = { fatalErrors: TestError[]; }; -export const { isTTY, ttyWidth, colors } = (() => { +export type Screen = { + resolveFiles: 'cwd' | 'rootDir'; + colors: Colors; + isTTY: boolean; + ttyWidth: number; +}; + +export const noColors: Colors = { + bold: (t: string) => t, + cyan: (t: string) => t, + dim: (t: string) => t, + gray: (t: string) => t, + green: (t: string) => t, + red: (t: string) => t, + yellow: (t: string) => t, + black: (t: string) => t, + blue: (t: string) => t, + magenta: (t: string) => t, + white: (t: string) => t, + grey: (t: string) => t, + bgBlack: (t: string) => t, + bgRed: (t: string) => t, + bgGreen: (t: string) => t, + bgYellow: (t: string) => t, + bgBlue: (t: string) => t, + bgMagenta: (t: string) => t, + bgCyan: (t: string) => t, + bgWhite: (t: string) => t, + strip: (t: string) => t, + stripColors: (t: string) => t, + reset: (t: string) => t, + italic: (t: string) => t, + underline: (t: string) => t, + inverse: (t: string) => t, + hidden: (t: string) => t, + strikethrough: (t: string) => t, + rainbow: (t: string) => t, + zebra: (t: string) => t, + america: (t: string) => t, + trap: (t: string) => t, + random: (t: string) => t, + zalgo: (t: string) => t, + + enabled: false, + enable: () => {}, + disable: () => {}, + setTheme: () => {}, +}; + +// Output goes to terminal. +export const terminalScreen: Screen = (() => { let isTTY = !!process.stdout.isTTY; let ttyWidth = process.stdout.columns || 0; if (process.env.PLAYWRIGHT_FORCE_TTY === 'false' || process.env.PLAYWRIGHT_FORCE_TTY === '0') { @@ -63,20 +115,33 @@ export const { isTTY, ttyWidth, colors } = (() => { else if (process.env.DEBUG_COLORS || process.env.FORCE_COLOR) useColors = true; - const colors = useColors ? realColors : { - bold: (t: string) => t, - cyan: (t: string) => t, - dim: (t: string) => t, - gray: (t: string) => t, - green: (t: string) => t, - red: (t: string) => t, - yellow: (t: string) => t, - enabled: false, + const colors = useColors ? realColors : noColors; + return { + resolveFiles: 'cwd', + isTTY, + ttyWidth, + colors }; - return { isTTY, ttyWidth, colors }; })(); -export class BaseReporter implements ReporterV2 { +// Output does not go to terminal, but colors are controlled with terminal env vars. +export const nonTerminalScreen: Screen = { + colors: terminalScreen.colors, + isTTY: false, + ttyWidth: 0, + resolveFiles: 'rootDir', +}; + +// Internal output for post-processing, should always contain real colors. +export const internalScreen: Screen = { + colors: realColors, + isTTY: false, + ttyWidth: 0, + resolveFiles: 'rootDir', +}; + +export class TerminalReporter implements ReporterV2 { + screen: Screen = terminalScreen; config!: FullConfig; suite!: Suite; totalTestCount = 0; @@ -122,7 +187,7 @@ export class BaseReporter implements ReporterV2 { if (result.status !== 'skipped' && result.status !== test.expectedStatus) ++this._failureCount; const projectName = test.titlePath()[1]; - const relativePath = relativeTestPath(this.config, test); + const relativePath = relativeTestPath(this.screen, this.config, test); const fileAndProject = (projectName ? `[${projectName}] › ` : '') + relativePath; const entry = this.fileDurations.get(fileAndProject) || { duration: 0, workers: new Set() }; entry.duration += result.duration; @@ -139,11 +204,11 @@ export class BaseReporter implements ReporterV2 { } protected fitToScreen(line: string, prefix?: string): string { - if (!ttyWidth) { + if (!this.screen.ttyWidth) { // Guard against the case where we cannot determine available width. return line; } - return fitToWidth(line, ttyWidth, prefix); + return fitToWidth(line, this.screen.ttyWidth, prefix); } protected generateStartingMessage() { @@ -151,7 +216,7 @@ export class BaseReporter implements ReporterV2 { const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; if (!this.totalTestCount) return ''; - return '\n' + colors.dim('Running ') + this.totalTestCount + colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`); + return '\n' + this.screen.colors.dim('Running ') + this.totalTestCount + this.screen.colors.dim(` test${this.totalTestCount !== 1 ? 's' : ''} using `) + jobs + this.screen.colors.dim(` worker${jobs !== 1 ? 's' : ''}${shardDetails}`); } protected getSlowTests(): [string, number][] { @@ -168,28 +233,28 @@ export class BaseReporter implements ReporterV2 { protected generateSummaryMessage({ didNotRun, skipped, expected, interrupted, unexpected, flaky, fatalErrors }: TestSummary) { const tokens: string[] = []; if (unexpected.length) { - tokens.push(colors.red(` ${unexpected.length} failed`)); + tokens.push(this.screen.colors.red(` ${unexpected.length} failed`)); for (const test of unexpected) - tokens.push(colors.red(formatTestHeader(this.config, test, { indent: ' ' }))); + tokens.push(this.screen.colors.red(this.formatTestHeader(test, { indent: ' ' }))); } if (interrupted.length) { - tokens.push(colors.yellow(` ${interrupted.length} interrupted`)); + tokens.push(this.screen.colors.yellow(` ${interrupted.length} interrupted`)); for (const test of interrupted) - tokens.push(colors.yellow(formatTestHeader(this.config, test, { indent: ' ' }))); + tokens.push(this.screen.colors.yellow(this.formatTestHeader(test, { indent: ' ' }))); } if (flaky.length) { - tokens.push(colors.yellow(` ${flaky.length} flaky`)); + tokens.push(this.screen.colors.yellow(` ${flaky.length} flaky`)); for (const test of flaky) - tokens.push(colors.yellow(formatTestHeader(this.config, test, { indent: ' ' }))); + tokens.push(this.screen.colors.yellow(this.formatTestHeader(test, { indent: ' ' }))); } if (skipped) - tokens.push(colors.yellow(` ${skipped} skipped`)); + tokens.push(this.screen.colors.yellow(` ${skipped} skipped`)); if (didNotRun) - tokens.push(colors.yellow(` ${didNotRun} did not run`)); + tokens.push(this.screen.colors.yellow(` ${didNotRun} did not run`)); if (expected) - tokens.push(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.result.duration)})`)); + tokens.push(this.screen.colors.green(` ${expected} passed`) + this.screen.colors.dim(` (${milliseconds(this.result.duration)})`)); if (fatalErrors.length && expected + unexpected.length + interrupted.length + flaky.length > 0) - tokens.push(colors.red(` ${fatalErrors.length === 1 ? '1 error was not a part of any test' : fatalErrors.length + ' errors were not a part of any test'}, see above for details`)); + tokens.push(this.screen.colors.red(` ${fatalErrors.length === 1 ? '1 error was not a part of any test' : fatalErrors.length + ' errors were not a part of any test'}, see above for details`)); return tokens.join('\n'); } @@ -248,17 +313,17 @@ export class BaseReporter implements ReporterV2 { private _printFailures(failures: TestCase[]) { console.log(''); failures.forEach((test, index) => { - console.log(formatFailure(this.config, test, index + 1)); + console.log(this.formatFailure(test, index + 1)); }); } private _printSlowTests() { const slowTests = this.getSlowTests(); slowTests.forEach(([file, duration]) => { - console.log(colors.yellow(' Slow test file: ') + file + colors.yellow(` (${milliseconds(duration)})`)); + console.log(this.screen.colors.yellow(' Slow test file: ') + file + this.screen.colors.yellow(` (${milliseconds(duration)})`)); }); if (slowTests.length) - console.log(colors.yellow(' Consider running tests from slow files in parallel, see https://playwright.dev/docs/test-parallel.')); + console.log(this.screen.colors.yellow(' Consider running tests from slow files in parallel, see https://playwright.dev/docs/test-parallel.')); } private _printSummary(summary: string) { @@ -269,21 +334,37 @@ export class BaseReporter implements ReporterV2 { willRetry(test: TestCase): boolean { return test.outcome() === 'unexpected' && test.results.length <= test.retries; } + + formatTestTitle(test: TestCase, step?: TestStep, omitLocation: boolean = false): string { + return formatTestTitle(this.screen, this.config, test, step, omitLocation); + } + + formatTestHeader(test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string { + return formatTestHeader(this.screen, this.config, test, options); + } + + formatFailure(test: TestCase, index?: number): string { + return formatFailure(this.screen, this.config, test, index); + } + + formatError(error: TestError): ErrorDetails { + return formatError(this.screen, error); + } } -export function formatFailure(config: FullConfig, test: TestCase, index?: number): string { +export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number): string { const lines: string[] = []; - const header = formatTestHeader(config, test, { indent: ' ', index, mode: 'error' }); - lines.push(colors.red(header)); + const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error' }); + lines.push(screen.colors.red(header)); for (const result of test.results) { const resultLines: string[] = []; - const errors = formatResultFailure(test, result, ' ', colors.enabled); + const errors = formatResultFailure(screen, test, result, ' '); if (!errors.length) continue; const retryLines = []; if (result.retry) { retryLines.push(''); - retryLines.push(colors.gray(separator(` Retry #${result.retry}`))); + retryLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`))); } resultLines.push(...retryLines); resultLines.push(...errors.map(error => '\n' + error.message)); @@ -293,16 +374,16 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number if (!attachment.path && !hasPrintableContent) continue; resultLines.push(''); - resultLines.push(colors.cyan(separator(` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`))); + resultLines.push(screen.colors.cyan(separator(screen, ` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`))); if (attachment.path) { const relativePath = path.relative(process.cwd(), attachment.path); - resultLines.push(colors.cyan(` ${relativePath}`)); + resultLines.push(screen.colors.cyan(` ${relativePath}`)); // Make this extensible if (attachment.name === 'trace') { const packageManagerCommand = getPackageManagerExecCommand(); - resultLines.push(colors.cyan(` Usage:`)); + resultLines.push(screen.colors.cyan(` Usage:`)); resultLines.push(''); - resultLines.push(colors.cyan(` ${packageManagerCommand} playwright show-trace ${quotePathIfNeeded(relativePath)}`)); + resultLines.push(screen.colors.cyan(` ${packageManagerCommand} playwright show-trace ${quotePathIfNeeded(relativePath)}`)); resultLines.push(''); } } else { @@ -311,10 +392,10 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number if (text.length > 300) text = text.slice(0, 300) + '...'; for (const line of text.split('\n')) - resultLines.push(colors.cyan(` ${line}`)); + resultLines.push(screen.colors.cyan(` ${line}`)); } } - resultLines.push(colors.cyan(separator(' '))); + resultLines.push(screen.colors.cyan(separator(screen, ' '))); } lines.push(...resultLines); } @@ -322,11 +403,11 @@ export function formatFailure(config: FullConfig, test: TestCase, index?: number return lines.join('\n'); } -export function formatRetry(result: TestResult) { +export function formatRetry(screen: Screen, result: TestResult) { const retryLines = []; if (result.retry) { retryLines.push(''); - retryLines.push(colors.gray(separator(` Retry #${result.retry}`))); + retryLines.push(screen.colors.gray(separator(screen, ` Retry #${result.retry}`))); } return retryLines; } @@ -337,22 +418,22 @@ function quotePathIfNeeded(path: string): string { return path; } -export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string, highlightCode: boolean): ErrorDetails[] { +export function formatResultFailure(screen: Screen, test: TestCase, result: TestResult, initialIndent: string): ErrorDetails[] { const errorDetails: ErrorDetails[] = []; if (result.status === 'passed' && test.expectedStatus === 'failed') { errorDetails.push({ - message: indent(colors.red(`Expected to fail, but passed.`), initialIndent), + message: indent(screen.colors.red(`Expected to fail, but passed.`), initialIndent), }); } if (result.status === 'interrupted') { errorDetails.push({ - message: indent(colors.red(`Test was interrupted.`), initialIndent), + message: indent(screen.colors.red(`Test was interrupted.`), initialIndent), }); } for (const error of result.errors) { - const formattedError = formatError(error, highlightCode); + const formattedError = formatError(screen, error); errorDetails.push({ message: indent(formattedError.message, initialIndent), location: formattedError.location, @@ -361,12 +442,14 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI return errorDetails; } -export function relativeFilePath(config: FullConfig, file: string): string { - return path.relative(config.rootDir, file) || path.basename(file); +export function relativeFilePath(screen: Screen, config: FullConfig, file: string): string { + if (screen.resolveFiles === 'cwd') + return path.relative(process.cwd(), file); + return path.relative(config.rootDir, file); } -function relativeTestPath(config: FullConfig, test: TestCase): string { - return relativeFilePath(config, test.location.file); +function relativeTestPath(screen: Screen, config: FullConfig, test: TestCase): string { + return relativeFilePath(screen, config, test.location.file); } export function stepSuffix(step: TestStep | undefined) { @@ -374,22 +457,22 @@ export function stepSuffix(step: TestStep | undefined) { return stepTitles.map(t => t.split('\n')[0]).map(t => ' › ' + t).join(''); } -export function formatTestTitle(config: FullConfig, test: TestCase, step?: TestStep, omitLocation: boolean = false): string { +function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, step?: TestStep, omitLocation: boolean = false): string { // root, project, file, ...describes, test const [, projectName, , ...titles] = test.titlePath(); let location; if (omitLocation) - location = `${relativeTestPath(config, test)}`; + location = `${relativeTestPath(screen, config, test)}`; else - location = `${relativeTestPath(config, test)}:${test.location.line}:${test.location.column}`; + location = `${relativeTestPath(screen, config, test)}:${test.location.line}:${test.location.column}`; const projectTitle = projectName ? `[${projectName}] › ` : ''; const testTitle = `${projectTitle}${location} › ${titles.join(' › ')}`; const extraTags = test.tags.filter(t => !testTitle.includes(t)); return `${testTitle}${stepSuffix(step)}${extraTags.length ? ' ' + extraTags.join(' ') : ''}`; } -export function formatTestHeader(config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string { - const title = formatTestTitle(config, test); +function formatTestHeader(screen: Screen, config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string { + const title = formatTestTitle(screen, config, test); const header = `${options.indent || ''}${options.index ? options.index + ') ' : ''}${title}`; let fullHeader = header; @@ -412,10 +495,10 @@ export function formatTestHeader(config: FullConfig, test: TestCase, options: { } fullHeader = header + (stepPaths.size === 1 ? stepPaths.values().next().value : ''); } - return separator(fullHeader); + return separator(screen, fullHeader); } -export function formatError(error: TestError, highlightCode: boolean): ErrorDetails { +export function formatError(screen: Screen, error: TestError): ErrorDetails { const message = error.message || error.value || ''; const stack = error.stack; if (!stack && !error.location) @@ -430,21 +513,21 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta if (error.snippet) { let snippet = error.snippet; - if (!highlightCode) + if (!screen.colors.enabled) snippet = stripAnsiEscapes(snippet); tokens.push(''); tokens.push(snippet); } if (parsedStack && parsedStack.stackLines.length) - tokens.push(colors.dim(parsedStack.stackLines.join('\n'))); + tokens.push(screen.colors.dim(parsedStack.stackLines.join('\n'))); let location = error.location; if (parsedStack && !location) location = parsedStack.location; if (error.cause) - tokens.push(colors.dim('[cause]: ') + formatError(error.cause, highlightCode).message); + tokens.push(screen.colors.dim('[cause]: ') + formatError(screen, error.cause).message); return { location, @@ -452,11 +535,11 @@ export function formatError(error: TestError, highlightCode: boolean): ErrorDeta }; } -export function separator(text: string = ''): string { +export function separator(screen: Screen, text: string = ''): string { if (text) text += ' '; - const columns = Math.min(100, ttyWidth || 100); - return text + colors.dim('─'.repeat(Math.max(0, columns - text.length))); + const columns = Math.min(100, screen.ttyWidth || 100); + return text + screen.colors.dim('─'.repeat(Math.max(0, columns - text.length))); } function indent(lines: string, tab: string) { diff --git a/packages/playwright/src/reporters/dot.ts b/packages/playwright/src/reporters/dot.ts index 169af3b1e7..7f635f8214 100644 --- a/packages/playwright/src/reporters/dot.ts +++ b/packages/playwright/src/reporters/dot.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { colors, BaseReporter, formatError } from './base'; +import { TerminalReporter } from './base'; import type { FullResult, TestCase, TestResult, Suite, TestError } from '../../types/testReporter'; -class DotReporter extends BaseReporter { +class DotReporter extends TerminalReporter { private _counter = 0; override onBegin(suite: Suite) { @@ -45,23 +45,23 @@ class DotReporter extends BaseReporter { } ++this._counter; if (result.status === 'skipped') { - process.stdout.write(colors.yellow('°')); + process.stdout.write(this.screen.colors.yellow('°')); return; } if (this.willRetry(test)) { - process.stdout.write(colors.gray('×')); + process.stdout.write(this.screen.colors.gray('×')); return; } switch (test.outcome()) { - case 'expected': process.stdout.write(colors.green('·')); break; - case 'unexpected': process.stdout.write(colors.red(result.status === 'timedOut' ? 'T' : 'F')); break; - case 'flaky': process.stdout.write(colors.yellow('±')); break; + case 'expected': process.stdout.write(this.screen.colors.green('·')); break; + case 'unexpected': process.stdout.write(this.screen.colors.red(result.status === 'timedOut' ? 'T' : 'F')); break; + case 'flaky': process.stdout.write(this.screen.colors.yellow('±')); break; } } override onError(error: TestError): void { super.onError(error); - console.log('\n' + formatError(error, colors.enabled).message); + console.log('\n' + this.formatError(error).message); this._counter = 0; } diff --git a/packages/playwright/src/reporters/github.ts b/packages/playwright/src/reporters/github.ts index c178cce64d..e7ec7198fb 100644 --- a/packages/playwright/src/reporters/github.ts +++ b/packages/playwright/src/reporters/github.ts @@ -16,7 +16,7 @@ import { ms as milliseconds } from 'playwright-core/lib/utilsBundle'; import path from 'path'; -import { BaseReporter, colors, formatError, formatResultFailure, formatRetry, formatTestHeader, formatTestTitle, stripAnsiEscapes } from './base'; +import { TerminalReporter, formatResultFailure, formatRetry, noColors, stripAnsiEscapes } from './base'; import type { TestCase, FullResult, TestError } from '../../types/testReporter'; type GitHubLogType = 'debug' | 'notice' | 'warning' | 'error'; @@ -56,9 +56,14 @@ class GitHubLogger { } } -export class GitHubReporter extends BaseReporter { +export class GitHubReporter extends TerminalReporter { githubLogger = new GitHubLogger(); + constructor(options: { omitFailures?: boolean } = {}) { + super(options); + this.screen = { ...this.screen, colors: noColors }; + } + printsToStdio() { return false; } @@ -69,7 +74,7 @@ export class GitHubReporter extends BaseReporter { } override onError(error: TestError) { - const errorMessage = formatError(error, false).message; + const errorMessage = this.formatError(error).message; this.githubLogger.error(errorMessage); } @@ -100,10 +105,10 @@ export class GitHubReporter extends BaseReporter { private _printFailureAnnotations(failures: TestCase[]) { failures.forEach((test, index) => { - const title = formatTestTitle(this.config, test); - const header = formatTestHeader(this.config, test, { indent: ' ', index: index + 1, mode: 'error' }); + const title = this.formatTestTitle(test); + const header = this.formatTestHeader(test, { indent: ' ', index: index + 1, mode: 'error' }); for (const result of test.results) { - const errors = formatResultFailure(test, result, ' ', colors.enabled); + const errors = formatResultFailure(this.screen, test, result, ' '); for (const error of errors) { const options: GitHubLogOptions = { file: workspaceRelativePath(error.location?.file || test.location.file), @@ -113,7 +118,7 @@ export class GitHubReporter extends BaseReporter { options.line = error.location.line; options.col = error.location.column; } - const message = [header, ...formatRetry(result), error.message].join('\n'); + const message = [header, ...formatRetry(this.screen, result), error.message].join('\n'); this.githubLogger.error(message, options); } } diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 62158eef6d..e14be98f63 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import { open } from 'playwright-core/lib/utilsBundle'; +import { colors, open } from 'playwright-core/lib/utilsBundle'; import { MultiMap, getPackageManagerExecCommand } from 'playwright-core/lib/utils'; import fs from 'fs'; import path from 'path'; @@ -23,7 +23,7 @@ import { Transform } from 'stream'; import { codeFrameColumns } from '../transform/babelBundle'; import type * as api from '../../types/testReporter'; import { HttpServer, assert, calculateSha1, copyFileAndMakeWritable, gracefullyProcessExitDoNotHang, removeFolders, sanitizeForFilePath, toPosixPath } from 'playwright-core/lib/utils'; -import { colors, formatError, formatResultFailure, stripAnsiEscapes } from './base'; +import { formatError, formatResultFailure, internalScreen, stripAnsiEscapes } from './base'; import { resolveReporterOutputPath } from '../util'; import type { Metadata } from '../../types/test'; import type { ZipFile } from 'playwright-core/lib/zipBundle'; @@ -297,7 +297,7 @@ class HtmlBuilder { files: [...data.values()].map(e => e.testFileSummary), projectNames: projectSuites.map(r => r.project()!.name), stats: { ...[...data.values()].reduce((a, e) => addStats(a, e.testFileSummary.stats), emptyStats()) }, - errors: topLevelErrors.map(error => formatError(error, true).message), + errors: topLevelErrors.map(error => formatError(internalScreen, error).message), }; htmlReport.files.sort((f1, f2) => { const w1 = f1.stats.unexpected * 1000 + f1.stats.flaky; @@ -506,7 +506,7 @@ class HtmlBuilder { startTime: result.startTime.toISOString(), retry: result.retry, steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)), - errors: formatResultFailure(test, result, '', true).map(error => error.message), + errors: formatResultFailure(internalScreen, test, result, '').map(error => error.message), status: result.status, attachments: this._serializeAttachments([ ...result.attachments, diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 3526f89e0d..da5e667ffd 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -18,7 +18,7 @@ import fs from 'fs'; import { codeFrameColumns } from '../transform/babelBundle'; import type { FullConfig, TestCase, TestError, TestResult, FullResult, TestStep } from '../../types/testReporter'; import { Suite } from '../common/test'; -import { colors, prepareErrorStack, relativeFilePath } from './base'; +import { internalScreen, prepareErrorStack, relativeFilePath } from './base'; import type { ReporterV2 } from './reporterV2'; import { monotonicTime } from 'playwright-core/lib/utils'; import { Multiplexer } from './multiplexer'; @@ -125,7 +125,7 @@ function addLocationAndSnippetToError(config: FullConfig, error: TestError, file const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode: true }); // Convert /var/folders to /private/var/folders on Mac. if (!file || fs.realpathSync(file) !== location.file) { - tokens.push(colors.gray(` at `) + `${relativeFilePath(config, location.file)}:${location.line}`); + tokens.push(internalScreen.colors.gray(` at `) + `${relativeFilePath(internalScreen, config, location.file)}:${location.line}`); tokens.push(''); } tokens.push(codeFrame); diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index e2b7ddf872..3c827aea78 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep, JSONReportError } from '../../types/testReporter'; -import { formatError, prepareErrorStack, resolveOutputFile } from './base'; +import { formatError, nonTerminalScreen, prepareErrorStack, resolveOutputFile } from './base'; import { MultiMap, toPosixPath } from 'playwright-core/lib/utils'; import { getProjectId } from '../common/config'; import type { ReporterV2 } from './reporterV2'; @@ -222,7 +222,7 @@ class JSONReporter implements ReporterV2 { } private _serializeError(error: TestError): JSONReportError { - return formatError(error, true); + return formatError(nonTerminalScreen, error); } private _serializeTestStep(step: TestStep): JSONReportTestStep { diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts index f193f4f79d..9140fb19d8 100644 --- a/packages/playwright/src/reporters/junit.ts +++ b/packages/playwright/src/reporters/junit.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import type { FullConfig, FullResult, Suite, TestCase } from '../../types/testReporter'; -import { formatFailure, resolveOutputFile, stripAnsiEscapes } from './base'; +import { formatFailure, nonTerminalScreen, resolveOutputFile, stripAnsiEscapes } from './base'; import { getAsBooleanFromENV } from 'playwright-core/lib/utils'; import type { ReporterV2 } from './reporterV2'; @@ -188,7 +188,7 @@ class JUnitReporter implements ReporterV2 { message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`, type: 'FAILURE', }, - text: stripAnsiEscapes(formatFailure(this.config, test)) + text: stripAnsiEscapes(formatFailure(nonTerminalScreen, this.config, test)) }); } diff --git a/packages/playwright/src/reporters/line.ts b/packages/playwright/src/reporters/line.ts index de5fc61703..96e393c1cf 100644 --- a/packages/playwright/src/reporters/line.ts +++ b/packages/playwright/src/reporters/line.ts @@ -14,10 +14,10 @@ * limitations under the License. */ -import { colors, BaseReporter, formatError, formatFailure, formatTestTitle } from './base'; +import { TerminalReporter } from './base'; import type { TestCase, Suite, TestResult, FullResult, TestStep, TestError } from '../../types/testReporter'; -class LineReporter extends BaseReporter { +class LineReporter extends TerminalReporter { private _current = 0; private _failures = 0; private _lastTest: TestCase | undefined; @@ -50,7 +50,7 @@ class LineReporter extends BaseReporter { stream.write(`\u001B[1A\u001B[2K`); if (test && this._lastTest !== test) { // Write new header for the output. - const title = colors.dim(formatTestTitle(this.config, test)); + const title = this.screen.colors.dim(this.formatTestTitle(test)); stream.write(this.fitToScreen(title) + `\n`); this._lastTest = test; } @@ -82,7 +82,7 @@ class LineReporter extends BaseReporter { if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected' || result.status === 'interrupted')) { if (!process.env.PW_TEST_DEBUG_REPORTERS) process.stdout.write(`\u001B[1A\u001B[2K`); - console.log(formatFailure(this.config, test, ++this._failures)); + console.log(this.formatFailure(test, ++this._failures)); console.log(); } } @@ -90,8 +90,8 @@ class LineReporter extends BaseReporter { private _updateLine(test: TestCase, result: TestResult, step?: TestStep) { const retriesPrefix = this.totalTestCount < this._current ? ` (retries)` : ``; const prefix = `[${this._current}/${this.totalTestCount}]${retriesPrefix} `; - const currentRetrySuffix = result.retry ? colors.yellow(` (retry #${result.retry})`) : ''; - const title = formatTestTitle(this.config, test, step) + currentRetrySuffix; + const currentRetrySuffix = result.retry ? this.screen.colors.yellow(` (retry #${result.retry})`) : ''; + const title = this.formatTestTitle(test, step) + currentRetrySuffix; if (process.env.PW_TEST_DEBUG_REPORTERS) process.stdout.write(`${prefix + title}\n`); else @@ -101,7 +101,7 @@ class LineReporter extends BaseReporter { override onError(error: TestError): void { super.onError(error); - const message = formatError(error, colors.enabled).message + '\n'; + const message = this.formatError(error).message + '\n'; if (!process.env.PW_TEST_DEBUG_REPORTERS && this._didBegin) process.stdout.write(`\u001B[1A\u001B[2K`); process.stdout.write(message); diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts index 8704bfe104..4f885946e5 100644 --- a/packages/playwright/src/reporters/list.ts +++ b/packages/playwright/src/reporters/list.ts @@ -15,7 +15,7 @@ */ import { ms as milliseconds } from 'playwright-core/lib/utilsBundle'; -import { colors, BaseReporter, formatError, formatTestTitle, isTTY, stepSuffix, stripAnsiEscapes, ttyWidth } from './base'; +import { TerminalReporter, stepSuffix, stripAnsiEscapes } from './base'; import type { FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; import { getAsBooleanFromENV } from 'playwright-core/lib/utils'; @@ -24,7 +24,7 @@ const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && proces const POSITIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'ok' : '✓'; const NEGATIVE_STATUS_MARK = DOES_NOT_SUPPORT_UTF8_IN_TERMINAL ? 'x' : '✘'; -class ListReporter extends BaseReporter { +class ListReporter extends TerminalReporter { private _lastRow = 0; private _lastColumn = 0; private _testRows = new Map(); @@ -52,12 +52,12 @@ class ListReporter extends BaseReporter { const index = String(this._resultIndex.size + 1); this._resultIndex.set(result, index); - if (!isTTY) + if (!this.screen.isTTY) return; this._maybeWriteNewLine(); this._testRows.set(test, this._lastRow); const prefix = this._testPrefix(index, ''); - const line = colors.dim(formatTestTitle(this.config, test)) + this._retrySuffix(result); + const line = this.screen.colors.dim(this.formatTestTitle(test)) + this._retrySuffix(result); this._appendLine(line, prefix); } @@ -87,17 +87,17 @@ class ListReporter extends BaseReporter { return; const testIndex = this._resultIndex.get(result) || ''; - if (!isTTY) + if (!this.screen.isTTY) return; if (this._printSteps) { this._maybeWriteNewLine(); this._stepRows.set(step, this._lastRow); const prefix = this._testPrefix(this.getStepIndex(testIndex, result, step), ''); - const line = test.title + colors.dim(stepSuffix(step)); + const line = test.title + this.screen.colors.dim(stepSuffix(step)); this._appendLine(line, prefix); } else { - this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); + this._updateLine(this._testRows.get(test)!, this.screen.colors.dim(this.formatTestTitle(test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); } } @@ -107,20 +107,20 @@ class ListReporter extends BaseReporter { const testIndex = this._resultIndex.get(result) || ''; if (!this._printSteps) { - if (isTTY) - this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step.parent)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); + if (this.screen.isTTY) + this._updateLine(this._testRows.get(test)!, this.screen.colors.dim(this.formatTestTitle(test, step.parent)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); return; } const index = this.getStepIndex(testIndex, result, step); - const title = isTTY ? test.title + colors.dim(stepSuffix(step)) : formatTestTitle(this.config, test, step); + const title = this.screen.isTTY ? test.title + this.screen.colors.dim(stepSuffix(step)) : this.formatTestTitle(test, step); const prefix = this._testPrefix(index, ''); let text = ''; if (step.error) - text = colors.red(title); + text = this.screen.colors.red(title); else text = title; - text += colors.dim(` (${milliseconds(step.duration)})`); + text += this.screen.colors.dim(` (${milliseconds(step.duration)})`); this._updateOrAppendLine(this._stepRows.get(step)!, text, prefix); } @@ -134,7 +134,7 @@ class ListReporter extends BaseReporter { private _updateLineCountAndNewLineFlagForOutput(text: string) { this._needNewLine = text[text.length - 1] !== '\n'; - if (!ttyWidth) + if (!this.screen.ttyWidth) return; for (const ch of text) { if (ch === '\n') { @@ -143,7 +143,7 @@ class ListReporter extends BaseReporter { continue; } ++this._lastColumn; - if (this._lastColumn > ttyWidth) { + if (this._lastColumn > this.screen.ttyWidth) { this._lastColumn = 0; ++this._lastRow; } @@ -161,7 +161,7 @@ class ListReporter extends BaseReporter { override onTestEnd(test: TestCase, result: TestResult) { super.onTestEnd(test, result); - const title = formatTestTitle(this.config, test); + const title = this.formatTestTitle(test); let prefix = ''; let text = ''; @@ -174,26 +174,26 @@ class ListReporter extends BaseReporter { } if (result.status === 'skipped') { - prefix = this._testPrefix(index, colors.green('-')); + prefix = this._testPrefix(index, this.screen.colors.green('-')); // Do not show duration for skipped. - text = colors.cyan(title) + this._retrySuffix(result); + text = this.screen.colors.cyan(title) + this._retrySuffix(result); } else { const statusMark = result.status === 'passed' ? POSITIVE_STATUS_MARK : NEGATIVE_STATUS_MARK; if (result.status === test.expectedStatus) { - prefix = this._testPrefix(index, colors.green(statusMark)); + prefix = this._testPrefix(index, this.screen.colors.green(statusMark)); text = title; } else { - prefix = this._testPrefix(index, colors.red(statusMark)); - text = colors.red(title); + prefix = this._testPrefix(index, this.screen.colors.red(statusMark)); + text = this.screen.colors.red(title); } - text += this._retrySuffix(result) + colors.dim(` (${milliseconds(result.duration)})`); + text += this._retrySuffix(result) + this.screen.colors.dim(` (${milliseconds(result.duration)})`); } this._updateOrAppendLine(this._testRows.get(test)!, text, prefix); } private _updateOrAppendLine(row: number, text: string, prefix: string) { - if (isTTY) { + if (this.screen.isTTY) { this._updateLine(row, text, prefix); } else { this._maybeWriteNewLine(); @@ -234,17 +234,17 @@ class ListReporter extends BaseReporter { private _testPrefix(index: string, statusMark: string) { const statusMarkLength = stripAnsiEscapes(statusMark).length; - return ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + colors.dim(index + ' '); + return ' ' + statusMark + ' '.repeat(3 - statusMarkLength) + this.screen.colors.dim(index + ' '); } private _retrySuffix(result: TestResult) { - return (result.retry ? colors.yellow(` (retry #${result.retry})`) : ''); + return (result.retry ? this.screen.colors.yellow(` (retry #${result.retry})`) : ''); } override onError(error: TestError): void { super.onError(error); this._maybeWriteNewLine(); - const message = formatError(error, colors.enabled).message + '\n'; + const message = this.formatError(error).message + '\n'; this._updateLineCountAndNewLineFlagForOutput(message); process.stdout.write(message); } diff --git a/packages/playwright/src/reporters/markdown.ts b/packages/playwright/src/reporters/markdown.ts index fe8c83cdd4..2b6bcbf063 100644 --- a/packages/playwright/src/reporters/markdown.ts +++ b/packages/playwright/src/reporters/markdown.ts @@ -18,15 +18,14 @@ import fs from 'fs'; import path from 'path'; import type { FullResult, TestCase } from '../../types/testReporter'; import { resolveReporterOutputPath } from '../util'; -import { BaseReporter, formatTestTitle } from './base'; +import { TerminalReporter } from './base'; type MarkdownReporterOptions = { configDir: string, outputFile?: string; }; - -class MarkdownReporter extends BaseReporter { +class MarkdownReporter extends TerminalReporter { private _options: MarkdownReporterOptions; constructor(options: MarkdownReporterOptions) { @@ -75,7 +74,7 @@ class MarkdownReporter extends BaseReporter { private _printTestList(prefix: string, tests: TestCase[], lines: string[], suffix?: string) { for (const test of tests) - lines.push(`${prefix} ${formatTestTitle(this.config, test)}${suffix || ''}`); + lines.push(`${prefix} ${this.formatTestTitle(test)}${suffix || ''}`); lines.push(``); } } diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index 2bab152f08..b25cb84154 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -16,7 +16,8 @@ import path from 'path'; import type { FullConfig, TestError } from '../../types/testReporter'; -import { colors, formatError } from '../reporters/base'; +import { formatError, terminalScreen } from '../reporters/base'; +import type { Screen } from '../reporters/base'; import DotReporter from '../reporters/dot'; import EmptyReporter from '../reporters/empty'; import GitHubReporter from '../reporters/github'; @@ -88,14 +89,14 @@ interface ErrorCollectingReporter extends ReporterV2 { errors(): TestError[]; } -export function createErrorCollectingReporter(writeToConsole?: boolean): ErrorCollectingReporter { +export function createErrorCollectingReporter(screen: Screen, writeToConsole?: boolean): ErrorCollectingReporter { const errors: TestError[] = []; return { version: () => 'v2', onError(error: TestError) { errors.push(error); if (writeToConsole) - process.stdout.write(formatError(error, colors.enabled).message + '\n'); + process.stdout.write(formatError(screen, error).message + '\n'); }, errors: () => errors, }; @@ -160,6 +161,6 @@ class ListModeReporter implements ReporterV2 { onError(error: TestError) { // eslint-disable-next-line no-console - console.error('\n' + formatError(error, false).message); + console.error('\n' + formatError(terminalScreen, error).message); } } diff --git a/packages/playwright/src/runner/runner.ts b/packages/playwright/src/runner/runner.ts index 5a015ec755..a1e73657c9 100644 --- a/packages/playwright/src/runner/runner.ts +++ b/packages/playwright/src/runner/runner.ts @@ -24,6 +24,7 @@ import type { FullConfigInternal } from '../common/config'; import { affectedTestFiles } from '../transform/compilationCache'; import { InternalReporter } from '../reporters/internalReporter'; import { LastRunReporter } from './lastRun'; +import { terminalScreen } from '../reporters/base'; type ProjectConfigWithFiles = { name: string; @@ -98,7 +99,7 @@ export class Runner { } async findRelatedTestFiles(files: string[]): Promise { - const errorReporter = createErrorCollectingReporter(); + const errorReporter = createErrorCollectingReporter(terminalScreen); const reporter = new InternalReporter([errorReporter]); const status = await runTasks(new TestRun(this._config, reporter), [ ...createPluginSetupTasks(this._config), @@ -110,7 +111,7 @@ export class Runner { } async runDevServer() { - const reporter = new InternalReporter([createErrorCollectingReporter(true)]); + const reporter = new InternalReporter([createErrorCollectingReporter(terminalScreen, true)]); const status = await runTasks(new TestRun(this._config, reporter), [ ...createPluginSetupTasks(this._config), createLoadTask('in-process', { failOnLoadErrors: true, filterOnly: false }), @@ -121,7 +122,7 @@ export class Runner { } async clearCache() { - const reporter = new InternalReporter([createErrorCollectingReporter(true)]); + const reporter = new InternalReporter([createErrorCollectingReporter(terminalScreen, true)]); const status = await runTasks(new TestRun(this._config, reporter), [ ...createPluginSetupTasks(this._config), createClearCacheTask(this._config), diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.ts index 08fa4b9353..e3a61329e2 100644 --- a/packages/playwright/src/runner/testServer.ts +++ b/packages/playwright/src/runner/testServer.ts @@ -38,6 +38,7 @@ import { serializeError } from '../util'; import { baseFullConfig } from '../isomorphic/teleReceiver'; import { InternalReporter } from '../reporters/internalReporter'; import type { ReporterV2 } from '../reporters/reporterV2'; +import { internalScreen } from '../reporters/base'; const originalStdoutWrite = process.stdout.write; const originalStderrWrite = process.stderr.write; @@ -359,7 +360,7 @@ export class TestServerDispatcher implements TestServerInterface { } async findRelatedTestFiles(params: Parameters[0]): ReturnType { - const errorReporter = createErrorCollectingReporter(); + const errorReporter = createErrorCollectingReporter(internalScreen); const reporter = new InternalReporter([errorReporter]); const config = await this._loadConfigOrReportError(reporter); if (!config) diff --git a/packages/playwright/src/runner/watchMode.ts b/packages/playwright/src/runner/watchMode.ts index 310cdeb546..5c8ddf34a0 100644 --- a/packages/playwright/src/runner/watchMode.ts +++ b/packages/playwright/src/runner/watchMode.ts @@ -21,7 +21,7 @@ import type { ConfigLocation } from '../common/config'; import type { FullResult } from '../../types/testReporter'; import { colors } from 'playwright-core/lib/utilsBundle'; import { enquirer } from '../utilsBundle'; -import { separator } from '../reporters/base'; +import { separator, terminalScreen } from '../reporters/base'; import { PlaywrightServer } from 'playwright-core/lib/remote/playwrightServer'; import { TestServerDispatcher } from './testServer'; import { EventEmitter } from 'stream'; @@ -332,7 +332,7 @@ function readCommand() { return 'exit'; if (name === 'h') { - process.stdout.write(`${separator()} + process.stdout.write(`${separator(terminalScreen)} Run tests ${colors.bold('enter')} ${colors.dim('run tests')} ${colors.bold('f')} ${colors.dim('run failed tests')} @@ -380,7 +380,7 @@ function printConfiguration(options: WatchModeOptions, title?: string) { tokens.push(colors.dim(`(${title})`)); tokens.push(colors.dim(`#${seq++}`)); const lines: string[] = []; - const sep = separator(); + const sep = separator(terminalScreen); lines.push('\x1Bc' + sep); lines.push(`${tokens.join(' ')}`); lines.push(`${colors.dim('Show & reuse browser:')} ${colors.bold(showBrowserServer ? 'on' : 'off')}`); @@ -388,7 +388,7 @@ function printConfiguration(options: WatchModeOptions, title?: string) { } function printBufferPrompt(dirtyTestFiles: Set, rootDir: string) { - const sep = separator(); + const sep = separator(terminalScreen); process.stdout.write('\x1Bc'); process.stdout.write(`${sep}\n`); @@ -404,7 +404,7 @@ function printBufferPrompt(dirtyTestFiles: Set, rootDir: string) { } function printPrompt() { - const sep = separator(); + const sep = separator(terminalScreen); process.stdout.write(` ${sep} ${colors.dim('Waiting for file changes. Press')} ${colors.bold('enter')} ${colors.dim('to run tests')}, ${colors.bold('q')} ${colors.dim('to quit or')} ${colors.bold('h')} ${colors.dim('for more options.')} diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 5ddb271b70..e16fbcc460 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -882,7 +882,10 @@ test('multiple output reports based on config', async ({ runInlineTest, mergeRep const reportFiles = await fs.promises.readdir(reportDir); reportFiles.sort(); expect(reportFiles).toEqual(['report-1.zip', 'report-2.zip']); - const { exitCode, output } = await mergeReports(reportDir, undefined, { additionalArgs: ['--config', test.info().outputPath('merged/playwright.config.ts')] }); + const { exitCode, output } = await mergeReports(reportDir, undefined, { + cwd: test.info().outputPath('merged'), + additionalArgs: ['--config', 'playwright.config.ts'], + }); expect(exitCode).toBe(0); // Check that line reporter was called.