diff --git a/docs/src/test-advanced-js.md b/docs/src/test-advanced-js.md index 1c668d98d4..2845d4573f 100644 --- a/docs/src/test-advanced-js.md +++ b/docs/src/test-advanced-js.md @@ -38,7 +38,7 @@ These options would be typically different between local development and CI oper - `'failures-only'` - only preserve output for failed tests. - `projects: Project[]` - Multiple [projects](#projects) configuration. - `quiet: boolean` - Whether to suppress stdout and stderr from the tests. -- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit'` - The reporter to use. See [reporters](./test-reporters.md) for details. +- `reporter: 'list' | 'line' | 'dot' | 'json' | 'junit' | 'github'` - The reporter to use. See [reporters](./test-reporters.md) for details. - `reportSlowTests: { max: number, threshold: number } | null` - Whether to report slow tests. When `null`, slow tests are not reported. Otherwise, tests that took more than `threshold` milliseconds are reported as slow, but no more than `max` number of them. Passing zero as `max` reports all slow tests that exceed the threshold. - `shard: { total: number, current: number } | null` - [Shard](./test-parallel.md#shard-tests-between-multiple-machines) information. - `updateSnapshots: boolean` - Whether to update expected snapshots with the actual results produced by the test run. diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index 486ed8de42..db8464e2bf 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -98,6 +98,34 @@ const config: PlaywrightTestConfig = { export default config; ``` +### Reporter for GitHub Actions + +You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + // 'github' for GitHub Actions CI to generate annotations, default 'list' when running locally + reporter: process.env.CI ? 'github' : 'list', +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +const config: PlaywrightTestConfig = { + // 'github' for GitHub Actions CI to generate annotations, default 'list' when running locally + reporter: process.env.CI ? 'github' : 'list', +}; +export default config; +``` + ## Built-in reporters All built-in reporters show detailed information about failures, and mostly differ in verbosity for successful runs. diff --git a/package-lock.json b/package-lock.json index 4f17825d9b..14dce33f9e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -62,6 +62,7 @@ "@types/extract-zip": "^1.6.2", "@types/mime": "^2.0.3", "@types/minimatch": "^3.0.3", + "@types/ms": "^0.7.31", "@types/node": "^14.17.15", "@types/pixelmatch": "^5.2.1", "@types/pngjs": "^3.4.2", @@ -1485,6 +1486,12 @@ "integrity": "sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==", "dev": true }, + "node_modules/@types/ms": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz", + "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==", + "dev": true + }, "node_modules/@types/node": { "version": "14.17.15", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.15.tgz", @@ -11936,6 +11943,12 @@ "integrity": "sha512-1z8k4wzFnNjVK/tlxvrWuK5WMt6mydWWP7+zvH5eFep4oj+UkrfiJTRtjCeBXNpwaA/FYqqtb4/QS4ianFpIRA==", "dev": true }, + "@types/ms": { + "version": "0.7.31", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-0.7.31.tgz", + "integrity": "sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==", + "dev": true + }, "@types/node": { "version": "14.17.15", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.17.15.tgz", diff --git a/package.json b/package.json index df56ca2bce..653e226a08 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "@types/extract-zip": "^1.6.2", "@types/mime": "^2.0.3", "@types/minimatch": "^3.0.3", + "@types/ms": "^0.7.31", "@types/node": "^14.17.15", "@types/pixelmatch": "^5.2.1", "@types/pngjs": "^3.4.2", @@ -136,4 +137,4 @@ "xml2js": "^0.4.23", "yaml": "^1.10.0" } -} +} \ No newline at end of file diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index 3f301c1cfd..093a57f1c4 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -17,7 +17,6 @@ import { codeFrameColumns } from '@babel/code-frame'; import colors from 'colors/safe'; import fs from 'fs'; -// @ts-ignore import milliseconds from 'ms'; import path from 'path'; import StackUtils from 'stack-utils'; @@ -25,9 +24,35 @@ import { FullConfig, TestCase, Suite, TestResult, TestError, Reporter, FullResul const stackUtils = new StackUtils(); -type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; +export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; +export const kOutputSymbol = Symbol('output'); export type PositionInFile = { column: number; line: number }; -const kOutputSymbol = Symbol('output'); + +type Annotation = { + filePath: string; + title: string; + message: string; + position?: PositionInFile; +}; + +type FailureDetails = { + tokens: string[]; + position?: PositionInFile; +}; + +type ErrorDetails = { + message: string; + position?: PositionInFile; +}; + +type TestSummary = { + skipped: number; + expected: number; + skippedWithError: TestCase[]; + unexpected: TestCase[]; + flaky: TestCase[]; + failuresToPrint: TestCase[]; +}; export class BaseReporter implements Reporter { duration = 0; @@ -76,21 +101,40 @@ export class BaseReporter implements Reporter { this.result = result; } - private _printSlowTests() { + protected getSlowTests(): [string, number][] { if (!this.config.reportSlowTests) - return; + return []; const fileDurations = [...this.fileDurations.entries()]; fileDurations.sort((a, b) => b[1] - a[1]); const count = Math.min(fileDurations.length, this.config.reportSlowTests.max || Number.POSITIVE_INFINITY); - for (let i = 0; i < count; ++i) { - const duration = fileDurations[i][1]; - if (duration <= this.config.reportSlowTests.threshold) - break; - console.log(colors.yellow(' Slow test: ') + fileDurations[i][0] + colors.yellow(` (${milliseconds(duration)})`)); - } + const threshold = this.config.reportSlowTests.threshold; + return fileDurations.filter(([,duration]) => duration > threshold).slice(0, count); } - epilogue(full: boolean) { + protected generateSummaryMessage({ skipped, expected, unexpected, flaky }: TestSummary) { + const tokens: string[] = []; + tokens.push(''); + if (unexpected.length) { + tokens.push(colors.red(` ${unexpected.length} failed`)); + for (const test of unexpected) + tokens.push(colors.red(formatTestHeader(this.config, test, ' '))); + } + if (flaky.length) { + tokens.push(colors.yellow(` ${flaky.length} flaky`)); + for (const test of flaky) + tokens.push(colors.yellow(formatTestHeader(this.config, test, ' '))); + } + if (skipped) + tokens.push(colors.yellow(` ${skipped} skipped`)); + if (expected) + tokens.push(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); + if (this.result.status === 'timedout') + tokens.push(colors.red(` Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`)); + + return tokens.join('\n'); + } + + protected generateSummary(): TestSummary { let skipped = 0; let expected = 0; const skippedWithError: TestCase[] = []; @@ -112,96 +156,126 @@ export class BaseReporter implements Reporter { }); const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError]; - if (full && failuresToPrint.length) { - console.log(''); - this._printFailures(failuresToPrint); - } + return { + skipped, + expected, + skippedWithError, + unexpected, + flaky, + failuresToPrint + }; + } + epilogue(full: boolean) { + const summary = this.generateSummary(); + const summaryMessage = this.generateSummaryMessage(summary); + if (full && summary.failuresToPrint.length) + this._printFailures(summary.failuresToPrint); this._printSlowTests(); - - console.log(''); - if (unexpected.length) { - console.log(colors.red(` ${unexpected.length} failed`)); - for (const test of unexpected) - console.log(colors.red(formatTestHeader(this.config, test, ' '))); - } - if (flaky.length) { - console.log(colors.yellow(` ${flaky.length} flaky`)); - for (const test of flaky) - console.log(colors.yellow(formatTestHeader(this.config, test, ' '))); - } - if (skipped) - console.log(colors.yellow(` ${skipped} skipped`)); - if (expected) - console.log(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); - if (this.result.status === 'timedout') - console.log(colors.red(` Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`)); + this._printSummary(summaryMessage); } private _printFailures(failures: TestCase[]) { + console.log(''); failures.forEach((test, index) => { - console.log(formatFailure(this.config, test, index + 1, this.printTestOutput)); + console.log(formatFailure(this.config, test, { + index: index + 1, + includeStdio: this.printTestOutput + }).message); }); } + private _printSlowTests() { + this.getSlowTests().forEach(([file, duration]) => { + console.log(colors.yellow(' Slow test: ') + file + colors.yellow(` (${milliseconds(duration)})`)); + }); + } + + private _printSummary(summary: string){ + console.log(''); + console.log(summary); + } + willRetry(test: TestCase): boolean { return test.outcome() === 'unexpected' && test.results.length <= test.retries; } } -export function formatFailure(config: FullConfig, test: TestCase, index?: number, stdio?: boolean): string { +export function formatFailure(config: FullConfig, test: TestCase, options: {index?: number, includeStdio?: boolean, includeAttachments?: boolean, filePath?: string} = {}): { + message: string, + annotations: Annotation[] +} { + const { index, includeStdio, includeAttachments = true, filePath } = options; const lines: string[] = []; - lines.push(colors.red(formatTestHeader(config, test, ' ', index))); + const title = formatTestTitle(config, test); + const annotations: Annotation[] = []; + const header = formatTestHeader(config, test, ' ', index); + lines.push(colors.red(header)); for (const result of test.results) { - const resultTokens = formatResultFailure(test, result, ' '); + const resultLines: string[] = []; + const { tokens: resultTokens, position } = formatResultFailure(test, result, ' '); if (!resultTokens.length) continue; if (result.retry) { - lines.push(''); - lines.push(colors.gray(pad(` Retry #${result.retry}`, '-'))); + resultLines.push(''); + resultLines.push(colors.gray(pad(` Retry #${result.retry}`, '-'))); } - lines.push(...resultTokens); - for (let i = 0; i < result.attachments.length; ++i) { - const attachment = result.attachments[i]; - lines.push(''); - lines.push(colors.cyan(pad(` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-'))); - if (attachment.path) { - const relativePath = path.relative(process.cwd(), attachment.path); - lines.push(colors.cyan(` ${relativePath}`)); - // Make this extensible - if (attachment.name === 'trace') { - lines.push(colors.cyan(` Usage:`)); - lines.push(''); - lines.push(colors.cyan(` npx playwright show-trace ${relativePath}`)); - lines.push(''); - } - } else { - if (attachment.contentType.startsWith('text/')) { - let text = attachment.body!.toString(); - if (text.length > 300) - text = text.slice(0, 300) + '...'; - lines.push(colors.cyan(` ${text}`)); + resultLines.push(...resultTokens); + if (includeAttachments) { + for (let i = 0; i < result.attachments.length; ++i) { + const attachment = result.attachments[i]; + resultLines.push(''); + resultLines.push(colors.cyan(pad(` attachment #${i + 1}: ${attachment.name} (${attachment.contentType})`, '-'))); + if (attachment.path) { + const relativePath = path.relative(process.cwd(), attachment.path); + resultLines.push(colors.cyan(` ${relativePath}`)); + // Make this extensible + if (attachment.name === 'trace') { + resultLines.push(colors.cyan(` Usage:`)); + resultLines.push(''); + resultLines.push(colors.cyan(` npx playwright show-trace ${relativePath}`)); + resultLines.push(''); + } + } else { + if (attachment.contentType.startsWith('text/')) { + let text = attachment.body!.toString(); + if (text.length > 300) + text = text.slice(0, 300) + '...'; + resultLines.push(colors.cyan(` ${text}`)); + } } + resultLines.push(colors.cyan(pad(' ', '-'))); } - lines.push(colors.cyan(pad(' ', '-'))); } const output = ((result as any)[kOutputSymbol] || []) as TestResultOutput[]; - if (stdio && output.length) { + if (includeStdio && output.length) { const outputText = output.map(({ chunk, type }) => { const text = chunk.toString('utf8'); if (type === 'stderr') return colors.red(stripAnsiEscapes(text)); return text; }).join(''); - lines.push(''); - lines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-')); + resultLines.push(''); + resultLines.push(colors.gray(pad('--- Test output', '-')) + '\n\n' + outputText + '\n' + pad('', '-')); } + if (filePath) { + annotations.push({ + filePath, + position, + title, + message: [header, ...resultLines].join('\n'), + }); + } + lines.push(...resultLines); } lines.push(''); - return lines.join('\n'); + return { + message: lines.join('\n'), + annotations + }; } -export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string): string[] { +export function formatResultFailure(test: TestCase, result: TestResult, initialIndent: string): FailureDetails { const resultTokens: string[] = []; if (result.status === 'timedOut') { resultTokens.push(''); @@ -211,9 +285,15 @@ export function formatResultFailure(test: TestCase, result: TestResult, initialI resultTokens.push(''); resultTokens.push(indent(colors.red(`Expected to fail, but passed.`), initialIndent)); } - if (result.error !== undefined) - resultTokens.push(indent(formatError(result.error, test.location.file), initialIndent)); - return resultTokens; + let error: ErrorDetails | undefined = undefined; + if (result.error !== undefined) { + error = formatError(result.error, test.location.file); + resultTokens.push(indent(error.message, initialIndent)); + } + return { + tokens: resultTokens, + position: error?.position, + }; } function relativeTestPath(config: FullConfig, test: TestCase): string { @@ -239,14 +319,16 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in return pad(header, '='); } -export function formatError(error: TestError, file?: string) { +export function formatError(error: TestError, file?: string): ErrorDetails { const stack = error.stack; const tokens = ['']; + let positionInFile: PositionInFile | undefined; if (stack) { const { message, stackLines, position } = prepareErrorStack( stack, file ); + positionInFile = position; tokens.push(message); const codeFrame = generateCodeFrame(file, position); @@ -261,7 +343,10 @@ export function formatError(error: TestError, file?: string) { } else if (error.value) { tokens.push(error.value); } - return tokens.join('\n'); + return { + position: positionInFile, + message: tokens.join('\n'), + }; } function pad(line: string, char: string): string { @@ -306,7 +391,7 @@ export function prepareErrorStack(stack: string, file?: string): { }; } -function positionInFile(stackLines: string[], file: string): { column: number; line: number; } | undefined { +function positionInFile(stackLines: string[], file: string): PositionInFile | undefined { // Stack will have /private/var/folders instead of /var/folders on Mac. file = fs.realpathSync(file); for (const line of stackLines) { diff --git a/src/test/reporters/github.ts b/src/test/reporters/github.ts new file mode 100644 index 0000000000..b345fd2f59 --- /dev/null +++ b/src/test/reporters/github.ts @@ -0,0 +1,123 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import milliseconds from 'ms'; +import path from 'path'; +import { BaseReporter, formatFailure } from './base'; +import { TestCase, FullResult } from '../../../types/testReporter'; + +type GitHubLogType = 'debug' | 'notice' | 'warning' | 'error'; + +type GitHubLogOptions = Partial<{ + title: string; + file: string; + col: number; + endColumn: number; + line: number; + endLine: number; +}>; + +class GitHubLogger { + private _isGitHubAction: boolean = !!process.env.GITHUB_ACTION; + + private _log(message: string, type: GitHubLogType = 'notice', options: GitHubLogOptions = {}) { + if (this._isGitHubAction) + message = message.replace(/\n/g, '%0A'); + const configs = Object.entries(options) + .map(([key, option]) => `${key}=${option}`) + .join(','); + console.log(`::${type} ${configs}::${message}`); + } + + debug(message: string, options?: GitHubLogOptions) { + this._log(message, 'debug', options); + } + + error(message: string, options?: GitHubLogOptions) { + this._log(message, 'error', options); + } + + notice(message: string, options?: GitHubLogOptions) { + this._log(message, 'notice', options); + } + + warning(message: string, options?: GitHubLogOptions) { + this._log(message, 'warning', options); + } +} + +export class GitHubReporter extends BaseReporter { + githubLogger = new GitHubLogger(); + + override async onEnd(result: FullResult) { + super.onEnd(result); + this._printAnnotations(); + } + + private _printAnnotations() { + const summary = this.generateSummary(); + const summaryMessage = this.generateSummaryMessage(summary); + if (summary.failuresToPrint.length) + this._printFailureAnnotations(summary.failuresToPrint); + this._printSlowTestAnnotations(); + this._printSummaryAnnotation(summaryMessage); + } + + private _printSlowTestAnnotations() { + this.getSlowTests().forEach(([file, duration]) => { + const filePath = workspaceRelativePath(path.join(process.cwd(), file)); + this.githubLogger.warning(`${filePath} took ${milliseconds(duration)}`, { + title: 'Slow Test', + file: filePath, + }); + }); + } + + private _printSummaryAnnotation(summary: string){ + this.githubLogger.notice(summary, { + title: '🎭 Playwright Run Summary' + }); + } + + private _printFailureAnnotations(failures: TestCase[]) { + failures.forEach((test, index) => { + const filePath = workspaceRelativePath(test.location.file); + const { annotations } = formatFailure(this.config, test, { + filePath, + index: index + 1, + includeStdio: true, + includeAttachments: false, + }); + annotations.forEach(({ filePath, title, message, position }) => { + const options: GitHubLogOptions = { + file: filePath, + title, + }; + if (position) { + options.line = position.line; + options.col = position.column; + } + this.githubLogger.error(message, options); + }); + }); + } +} + +function workspaceRelativePath(filePath: string): string { + return path.relative(process.env['GITHUB_WORKSPACE'] ?? '', filePath); +} + +export default GitHubReporter; diff --git a/src/test/reporters/junit.ts b/src/test/reporters/junit.ts index aa4875825e..d4e93b317b 100644 --- a/src/test/reporters/junit.ts +++ b/src/test/reporters/junit.ts @@ -142,7 +142,7 @@ class JUnitReporter implements Reporter { message: `${path.basename(test.location.file)}:${test.location.line}:${test.location.column} ${test.title}`, type: 'FAILURE', }, - text: stripAnsiEscapes(formatFailure(this.config, test)) + text: stripAnsiEscapes(formatFailure(this.config, test).message) }); } diff --git a/src/test/reporters/line.ts b/src/test/reporters/line.ts index 547fd653a4..cf442c6c8a 100644 --- a/src/test/reporters/line.ts +++ b/src/test/reporters/line.ts @@ -61,7 +61,9 @@ class LineReporter extends BaseReporter { process.stdout.write(`\u001B[1A\u001B[2K${title}\n`); if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) { process.stdout.write(`\u001B[1A\u001B[2K`); - console.log(formatFailure(this.config, test, ++this._failures)); + console.log(formatFailure(this.config, test, { + index: ++this._failures + }).message); console.log(); } } diff --git a/src/test/reporters/list.ts b/src/test/reporters/list.ts index f5edf1f175..83f2122df1 100644 --- a/src/test/reporters/list.ts +++ b/src/test/reporters/list.ts @@ -16,7 +16,6 @@ /* eslint-disable no-console */ import colors from 'colors/safe'; -// @ts-ignore import milliseconds from 'ms'; import { BaseReporter, formatTestTitle } from './base'; import { FullConfig, FullResult, Suite, TestCase, TestResult, TestStep } from '../../../types/testReporter'; diff --git a/src/test/reporters/raw.ts b/src/test/reporters/raw.ts index 260c9d8f73..ae306b1a5c 100644 --- a/src/test/reporters/raw.ts +++ b/src/test/reporters/raw.ts @@ -181,7 +181,7 @@ class RawReporter { startTime: result.startTime.toISOString(), duration: result.duration, status: result.status, - error: formatResultFailure(test, result, '').join('').trim(), + error: formatResultFailure(test, result, '').tokens.join('').trim(), attachments: this._createAttachments(result), steps: this._serializeSteps(test, result.steps) }; diff --git a/src/test/runner.ts b/src/test/runner.ts index bc2c49ac7f..e29c80572f 100644 --- a/src/test/runner.ts +++ b/src/test/runner.ts @@ -27,6 +27,7 @@ import { Loader } from './loader'; import { Reporter } from '../../types/testReporter'; import { Multiplexer } from './reporters/multiplexer'; import DotReporter from './reporters/dot'; +import GitHubReporter from './reporters/github'; import LineReporter from './reporters/line'; import ListReporter from './reporters/list'; import JSONReporter from './reporters/json'; @@ -68,6 +69,7 @@ export class Runner { dot: list ? ListModeReporter : DotReporter, line: list ? ListModeReporter : LineReporter, list: list ? ListModeReporter : ListReporter, + github: GitHubReporter, json: JSONReporter, junit: JUnitReporter, null: EmptyReporter, @@ -539,5 +541,5 @@ class ListModeReporter implements Reporter { } } -export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null'] as const; +export const builtInReporters = ['list', 'line', 'dot', 'json', 'junit', 'null', 'github'] as const; export type BuiltInReporter = typeof builtInReporters[number]; diff --git a/tests/playwright-test/github-reporter.spec.ts b/tests/playwright-test/github-reporter.spec.ts new file mode 100644 index 0000000000..685b440cc2 --- /dev/null +++ b/tests/playwright-test/github-reporter.spec.ts @@ -0,0 +1,88 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect, stripAscii } from './playwright-test-fixtures'; +import { relativeFilePath } from '../../src/test/util'; + +test('print GitHub annotations for success', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('example1', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }, { reporter: 'github' }, { GITHUB_ACTION: 'true' }); + const text = stripAscii(result.output); + expect(text).not.toContain('::error'); + expect(text).toContain('::notice title=🎭 Playwright Run Summary::%0A 1 passed'); + expect(result.exitCode).toBe(0); +}); + +test('print GitHub annotations with newline if not in CI', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('example1', async ({}) => { + expect(1 + 1).toBe(2); + }); + ` + }, { reporter: 'github' }, { GITHUB_ACTION: '' }); + const text = stripAscii(result.output); + expect(text).not.toContain('::error'); + expect(text).toContain(`::notice title=🎭 Playwright Run Summary:: + 1 passed `); + expect(result.exitCode).toBe(0); +}); + + +test('print GitHub annotations for failed tests', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('example', async ({}) => { + expect(1 + 1).toBe(3); + }); + ` + }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true', GITHUB_WORKSPACE: process.cwd() }); + const text = stripAscii(result.output); + const testPath = relativeFilePath(testInfo.outputPath('a.test.js')); + expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #1`); + expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #2`); + expect(text).toContain(`::error file=${testPath},title=a.test.js:6:7 › example,line=7,col=23:: 1) a.test.js:6:7 › example =======================================================================%0A%0A Retry #3`); + expect(result.exitCode).toBe(1); +}); + + +test('print GitHub annotations for slow tests', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { + reportSlowTests: { max: 0, threshold: 100 } + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('slow test', async ({}) => { + await new Promise(f => setTimeout(f, 200)); + }); + ` + }, { retries: 3, reporter: 'github' }, { GITHUB_ACTION: 'true', GITHUB_WORKSPACE: '' }); + const text = stripAscii(result.output); + expect(text).toContain('::warning title=Slow Test,file=a.test.js::a.test.js took 2'); + expect(text).toContain('::notice title=🎭 Playwright Run Summary::%0A 1 passed'); + expect(result.exitCode).toBe(0); +}); \ No newline at end of file