diff --git a/src/test/reporters/base.ts b/src/test/reporters/base.ts index de60912c7b..e6f73107e6 100644 --- a/src/test/reporters/base.ts +++ b/src/test/reporters/base.ts @@ -110,9 +110,10 @@ export class BaseReporter implements Reporter { } }); - if (full && (unexpected.length || skippedWithError.length)) { + const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError]; + if (full && failuresToPrint.length) { console.log(''); - this._printFailures([...unexpected, ...skippedWithError]); + this._printFailures(failuresToPrint); } this._printSlowTests(); @@ -147,7 +148,7 @@ export class BaseReporter implements Reporter { } willRetry(test: TestCase, result: TestResult): boolean { - return result.status !== 'passed' && result.status !== test.expectedStatus && test.results.length <= test.retries; + return test.outcome() === 'unexpected' && result.status !== 'passed' && test.results.length <= test.retries; } } diff --git a/src/test/reporters/line.ts b/src/test/reporters/line.ts index ff5c5d9517..63ebf2256c 100644 --- a/src/test/reporters/line.ts +++ b/src/test/reporters/line.ts @@ -59,7 +59,7 @@ class LineReporter extends BaseReporter { const width = process.stdout.columns! - 1; const title = `[${++this._current}/${this._total}] ${formatTestTitle(this.config, test)}`.substring(0, width); process.stdout.write(`\u001B[1A\u001B[2K${title}\n`); - if (!this.willRetry(test, result) && !test.ok()) { + if (!this.willRetry(test, result) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) { process.stdout.write(`\u001B[1A\u001B[2K`); console.log(formatFailure(this.config, test, ++this._failures)); console.log(); diff --git a/tests/playwright-test/base-reporter.spec.ts b/tests/playwright-test/base-reporter.spec.ts index b264ce422a..9653c09882 100644 --- a/tests/playwright-test/base-reporter.spec.ts +++ b/tests/playwright-test/base-reporter.spec.ts @@ -181,3 +181,32 @@ test('should print stdio for failures', async ({ runInlineTest }) => { 'my log 2\n', ].join('')); }); + +test('should print flaky failures', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('foobar', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + ` + }, { retries: '1', reporter: 'list' }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + expect(stripAscii(result.output)).toContain('expect(testInfo.retry).toBe(1)'); +}); + +test('should print flaky timeouts', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('foobar', async ({}, testInfo) => { + if (!testInfo.retry) + await new Promise(f => setTimeout(f, 2000)); + }); + ` + }, { retries: '1', reporter: 'list', timeout: '1000' }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + expect(stripAscii(result.output)).toContain('Timeout of 1000ms exceeded.'); +}); diff --git a/tests/playwright-test/dot-reporter.spec.ts b/tests/playwright-test/dot-reporter.spec.ts index 3e39218c0b..6af251995b 100644 --- a/tests/playwright-test/dot-reporter.spec.ts +++ b/tests/playwright-test/dot-reporter.spec.ts @@ -71,7 +71,7 @@ test('render flaky', async ({ runInlineTest }) => { expect(text).toContain('×××±'); expect(result.output).toContain(colors.yellow('±')); expect(text).toContain('1 flaky'); - expect(text).not.toContain('Retry #1'); + expect(text).toContain('Retry #1'); expect(result.exitCode).toBe(0); }); diff --git a/tests/playwright-test/line-reporter.spec.ts b/tests/playwright-test/line-reporter.spec.ts index ad52b485da..56bbaefa57 100644 --- a/tests/playwright-test/line-reporter.spec.ts +++ b/tests/playwright-test/line-reporter.spec.ts @@ -48,3 +48,17 @@ test('render flaky', async ({ runInlineTest }) => { expect(text).toContain('1 flaky'); expect(result.exitCode).toBe(0); }); + +test('should print flaky failures', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': ` + const { test } = pwt; + test('foobar', async ({}, testInfo) => { + expect(testInfo.retry).toBe(1); + }); + ` + }, { retries: '1', reporter: 'line' }); + expect(result.exitCode).toBe(0); + expect(result.flaky).toBe(1); + expect(stripAscii(result.output)).toContain('expect(testInfo.retry).toBe(1)'); +}); diff --git a/tests/playwright-test/stdio.spec.ts b/tests/playwright-test/stdio.spec.ts index e3f0387adb..26d87f4ffc 100644 --- a/tests/playwright-test/stdio.spec.ts +++ b/tests/playwright-test/stdio.spec.ts @@ -61,3 +61,18 @@ test('should get stdio from env afterAll', async ({runInlineTest}) => { ]); }); +test('should ignore stdio when quiet', async ({runInlineTest}) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + module.exports = { quiet: true }; + `, + 'a.spec.js': ` + const { test } = pwt; + test('is a test', () => { + console.log('\\n%% stdout in a test'); + console.error('\\n%% stderr in a test'); + }); + ` + }, { reporter: 'list' }, { PWTEST_SKIP_TEST_OUTPUT: '' }); + expect(result.output).not.toContain('%%'); +});