feat(test runner): show failure details for flaky test runs (#8332)

Currently, we just say "foo.spec.ts > my test" is flaky, but do not
show how exactly the failed run went.

src/test/reporters/base.ts

@@ -110,9 +110,10 @@ export class BaseReporter implements Reporter  {
       }
     });
 
-    if (full && (unexpected.length || skippedWithError.length)) {
+    const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError];
+    if (full && failuresToPrint.length) {
       console.log('');
-      this._printFailures([...unexpected, ...skippedWithError]);
+      this._printFailures(failuresToPrint);
     }
 
     this._printSlowTests();
@@ -147,7 +148,7 @@ export class BaseReporter implements Reporter  {
   }
 
   willRetry(test: TestCase, result: TestResult): boolean {
-    return result.status !== 'passed' && result.status !== test.expectedStatus && test.results.length <= test.retries;
+    return test.outcome() === 'unexpected' && result.status !== 'passed' && test.results.length <= test.retries;
   }
 }
 

src/test/reporters/line.ts

@@ -59,7 +59,7 @@ class LineReporter extends BaseReporter {
     const width = process.stdout.columns! - 1;
     const title = `[${++this._current}/${this._total}] ${formatTestTitle(this.config, test)}`.substring(0, width);
     process.stdout.write(`\u001B[1A\u001B[2K${title}\n`);
-    if (!this.willRetry(test, result) && !test.ok()) {
+    if (!this.willRetry(test, result) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) {
       process.stdout.write(`\u001B[1A\u001B[2K`);
       console.log(formatFailure(this.config, test, ++this._failures));
       console.log();

tests/playwright-test/base-reporter.spec.ts

@@ -181,3 +181,32 @@ test('should print stdio for failures', async ({ runInlineTest }) => {
     'my log 2\n',
   ].join(''));
 });
+
+test('should print flaky failures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('foobar', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `
+  }, { retries: '1', reporter: 'list' });
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(1);
+  expect(stripAscii(result.output)).toContain('expect(testInfo.retry).toBe(1)');
+});
+
+test('should print flaky timeouts', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('foobar', async ({}, testInfo) => {
+        if (!testInfo.retry)
+          await new Promise(f => setTimeout(f, 2000));
+      });
+    `
+  }, { retries: '1', reporter: 'list', timeout: '1000' });
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(1);
+  expect(stripAscii(result.output)).toContain('Timeout of 1000ms exceeded.');
+});

tests/playwright-test/dot-reporter.spec.ts

@@ -71,7 +71,7 @@ test('render flaky', async ({ runInlineTest }) => {
   expect(text).toContain('×××±');
   expect(result.output).toContain(colors.yellow('±'));
   expect(text).toContain('1 flaky');
-  expect(text).not.toContain('Retry #1');
+  expect(text).toContain('Retry #1');
   expect(result.exitCode).toBe(0);
 });
 

tests/playwright-test/line-reporter.spec.ts

@@ -48,3 +48,17 @@ test('render flaky', async ({ runInlineTest }) => {
   expect(text).toContain('1 flaky');
   expect(result.exitCode).toBe(0);
 });
+
+test('should print flaky failures', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('foobar', async ({}, testInfo) => {
+        expect(testInfo.retry).toBe(1);
+      });
+    `
+  }, { retries: '1', reporter: 'line' });
+  expect(result.exitCode).toBe(0);
+  expect(result.flaky).toBe(1);
+  expect(stripAscii(result.output)).toContain('expect(testInfo.retry).toBe(1)');
+});

tests/playwright-test/stdio.spec.ts

@@ -61,3 +61,18 @@ test('should get stdio from env afterAll', async ({runInlineTest}) => {
   ]);
 });
 
+test('should ignore stdio when quiet', async ({runInlineTest}) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { quiet: true };
+    `,
+    'a.spec.js': `
+      const { test } = pwt;
+      test('is a test', () => {
+        console.log('\\n%% stdout in a test');
+        console.error('\\n%% stderr in a test');
+      });
+    `
+  }, { reporter: 'list' }, { PWTEST_SKIP_TEST_OUTPUT: '' });
+  expect(result.output).not.toContain('%%');
+});