test: make expectations for some reporter tests readable (#30470)

tests/playwright-test/reporter.spec.ts

@@ -14,27 +14,106 @@
  * limitations under the License.
  */
 
-import { test, expect, stripAnsi } from './playwright-test-fixtures';
+import { test, expect } from './playwright-test-fixtures';
-import fs from 'fs';
 
 const smallReporterJS = `
+const path = require('path');
+
+function formatLocation(location) {
+  if (!location)
+    return ' @ <no location>';
+  return ' @ ' + path.basename(location.file) + ':' + location.line;
+}
+
+function formatTitle(test) {
+  let titlePath = test.titlePath();
+  if (!titlePath[0])
+    [, ...titlePath] = titlePath;
+  return titlePath.join(' > ');
+}
+
+const asciiRegex = new RegExp('[\\\\u001B\\\\u009B][[\\\\]()#;?]*(?:(?:(?:[a-zA-Z\\\\d]*(?:;[-a-zA-Z\\\\d\\\\/#&.:=?%@~_]*)*)?\\\\u0007)|(?:(?:\\\\d{1,4}(?:;\\\\d{0,4})*)?[\\\\dA-PR-TZcf-ntqry=><~]))', 'g');
+function stripAnsi(str) {
+  return str.replace(asciiRegex, '');
+}
+
 class Reporter {
+  constructor(options) {
+    this.options = options;
+  }
+
+  trimError(message) {
+    if (this.options.skipErrorMessage)
+      return '<error message>';
+    const lines = message.split('\\n');
+    return lines[0].trimEnd();
+  }
+
+  printErrors(errors, prefix) {
+    for (const error of errors) {
+      const errorLocation = this.options.skipErrorLocation ? '' : formatLocation(error.location);
+      console.log((prefix || '  error: ') + this.trimError(error.message) + errorLocation);
+      if (this.options.printErrorSnippet && error.snippet) {
+        const lines = ['======', ...error.snippet.split('\\n'), '======'];
+        console.log(lines.map(line => stripAnsi('  ' + line).trimEnd()).join('\\n'));
+      }
+    }
+  }
+
+  printsToStdio() {
+    return true;
+  }
+
   onBegin(config, suite) {
-    console.log('\\n%%begin');
+    console.log(); // for nicer expectations
+    console.log('onBegin: ' + suite.allTests().length + ' tests total');
+    if (this.options.onBegin)
+      console.log('  options.onBegin=' + this.options.onBegin);
   }
-  onTestBegin(test) {}
+
-  onStdOut() {}
+  onTestBegin(test, result) {
-  onStdErr() {}
+    console.log('onTestBegin: ' + formatTitle(test) + '; retry #' + result.retry);
-  onTestEnd(test, result) {}
+  }
-  onTimeout() {}
+
+  onTestEnd(test, result) {
+    console.log('onTestEnd: ' + formatTitle(test) + '; retry #' + result.retry);
+    this.printErrors(result.errors);
+  }
+
+  onStepBegin(test, result, step) {
+  }
+
+  onStepEnd(test, result, step) {
+    if (this.options.printSteps) {
+      console.log('onStepEnd: ' + step.title);
+      if (step.error)
+        this.printErrors([step.error]);
+    }
+  }
+
+  // For easier debugging.
+  onStdOut(data) {
+    process.stdout.write(data.toString());
+  }
+
+  // For easier debugging.
+  onStdErr(data) {
+    process.stderr.write(data.toString());
+  }
+
   onError(error) {
-    console.log('\\n%%got error: ' + error.message);
+    this.printErrors([error], 'onError: ');
   }
-  onEnd() {
+
-    console.log('\\n%%end');
+  async onEnd() {
+    await new Promise(f => setTimeout(f, 500));
+    console.log('onEnd');
+    if (this.options.onEnd)
+      console.log('  options.onEnd=' + this.options.onEnd);
   }
-  onExit() {
+
-    console.log('\\n%%exit');
+  async onExit() {
+    console.log('onExit');
   }
 }
 module.exports = Reporter;
@@ -46,52 +125,11 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
     test('should work with custom reporter', async ({ runInlineTest }) => {
       const result = await runInlineTest({
-        'reporter.ts': `
+        'reporter.ts': smallReporterJS,
-          class Reporter {
-            constructor(options) {
-              this.options = options;
-            }
-            onBegin(config, suite) {
-              console.log('\\n%%reporter-begin-' + this.options.begin + '%%');
-              console.log('\\n%%version-' + config.version);
-            }
-            onTestBegin(test) {
-              const projectName = test.titlePath()[1];
-              console.log('\\n%%reporter-testbegin-' + test.title + '-' + projectName + '%%');
-              const suite = test.parent;
-              if (!suite.tests.includes(test))
-                console.log('\\n%%error-inconsistent-parent');
-              if (test.parent.project().name !== projectName)
-                console.log('\\n%%error-inconsistent-project-name');
-            }
-            onStdOut() {
-              console.log('\\n%%reporter-stdout%%');
-            }
-            onStdErr() {
-              console.log('\\n%%reporter-stderr%%');
-            }
-            onTestEnd(test, result) {
-              console.log('\\n%%reporter-testend-' + test.title + '-' + test.titlePath()[1] + '%%');
-              if (!result.startTime)
-                console.log('\\n%%error-no-start-time');
-            }
-            onTimeout() {
-              console.log('\\n%%reporter-timeout%%');
-            }
-            onError() {
-              console.log('\\n%%reporter-error%%');
-            }
-            async onEnd() {
-              await new Promise(f => setTimeout(f, 500));
-              console.log('\\n%%reporter-end-' + this.options.end + '%%');
-            }
-          }
-          export default Reporter;
-        `,
         'playwright.config.ts': `
           module.exports = {
             reporter: [
-              [ './reporter.ts', { begin: 'begin', end: 'end' } ]
+              [ './reporter.ts', { onBegin: 'begin-data', onEnd: 'end-data' } ]
             ],
             projects: [
               { name: 'foo', repeatEach: 2 },
@@ -113,23 +151,25 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { reporter: '', workers: 1 });
 
       expect(result.exitCode).toBe(0);
-      expect(result.outputLines).toEqual([
+      expect(result.output).toBe(`
-        'reporter-begin-begin%%',
+onBegin: 3 tests total
-        'version-' + require('../../packages/playwright/package.json').version,
+  options.onBegin=begin-data
-        'reporter-testbegin-is run-foo%%',
+onTestBegin: foo > a.test.ts > is run; retry #0
-        'reporter-stdout%%',
+log
-        'reporter-stderr%%',
+error
-        'reporter-testend-is run-foo%%',
+onTestEnd: foo > a.test.ts > is run; retry #0
-        'reporter-testbegin-is run-foo%%',
+onTestBegin: foo > a.test.ts > is run; retry #0
-        'reporter-stdout%%',
+log
-        'reporter-stderr%%',
+error
-        'reporter-testend-is run-foo%%',
+onTestEnd: foo > a.test.ts > is run; retry #0
-        'reporter-testbegin-is run-bar%%',
+onTestBegin: bar > a.test.ts > is run; retry #0
-        'reporter-stdout%%',
+log
-        'reporter-stderr%%',
+error
-        'reporter-testend-is run-bar%%',
+onTestEnd: bar > a.test.ts > is run; retry #0
-        'reporter-end-end%%',
+onEnd
-      ]);
+  options.onEnd=end-data
+onExit
+`);
     });
 
     test('should work without a file extension', async ({ runInlineTest }) => {
@@ -148,11 +188,13 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { reporter: '', workers: 1 });
 
       expect(result.exitCode).toBe(0);
-      expect(result.outputLines).toEqual([
+      expect(result.output).toBe(`
-        'begin',
+onBegin: 1 tests total
-        'end',
+onTestBegin:  > a.test.ts > pass; retry #0
-        'exit',
+onTestEnd:  > a.test.ts > pass; retry #0
-      ]);
+onEnd
+onExit
+`);
     });
 
     test('should report onEnd after global teardown', async ({ runInlineTest }) => {
@@ -178,12 +220,15 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { reporter: '', workers: 1 });
 
       expect(result.exitCode).toBe(0);
-      expect(result.outputLines).toEqual([
+      expect(result.output).toBe(`
-        'begin',
+onBegin: 1 tests total
-        'global teardown',
+onTestBegin:  > a.test.ts > pass; retry #0
-        'end',
+onTestEnd:  > a.test.ts > pass; retry #0
-        'exit',
+
-      ]);
+%%global teardown
+onEnd
+onExit
+`);
     });
 
     test('should load reporter from node_modules', async ({ runInlineTest }) => {
@@ -202,11 +247,13 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { reporter: '', workers: 1 });
 
       expect(result.exitCode).toBe(0);
-      expect(result.outputLines).toEqual([
+      expect(result.output).toBe(`
-        'begin',
+onBegin: 1 tests total
-        'end',
+onTestBegin:  > a.test.ts > pass; retry #0
-        'exit',
+onTestEnd:  > a.test.ts > pass; retry #0
-      ]);
+onEnd
+onExit
+`);
     });
 
     test('should not have internal error when steps are finished after timeout', async ({ runInlineTest }) => {
@@ -247,7 +294,12 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { 'reporter': '', 'forbid-only': true });
 
       expect(result.exitCode).toBe(1);
-      expect(result.output).toContain(`%%got error: Error: item focused with '.only' is not allowed due to the '--forbid-only' CLI flag: \"a.test.ts pass\"`);
+      expect(result.output).toBe(`
+onBegin: 0 tests total
+onError: Error: item focused with '.only' is not allowed due to the '--forbid-only' CLI flag: "a.test.ts pass" @ a.test.ts:3
+onEnd
+onExit
+`);
     });
 
     test('should report no-tests error to reporter', async ({ runInlineTest }) => {
@@ -261,7 +313,12 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { 'reporter': '' });
 
       expect(result.exitCode).toBe(1);
-      expect(result.output).toContain(`%%got error: Error: No tests found`);
+      expect(result.output).toBe(`
+onBegin: 0 tests total
+onError: Error: No tests found @ <no location>
+onEnd
+onExit
+`);
     });
 
     test('should report require error to reporter', async ({ runInlineTest }) => {
@@ -278,7 +335,13 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { 'reporter': '' });
 
       expect(result.exitCode).toBe(1);
-      expect(result.output).toContain(`%%got error: Error: Oh my!`);
+      expect(result.output).toBe(`
+onBegin: 0 tests total
+onError: Error: Oh my! @ a.spec.js:2
+onError: Error: No tests found @ <no location>
+onEnd
+onExit
+`);
     });
 
     test('should report global setup error to reporter', async ({ runInlineTest }) => {
@@ -302,7 +365,12 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       }, { 'reporter': '' });
 
       expect(result.exitCode).toBe(1);
-      expect(result.output).toContain(`%%got error: Error: Oh my!`);
+      expect(result.output).toBe(`
+onBegin: 0 tests total
+onError: Error: Oh my! @ globalSetup.ts:3
+onEnd
+onExit
+`);
     });
 
     test('should report correct tests/suites when using grep', async ({ runInlineTest }) => {
@@ -404,32 +472,9 @@ var import_test = __toModule(require("@playwright/test"));
     });
 
     test('test and step error should have code snippet', async ({ runInlineTest }) => {
-      const testErrorFile = test.info().outputPath('testError.txt');
-      const stepErrorFile = test.info().outputPath('stepError.txt');
       const result = await runInlineTest({
-        'reporter.ts': `
+        'reporter.ts': smallReporterJS,
-        import fs from 'fs';
+        'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printSteps: true, printErrorSnippet: true }]] };`,
-        class Reporter {
-          onStepEnd(test, result, step) {
-            console.log('\\n%%onStepEnd: ' + step.error?.snippet?.length);
-            if (step.error?.snippet)
-              fs.writeFileSync('${stepErrorFile.replace(/\\/g, '\\\\')}', step.error?.snippet);
-          }
-          onTestEnd(test, result) {
-            console.log('\\n%%onTestEnd: ' + result.error?.snippet?.length);
-            if (result.error)
-              fs.writeFileSync('${testErrorFile.replace(/\\/g, '\\\\')}', result.error?.snippet);
-          }
-          onError(error) {
-            console.log('\\n%%onError: ' + error.snippet?.length);
-          }
-        }
-        module.exports = Reporter;`,
-        'playwright.config.ts': `
-          module.exports = {
-            reporter: './reporter',
-          };
-        `,
         'a.spec.js': `
           const { test, expect } = require('@playwright/test');
           test('test', async () => {
@@ -439,56 +484,76 @@ var import_test = __toModule(require("@playwright/test"));
           });
         `,
       }, { 'reporter': '', 'workers': 1 });
-
+      expect(result.output).toBe(`
-      expect(result.output).toContain('onTestEnd: 550');
+onBegin: 1 tests total
-      expect(result.output).toContain('onStepEnd: 550');
+onTestBegin:  > a.spec.js > test; retry #0
-      expect(stripAnsi(fs.readFileSync(testErrorFile, 'utf8'))).toBe(`  3 |           test('test', async () => {
+onStepEnd: Before Hooks
+onStepEnd: expect.toBe
+  error: Error: expect(received).toBe(expected) // Object.is equality @ a.spec.js:5
+  ======
+    3 |           test('test', async () => {
     4 |             await test.step('step', async () => {
-> 5 |               expect(1).toBe(2);
+  > 5 |               expect(1).toBe(2);
       |                         ^
     6 |             });
     7 |           });
-  8 |         `);
+    8 |
-      expect(stripAnsi(fs.readFileSync(stepErrorFile, 'utf8'))).toBe(`  3 |           test('test', async () => {
+  ======
+onStepEnd: step
+  error: Error: expect(received).toBe(expected) // Object.is equality @ a.spec.js:5
+  ======
+    3 |           test('test', async () => {
     4 |             await test.step('step', async () => {
-> 5 |               expect(1).toBe(2);
+  > 5 |               expect(1).toBe(2);
       |                         ^
     6 |             });
     7 |           });
-  8 |         `);
+    8 |
+  ======
+onStepEnd: After Hooks
+onStepEnd: Worker Cleanup
+onTestEnd:  > a.spec.js > test; retry #0
+  error: Error: expect(received).toBe(expected) // Object.is equality @ a.spec.js:5
+  ======
+    3 |           test('test', async () => {
+    4 |             await test.step('step', async () => {
+  > 5 |               expect(1).toBe(2);
+      |                         ^
+    6 |             });
+    7 |           });
+    8 |
+  ======
+onEnd
+onExit
+`);
     });
 
     test('onError should have code snippet', async ({ runInlineTest }) => {
-      const errorFile = test.info().outputPath('error.txt');
       const result = await runInlineTest({
-        'reporter.ts': `
+        'reporter.ts': smallReporterJS,
-        import fs from 'fs';
+        'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printSteps: true, printErrorSnippet: true }]] };`,
-        class Reporter {
-          onError(error) {
-            console.log('\\n%%onError: ' + error.snippet?.length);
-            fs.writeFileSync('${errorFile.replace(/\\/g, '\\\\')}', error.snippet);
-          }
-        }
-        module.exports = Reporter;`,
-        'playwright.config.ts': `
-          module.exports = {
-            reporter: './reporter',
-          };
-        `,
         'a.spec.js': `
           const { test, expect } = require('@playwright/test');
           throw new Error('test');
         `,
       }, { 'reporter': '', 'workers': 1 });
 
-      expect(result.output).toContain('onError: 412');
+      expect(result.output).toBe(`
-      expect(stripAnsi(fs.readFileSync(errorFile, 'utf8'))).toBe(`   at a.spec.js:3
+onBegin: 0 tests total
+onError: Error: test @ a.spec.js:3
+  ======
+     at a.spec.js:3
 
     1 |
     2 |           const { test, expect } = require('@playwright/test');
-> 3 |           throw new Error('test');
+  > 3 |           throw new Error('test');
       |                 ^
-  4 |         `);
+    4 |
+  ======
+onError: Error: No tests found @ <no location>
+onEnd
+onExit
+`);
     });
   });
 }