feat(test runner): improve reporters api (#7370)

- onEnd may return a Promise
- onEnd now takes a result for the full run
- onTimeout is replaced with onEnd(result)

src/test/dispatcher.ts

@@ -171,7 +171,7 @@ export class Dispatcher {
       if (params.fatalError) {
         for (const { testId } of remaining) {
           const { test, result } = this._testById.get(testId)!;
-          this._reporter.onTestBegin?.(test);
+          this._reporter.onTestBegin(test);
           result.error = params.fatalError;
           this._reportTestEnd(test, result, 'failed');
           failedTestIds.add(testId);

src/test/reporter.ts

@@ -60,13 +60,15 @@ export interface TestResult {
   stdout: (string | Buffer)[];
   stderr: (string | Buffer)[];
 }
+export type FullResult = {
+  status: 'passed' | 'failed' | 'timedout' | 'interrupted';
+};
 export interface Reporter {
   onBegin(config: FullConfig, suite: Suite): void;
   onTestBegin(test: Test): void;
   onStdOut(chunk: string | Buffer, test?: Test): void;
   onStdErr(chunk: string | Buffer, test?: Test): void;
   onTestEnd(test: Test, result: TestResult): void;
-  onTimeout(timeout: number): void;
   onError(error: TestError): void;
-  onEnd(): void;
+  onEnd(result: FullResult): void | Promise<void>;
 }

src/test/reporters/base.ts

@@ -21,7 +21,7 @@ import fs from 'fs';
 import milliseconds from 'ms';
 import path from 'path';
 import StackUtils from 'stack-utils';
-import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter } from '../reporter';
+import { FullConfig, TestStatus, Test, Spec, Suite, TestResult, TestError, Reporter, FullResult } from '../reporter';
 
 const stackUtils = new StackUtils();
 
@@ -29,7 +29,7 @@ export class BaseReporter implements Reporter  {
   duration = 0;
   config!: FullConfig;
   suite!: Suite;
-  timeout: number = 0;
+  result!: FullResult;
   fileDurations = new Map<string, number>();
   monotonicStartTime: number = 0;
 
@@ -66,12 +66,9 @@ export class BaseReporter implements Reporter  {
     console.log(formatError(error));
   }
 
-  onTimeout(timeout: number) {
-    this.timeout = timeout;
-  }
-
-  onEnd() {
+  async onEnd(result: FullResult) {
     this.duration = monotonicTime() - this.monotonicStartTime;
+    this.result = result;
   }
 
   private _printSlowTests() {
@@ -123,8 +120,8 @@ export class BaseReporter implements Reporter  {
       console.log(colors.yellow(`  ${skipped} skipped`));
     if (expected)
       console.log(colors.green(`  ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
-    if (this.timeout)
-      console.log(colors.red(`  Timed out waiting ${this.timeout / 1000}s for the entire test run`));
+    if (this.result.status === 'timedout')
+      console.log(colors.red(`  Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`));
   }
 
   private _printTestHeaders(tests: Test[]) {

src/test/reporters/dot.ts

@@ -16,7 +16,7 @@
 
 import colors from 'colors/safe';
 import { BaseReporter } from './base';
-import { Test, TestResult } from '../reporter';
+import { FullResult, Test, TestResult } from '../reporter';
 
 class DotReporter extends BaseReporter {
   private _counter = 0;
@@ -42,13 +42,8 @@ class DotReporter extends BaseReporter {
     }
   }
 
-  onTimeout(timeout: number) {
-    super.onTimeout(timeout);
-    this.onEnd();
-  }
-
-  onEnd() {
-    super.onEnd();
+  async onEnd(result: FullResult) {
+    await super.onEnd(result);
     process.stdout.write('\n');
     this.epilogue(true);
   }

src/test/reporters/empty.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { FullConfig, TestResult, Test, Suite, TestError, Reporter } from '../reporter';
+import { FullConfig, TestResult, Test, Suite, TestError, Reporter, FullResult } from '../reporter';
 
 class EmptyReporter implements Reporter {
   onBegin(config: FullConfig, suite: Suite) {}
@@ -22,9 +22,8 @@ class EmptyReporter implements Reporter {
   onStdOut(chunk: string | Buffer, test?: Test) {}
   onStdErr(chunk: string | Buffer, test?: Test) {}
   onTestEnd(test: Test, result: TestResult) {}
-  onTimeout(timeout: number) {}
   onError(error: TestError) {}
-  onEnd() {}
+  async onEnd(result: FullResult) {}
 }
 
 export default EmptyReporter;

src/test/reporters/json.ts

@@ -17,7 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 import EmptyReporter from './empty';
-import { FullConfig, Test, Suite, Spec, TestResult, TestError } from '../reporter';
+import { FullConfig, Test, Suite, Spec, TestResult, TestError, FullResult } from '../reporter';
 
 interface SerializedSuite {
   title: string;
@@ -50,15 +50,11 @@ class JSONReporter extends EmptyReporter {
     this.suite = suite;
   }
 
-  onTimeout() {
-    this.onEnd();
-  }
-
   onError(error: TestError): void {
     this._errors.push(error);
   }
 
-  onEnd() {
+  async onEnd(result: FullResult) {
     outputReport(this._serializeReport(), this._outputFile);
   }
 

src/test/reporters/junit.ts

@@ -17,7 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 import EmptyReporter from './empty';
-import { FullConfig, Suite, Test } from '../reporter';
+import { FullConfig, FullResult, Suite, Test } from '../reporter';
 import { monotonicTime } from '../util';
 import { formatFailure, formatTestTitle, stripAscii } from './base';
 
@@ -45,7 +45,7 @@ class JUnitReporter extends EmptyReporter {
     this.startTime = monotonicTime();
   }
 
-  onEnd() {
+  async onEnd(result: FullResult) {
     const duration = monotonicTime() - this.startTime;
     const children: XMLEntry[] = [];
     for (const suite of this.suite.suites)

src/test/reporters/line.ts

@@ -16,7 +16,7 @@
 
 import colors from 'colors/safe';
 import { BaseReporter, formatFailure, formatTestTitle } from './base';
-import { FullConfig, Test, Suite, TestResult } from '../reporter';
+import { FullConfig, Test, Suite, TestResult, FullResult } from '../reporter';
 
 class LineReporter extends BaseReporter {
   private _total = 0;
@@ -64,9 +64,9 @@ class LineReporter extends BaseReporter {
     }
   }
 
-  onEnd() {
+  async onEnd(result: FullResult) {
     process.stdout.write(`\u001B[1A\u001B[2K`);
-    super.onEnd();
+    await super.onEnd(result);
     this.epilogue(false);
   }
 }

src/test/reporters/list.ts

@@ -19,7 +19,7 @@ import colors from 'colors/safe';
 // @ts-ignore
 import milliseconds from 'ms';
 import { BaseReporter, formatTestTitle } from './base';
-import { FullConfig, Suite, Test, TestResult } from '../reporter';
+import { FullConfig, FullResult, Suite, Test, TestResult } from '../reporter';
 
 // Allow it in the Visual Studio Code Terminal and the new Windows Terminal
 const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION;
@@ -107,8 +107,8 @@ class ListReporter extends BaseReporter {
     }
   }
 
-  onEnd() {
-    super.onEnd();
+  async onEnd(result: FullResult) {
+    await super.onEnd(result);
     process.stdout.write('\n');
     this.epilogue(true);
   }

src/test/reporters/multiplexer.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { FullConfig, Suite, Test, TestError, TestResult, Reporter } from '../reporter';
+import { FullConfig, Suite, Test, TestError, TestResult, Reporter, FullResult } from '../reporter';
 
 export class Multiplexer implements Reporter {
   private _reporters: Reporter[];
@@ -48,14 +48,9 @@ export class Multiplexer implements Reporter {
       reporter.onTestEnd(test, result);
   }
 
-  onTimeout(timeout: number) {
+  async onEnd(result: FullResult) {
     for (const reporter of this._reporters)
-      reporter.onTimeout(timeout);
-  }
-
-  onEnd() {
-    for (const reporter of this._reporters)
-      reporter.onEnd();
+      await reporter.onEnd(result);
   }
 
   onError(error: TestError) {

src/test/runner.ts

@@ -99,7 +99,7 @@ export class Runner {
     if (timedOut) {
       if (!this._didBegin)
         this._reporter.onBegin(config, new Suite(''));
-      this._reporter.onTimeout(config.globalTimeout);
+      await this._reporter.onEnd({ status: 'timedout' });
       await this._flushOutput();
       return 'failed';
     }
@@ -251,11 +251,15 @@ export class Runner {
         await dispatcher.stop();
         hasWorkerErrors = dispatcher.hasWorkerErrors();
       }
-      this._reporter.onEnd();
 
-      if (sigint)
+      if (sigint) {
+        await this._reporter.onEnd({ status: 'interrupted' });
         return { status: 'sigint' };
-      return { status: hasWorkerErrors || rootSuite.findSpec(spec => !spec.ok()) ? 'failed' : 'passed' };
+      }
+
+      const failed = hasWorkerErrors || rootSuite.findSpec(spec => !spec.ok());
+      await this._reporter.onEnd({ status: failed ? 'failed' : 'passed' });
+      return { status: failed ? 'failed' : 'passed' };
     } finally {
       if (globalSetupResult && typeof globalSetupResult === 'function')
         await globalSetupResult(this._loader.fullConfig());

tests/playwright-test/config.spec.ts

@@ -350,7 +350,8 @@ test('should work with custom reporter', async ({ runInlineTest }) => {
         onError() {
           console.log('\\n%%reporter-error%%');
         }
-        onEnd() {
+        async onEnd() {
+          await new Promise(f => setTimeout(f, 500));
           console.log('\\n%%reporter-end%%' + this.options.end);
         }
       }