chore: introduce Reporter.onExit (#22176)

Fixes https://github.com/microsoft/playwright/issues/22173

docs/src/test-reporter-api/class-reporter.md

@@ -81,6 +81,7 @@ Here is a typical order of reporter calls:
 * [`method: Reporter.onStepBegin`] and [`method: Reporter.onStepEnd`] are called for each executed step inside the test. When steps are executed, test run has not finished yet.
 * [`method: Reporter.onTestEnd`] is called when test run has finished. By this time, [TestResult] is complete and you can use [`property: TestResult.status`], [`property: TestResult.error`] and more.
 * [`method: Reporter.onEnd`] is called once after all tests that should run had finished.
+* [`method: Reporter.onExit`] is called immediately before the test runner exits.
 
 Additionally, [`method: Reporter.onStdOut`] and [`method: Reporter.onStdErr`] are called when standard output is produced in the worker process, possibly during a test execution,
 and [`method: Reporter.onError`] is called when something went wrong outside of the test execution.
@@ -131,6 +132,12 @@ Called on some global error, for example unhandled exception in the worker proce
 
 The error.
 
+## optional async method: Reporter.onExit
+* since: v1.33
+
+Called immediately before test runner exists. At this point all the reporters
+have recived the [`method: Reporter.onEnd`] signal, so all the reports should
+be build. You can run the code that uploads the reports in this hook.
 
 ## optional method: Reporter.onStdErr
 * since: v1.10

packages/playwright-test/src/plugins/index.ts

@@ -14,12 +14,13 @@
  * limitations under the License.
  */
 
-import type { Suite, Reporter } from '../../types/testReporter';
+import type { Suite } from '../../types/testReporter';
 import type { FullConfig } from '../common/types';
+import type { Multiplexer } from '../reporters/multiplexer';
 
 export interface TestRunnerPlugin {
   name: string;
-  setup?(config: FullConfig, configDir: string, reporter: Reporter): Promise<void>;
+  setup?(config: FullConfig, configDir: string, reporter: Multiplexer): Promise<void>;
   begin?(suite: Suite): Promise<void>;
   end?(): Promise<void>;
   teardown?(): Promise<void>;

packages/playwright-test/src/plugins/webServerPlugin.ts

@@ -19,10 +19,11 @@ import net from 'net';
 import { debug } from 'playwright-core/lib/utilsBundle';
 import { raceAgainstTimeout, launchProcess, httpRequest } from 'playwright-core/lib/utils';
 
-import type { FullConfig, Reporter } from '../../types/testReporter';
+import type { FullConfig } from '../../types/testReporter';
 import type { TestRunnerPlugin } from '.';
 import type { FullConfigInternal } from '../common/types';
 import { envWithoutExperimentalLoaderOptions } from '../util';
+import type { Multiplexer } from '../reporters/multiplexer';
 
 
 export type WebServerPluginOptions = {
@@ -47,7 +48,7 @@ export class WebServerPlugin implements TestRunnerPlugin {
   private _processExitedPromise!: Promise<any>;
   private _options: WebServerPluginOptions;
   private _checkPortOnly: boolean;
-  private _reporter?: Reporter;
+  private _reporter?: Multiplexer;
   name = 'playwright:webserver';
 
   constructor(options: WebServerPluginOptions, checkPortOnly: boolean) {
@@ -55,7 +56,7 @@ export class WebServerPlugin implements TestRunnerPlugin {
     this._checkPortOnly = checkPortOnly;
   }
 
-  public async setup(config: FullConfig, configDir: string, reporter: Reporter) {
+  public async setup(config: FullConfig, configDir: string, reporter: Multiplexer) {
     this._reporter = reporter;
     this._isAvailable = getIsAvailableFunction(this._options.url, this._checkPortOnly, !!this._options.ignoreHTTPSErrors, this._reporter.onStdErr?.bind(this._reporter));
     this._options.cwd = this._options.cwd ? path.resolve(configDir, this._options.cwd) : configDir;
@@ -146,7 +147,7 @@ async function isPortUsed(port: number): Promise<boolean> {
   return await innerIsPortUsed('127.0.0.1') || await innerIsPortUsed('::1');
 }
 
-async function isURLAvailable(url: URL, ignoreHTTPSErrors: boolean, onStdErr: Reporter['onStdErr']) {
+async function isURLAvailable(url: URL, ignoreHTTPSErrors: boolean, onStdErr: Multiplexer['onStdErr']) {
   let statusCode = await httpStatusCode(url, ignoreHTTPSErrors, onStdErr);
   if (statusCode === 404 && url.pathname === '/') {
     const indexUrl = new URL(url);
@@ -156,7 +157,7 @@ async function isURLAvailable(url: URL, ignoreHTTPSErrors: boolean, onStdErr: Re
   return statusCode >= 200 && statusCode < 404;
 }
 
-async function httpStatusCode(url: URL, ignoreHTTPSErrors: boolean, onStdErr: Reporter['onStdErr']): Promise<number> {
+async function httpStatusCode(url: URL, ignoreHTTPSErrors: boolean, onStdErr: Multiplexer['onStdErr']): Promise<number> {
   return new Promise(resolve => {
     debugWebServer(`HTTP GET: ${url}`);
     httpRequest({
@@ -189,7 +190,7 @@ async function waitFor(waitFn: () => Promise<boolean>, cancellationToken: { canc
   }
 }
 
-function getIsAvailableFunction(url: string, checkPortOnly: boolean, ignoreHTTPSErrors: boolean, onStdErr: Reporter['onStdErr']) {
+function getIsAvailableFunction(url: string, checkPortOnly: boolean, ignoreHTTPSErrors: boolean, onStdErr: Multiplexer['onStdErr']) {
   const urlObject = new URL(url);
   if (!checkPortOnly)
     return () => isURLAvailable(urlObject, ignoreHTTPSErrors, onStdErr);

packages/playwright-test/src/reporters/html.ts

@@ -112,7 +112,7 @@ class HtmlReporter implements Reporter {
     this._buildResult = await builder.build({ ...this.config.metadata, duration }, reports);
   }
 
-  async _onExit() {
+  async onExit() {
     if (process.env.CI || !this._buildResult)
       return;
 

packages/playwright-test/src/reporters/multiplexer.ts

@@ -24,7 +24,7 @@ type StdIOChunk = {
   result?: TestResult;
 };
 
-export class Multiplexer implements Reporter {
+export class Multiplexer {
   private _reporters: Reporter[];
   private _deferred: { error?: TestError, stdout?: StdIOChunk, stderr?: StdIOChunk }[] | null = [];
   private _config!: FullConfig;
@@ -99,7 +99,7 @@ export class Multiplexer implements Reporter {
       await Promise.resolve().then(() => reporter.onEnd?.(result)).catch(e => console.error('Error in reporter', e));
 
     for (const reporter of this._reporters)
-      await Promise.resolve().then(() => (reporter as any)._onExit?.()).catch(e => console.error('Error in reporter', e));
+      await Promise.resolve().then(() => reporter.onExit?.()).catch(e => console.error('Error in reporter', e));
   }
 
   onError(error: TestError) {

packages/playwright-test/src/runner/dispatcher.ts

@@ -16,7 +16,7 @@
 
 import type { TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, RunPayload, SerializedConfig } from '../common/ipc';
 import { serializeConfig } from '../common/ipc';
-import type { TestResult, Reporter, TestStep, TestError } from '../../types/testReporter';
+import type { TestResult, TestStep, TestError } from '../../types/testReporter';
 import type { Suite } from '../common/test';
 import type { ProcessExitData } from './processHost';
 import type { TestCase } from '../common/test';
@@ -24,6 +24,7 @@ import { ManualPromise } from 'playwright-core/lib/utils';
 import { WorkerHost } from './workerHost';
 import type { TestGroup } from './testGroups';
 import type { FullConfigInternal } from '../common/types';
+import type { Multiplexer } from '../reporters/multiplexer';
 
 type TestResultData = {
   result: TestResult;
@@ -45,14 +46,14 @@ export class Dispatcher {
 
   private _testById = new Map<string, TestData>();
   private _config: FullConfigInternal;
-  private _reporter: Reporter;
+  private _reporter: Multiplexer;
   private _hasWorkerErrors = false;
   private _failureCount = 0;
 
   private _extraEnvByProjectId: EnvByProjectId = new Map();
   private _producedEnvByProjectId: EnvByProjectId = new Map();
 
-  constructor(config: FullConfigInternal, reporter: Reporter) {
+  constructor(config: FullConfigInternal, reporter: Multiplexer) {
     this._config = config;
     this._reporter = reporter;
   }

packages/playwright-test/src/runner/taskRunner.ts

@@ -16,22 +16,23 @@
 
 import { debug } from 'playwright-core/lib/utilsBundle';
 import { ManualPromise, monotonicTime } from 'playwright-core/lib/utils';
-import type { FullResult, Reporter, TestError } from '../../reporter';
+import type { FullResult, TestError } from '../../reporter';
 import { SigIntWatcher } from './sigIntWatcher';
 import { serializeError } from '../util';
+import type { Multiplexer } from '../reporters/multiplexer';
 
 type TaskTeardown = () => Promise<any> | undefined;
 export type Task<Context> = (context: Context, errors: TestError[]) => Promise<TaskTeardown | void> | undefined;
 
 export class TaskRunner<Context> {
   private _tasks: { name: string, task: Task<Context> }[] = [];
-  private _reporter: Reporter;
+  private _reporter: Multiplexer;
   private _hasErrors = false;
   private _interrupted = false;
   private _isTearDown = false;
   private _globalTimeoutForError: number;
 
-  constructor(reporter: Reporter, globalTimeoutForError: number) {
+  constructor(reporter: Multiplexer, globalTimeoutForError: number) {
     this._reporter = reporter;
     this._globalTimeoutForError = globalTimeoutForError;
   }

packages/playwright-test/types/testReporter.d.ts

@@ -372,6 +372,8 @@ export interface FullResult {
  *   [testResult.error](https://playwright.dev/docs/api/class-testresult#test-result-error) and more.
  * - [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) is called once after
  *   all tests that should run had finished.
+ * - [reporter.onExit()](https://playwright.dev/docs/api/class-reporter#reporter-on-exit) is called immediately
+ *   before the test runner exits.
  *
  * Additionally,
  * [reporter.onStdOut(chunk, test, result)](https://playwright.dev/docs/api/class-reporter#reporter-on-std-out) and
@@ -410,6 +412,13 @@ export interface Reporter {
    */
   onError?(error: TestError): void;
 
+  /**
+   * Called immediately before test runner exists. At this point all the reporters have recived the
+   * [reporter.onEnd(result)](https://playwright.dev/docs/api/class-reporter#reporter-on-end) signal, so all the reports
+   * should be build. You can run the code that uploads the reports in this hook.
+   */
+  onExit?(): Promise<void>;
+
   /**
    * Called when something has been written to the standard error in the worker process.
    * @param chunk Output chunk.

tests/playwright-test/reporter.spec.ts

@@ -33,6 +33,9 @@ class Reporter {
   onEnd() {
     console.log('\\n%%end');
   }
+  onExit() {
+    console.log('\\n%%exit');
+  }
 }
 module.exports = Reporter;
 `;
@@ -176,6 +179,7 @@ test('should work without a file extension', async ({ runInlineTest }) => {
   expect(result.outputLines).toEqual([
     'begin',
     'end',
+    'exit',
   ]);
 });
 
@@ -205,6 +209,7 @@ test('should report onEnd after global teardown', async ({ runInlineTest }) => {
     'begin',
     'global teardown',
     'end',
+    'exit',
   ]);
 });
 
@@ -227,6 +232,7 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => {
   expect(result.outputLines).toEqual([
     'begin',
     'end',
+    'exit',
   ]);
 });