diff --git a/docs/src/test-reporter-api/class-reporter.md b/docs/src/test-reporter-api/class-reporter.md index d9f4be78c7..48a5654208 100644 --- a/docs/src/test-reporter-api/class-reporter.md +++ b/docs/src/test-reporter-api/class-reporter.md @@ -106,8 +106,11 @@ The root suite that contains all projects, files and test cases. ## optional async method: Reporter.onEnd * since: v1.10 +- `result` ?<[Object]> + - `status` ?<[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">> Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] and Playwright Test will await it. +Reporter is allowed to override the status and hence affect the exit code of the test runner. ### param: Reporter.onEnd.result * since: v1.10 diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index a7e818e6ea..6de7a3f76a 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -306,8 +306,8 @@ export class TeleReporterReceiver { } } - private _onEnd(result: JsonFullResult): Promise | void { - return this._reporter.onEnd?.({ + private async _onEnd(result: JsonFullResult): Promise { + await this._reporter.onEnd?.({ status: result.status, startTime: new Date(result.startTime), duration: result.duration, diff --git a/packages/playwright/src/reporters/internalReporter.ts b/packages/playwright/src/reporters/internalReporter.ts index 4075e92c48..fefb089b0b 100644 --- a/packages/playwright/src/reporters/internalReporter.ts +++ b/packages/playwright/src/reporters/internalReporter.ts @@ -72,7 +72,7 @@ export class InternalReporter { // onBegin was not reported, emit it. this.onBegin(new Suite('', 'root')); } - await this._reporter.onEnd({ + return await this._reporter.onEnd({ ...result, startTime: this._startTime!, duration: monotonicTime() - this._monotonicStartTime!, diff --git a/packages/playwright/src/reporters/multiplexer.ts b/packages/playwright/src/reporters/multiplexer.ts index eae794470a..bfd91f3350 100644 --- a/packages/playwright/src/reporters/multiplexer.ts +++ b/packages/playwright/src/reporters/multiplexer.ts @@ -60,8 +60,12 @@ export class Multiplexer implements ReporterV2 { } async onEnd(result: FullResult) { - for (const reporter of this._reporters) - await wrapAsync(() => reporter.onEnd(result)); + for (const reporter of this._reporters) { + const outResult = await wrapAsync(() => reporter.onEnd(result)); + if (outResult?.status) + result.status = outResult.status; + } + return result; } async onExit() { @@ -93,9 +97,9 @@ export class Multiplexer implements ReporterV2 { } } -async function wrapAsync(callback: () => void | Promise) { +async function wrapAsync(callback: () => T | Promise) { try { - await callback(); + return await callback(); } catch (e) { console.error('Error in reporter', e); } diff --git a/packages/playwright/src/reporters/reporterV2.ts b/packages/playwright/src/reporters/reporterV2.ts index 914534441d..3d640f97c3 100644 --- a/packages/playwright/src/reporters/reporterV2.ts +++ b/packages/playwright/src/reporters/reporterV2.ts @@ -23,7 +23,7 @@ export interface ReporterV2 { onStdOut(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; onStdErr(chunk: string | Buffer, test?: TestCase, result?: TestResult): void; onTestEnd(test: TestCase, result: TestResult): void; - onEnd(result: FullResult): void | Promise; + onEnd(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; onExit(): void | Promise; onError(error: TestError): void; onStepBegin(test: TestCase, result: TestResult, step: TestStep): void; @@ -104,7 +104,7 @@ class ReporterV2Wrapper implements ReporterV2 { } async onEnd(result: FullResult) { - await this._reporter.onEnd?.(result); + return await this._reporter.onEnd?.(result); } async onExit() { diff --git a/packages/playwright/src/runner/runner.ts b/packages/playwright/src/runner/runner.ts index 55fb66f2d6..3592afb701 100644 --- a/packages/playwright/src/runner/runner.ts +++ b/packages/playwright/src/runner/runner.ts @@ -90,7 +90,10 @@ export class Runner { let status: FullResult['status'] = testRun.failureTracker.result(); if (status === 'passed' && taskStatus !== 'passed') status = taskStatus; - await reporter.onEnd({ status }); + const modifiedResult = await reporter.onEnd({ status }); + if (modifiedResult && modifiedResult.status) + status = modifiedResult.status; + await reporter.onExit(); // Calling process.exit() might truncate large stdout/stderr output. diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 2ace294ba7..42211ef9b5 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -410,7 +410,8 @@ export interface Reporter { onBegin?(config: FullConfig, suite: Suite): void; /** * Called after all tests have been run, or testing has been interrupted. Note that this method may return a [Promise] - * and Playwright Test will await it. + * and Playwright Test will await it. Reporter is allowed to override the status and hence affect the exit code of the + * test runner. * @param result Result of the full test run, `status` can be one of: * - `'passed'` - Everything went as expected. * - `'failed'` - Any test has failed. @@ -419,7 +420,7 @@ export interface Reporter { * been reached. * - `'interrupted'` - Interrupted by the user. */ - onEnd?(result: FullResult): void | Promise; + onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; /** * Called on some global error, for example unhandled exception in the worker process. * @param error The error. diff --git a/tests/playwright-test/reporter-onend.spec.ts b/tests/playwright-test/reporter-onend.spec.ts new file mode 100644 index 0000000000..38665ea2e6 --- /dev/null +++ b/tests/playwright-test/reporter-onend.spec.ts @@ -0,0 +1,40 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +const reporter = ` +class Reporter { + async onEnd() { + return { status: 'passed' }; + } +} +module.exports = Reporter; +`; + +test('should override exit code', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'reporter.ts': reporter, + 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('fail', async ({}) => { + expect(1 + 1).toBe(3); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index ead17e201c..61a20f42fa 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -55,7 +55,7 @@ export interface FullResult { export interface Reporter { onBegin?(config: FullConfig, suite: Suite): void; - onEnd?(result: FullResult): void | Promise; + onEnd?(result: FullResult): Promise<{ status?: FullResult['status'] } | undefined | void> | void; } export interface JSONReport {