diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 79946bf3db..c9a9727dc8 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -162,7 +162,7 @@ Errors thrown during test execution, if any. ## property: TestInfo.expectedStatus * since: v1.10 -- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> +- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">> Expected status for the currently running test. This is usually `'passed'`, except for a few cases: * `'skipped'` for skipped tests, e.g. with [`method: Test.skip#2`]; @@ -461,7 +461,7 @@ Suffix used to differentiate snapshots between multiple test configurations. For ## property: TestInfo.status * since: v1.10 -- type: ?<[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> +- type: ?<[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">> Actual status for the currently running test. Available after the test has finished in [`method: Test.afterEach`] hook and fixtures. diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index dbcb3c729d..de795d2028 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -18,7 +18,7 @@ Learn more about [test annotations](../test-annotations.md). ## property: TestCase.expectedStatus * since: v1.10 -- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> +- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">> Expected test status. * Tests marked as [`method: Test.skip#1`] or [`method: Test.fixme#1`] are expected to be `'skipped'`. diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index 0d5fbbe23c..f540c5e697 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -49,7 +49,7 @@ Start time of this particular test run. ## property: TestResult.status * since: v1.10 -- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">> +- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped"|"interrupted">> The status of this test result. See also [`property: TestCase.expectedStatus`]. diff --git a/packages/html-reporter/src/statusIcon.tsx b/packages/html-reporter/src/statusIcon.tsx index 3cec4fc0d3..155f14041b 100644 --- a/packages/html-reporter/src/statusIcon.tsx +++ b/packages/html-reporter/src/statusIcon.tsx @@ -18,7 +18,7 @@ import * as icons from './icons'; import './colors.css'; import './common.css'; -export function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky'): JSX.Element { +export function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' | 'expected' | 'unexpected' | 'flaky' | 'interrupted'): JSX.Element { switch (status) { case 'failed': case 'unexpected': @@ -31,6 +31,7 @@ export function statusIcon(status: 'failed' | 'timedOut' | 'skipped' | 'passed' case 'flaky': return icons.warning(); case 'skipped': + case 'interrupted': return icons.blank(); } } diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index c2501edd0b..409836a03b 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -38,7 +38,7 @@ type ErrorDetails = { type TestSummary = { skipped: number; expected: number; - skippedWithError: TestCase[]; + interrupted: TestCase[]; unexpected: TestCase[]; flaky: TestCase[]; failuresToPrint: TestCase[]; @@ -131,7 +131,7 @@ export class BaseReporter implements Reporter { return fileDurations.filter(([,duration]) => duration > threshold).slice(0, count); } - protected generateSummaryMessage({ skipped, expected, unexpected, flaky, fatalErrors }: TestSummary) { + protected generateSummaryMessage({ skipped, expected, interrupted, unexpected, flaky, fatalErrors }: TestSummary) { const tokens: string[] = []; if (fatalErrors.length) tokens.push(colors.red(` ${fatalErrors.length} fatal ${fatalErrors.length === 1 ? 'error' : 'errors'}`)); @@ -140,6 +140,11 @@ export class BaseReporter implements Reporter { for (const test of unexpected) tokens.push(colors.red(formatTestHeader(this.config, test, ' '))); } + if (interrupted.length) { + tokens.push(colors.red(` ${interrupted.length} interrupted`)); + for (const test of interrupted) + tokens.push(colors.red(formatTestHeader(this.config, test, ' '))); + } if (flaky.length) { tokens.push(colors.yellow(` ${flaky.length} flaky`)); for (const test of flaky) @@ -158,16 +163,21 @@ export class BaseReporter implements Reporter { protected generateSummary(): TestSummary { let skipped = 0; let expected = 0; - const skippedWithError: TestCase[] = []; + const interrupted: TestCase[] = []; + const interruptedToPrint: TestCase[] = []; const unexpected: TestCase[] = []; const flaky: TestCase[] = []; this.suite.allTests().forEach(test => { switch (test.outcome()) { case 'skipped': { - ++skipped; - if (test.results.some(result => !!result.error)) - skippedWithError.push(test); + if (test.results.some(result => result.status === 'interrupted')) { + if (test.results.some(result => !!result.error)) + interruptedToPrint.push(test); + interrupted.push(test); + } else { + ++skipped; + } break; } case 'expected': ++expected; break; @@ -176,11 +186,11 @@ export class BaseReporter implements Reporter { } }); - const failuresToPrint = [...unexpected, ...flaky, ...skippedWithError]; + const failuresToPrint = [...unexpected, ...flaky, ...interruptedToPrint]; return { skipped, expected, - skippedWithError, + interrupted, unexpected, flaky, failuresToPrint, @@ -313,6 +323,11 @@ export function formatResultFailure(config: FullConfig, test: TestCase, result: message: indent(colors.red(`Expected to fail, but passed.`), initialIndent), }); } + if (result.status === 'interrupted') { + errorDetails.push({ + message: indent(colors.red(`Test was interrupted.`), initialIndent), + }); + } for (const error of result.errors) { const formattedError = formatError(config, error, highlightCode, test.location.file); diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index 76c49fe628..2b9db8dc62 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -104,7 +104,7 @@ export type TestResult = { steps: TestStep[]; errors: string[]; attachments: TestAttachment[]; - status: 'passed' | 'failed' | 'timedOut' | 'skipped'; + status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; }; export type TestStep = { diff --git a/packages/playwright-test/src/reporters/line.ts b/packages/playwright-test/src/reporters/line.ts index 88ca4af977..9b68d9ecfa 100644 --- a/packages/playwright-test/src/reporters/line.ts +++ b/packages/playwright-test/src/reporters/line.ts @@ -79,7 +79,7 @@ class LineReporter extends BaseReporter { override onTestEnd(test: TestCase, result: TestResult) { super.onTestEnd(test, result); - if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected')) { + if (!this.willRetry(test) && (test.outcome() === 'flaky' || test.outcome() === 'unexpected' || result.status === 'interrupted')) { if (!process.env.PW_TEST_DEBUG_REPORTERS) process.stdout.write(`\u001B[1A\u001B[2K`); console.log(formatFailure(this.config, test, { diff --git a/packages/playwright-test/src/test.ts b/packages/playwright-test/src/test.ts index 5f50fc2fcc..5e835a0531 100644 --- a/packages/playwright-test/src/test.ts +++ b/packages/playwright-test/src/test.ts @@ -159,7 +159,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { } outcome(): 'skipped' | 'expected' | 'unexpected' | 'flaky' { - const nonSkipped = this.results.filter(result => result.status !== 'skipped'); + const nonSkipped = this.results.filter(result => result.status !== 'skipped' && result.status !== 'interrupted'); if (!nonSkipped.length) return 'skipped'; if (nonSkipped.every(result => result.status === this.expectedStatus)) diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index fd3bf13486..bb6c49ce26 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -74,9 +74,8 @@ export class WorkerRunner extends EventEmitter { // Interrupt current action. this._currentTest?._timeoutManager.interrupt(); - // TODO: mark test as 'interrupted' instead. if (this._currentTest && this._currentTest.status === 'passed') - this._currentTest.status = 'skipped'; + this._currentTest.status = 'interrupted'; } return this._runFinished; } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 300760fa57..bf6d8ad486 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -1294,7 +1294,7 @@ export interface FullConfig { webServer: TestConfigWebServer | null; } -export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; +export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; /** * `WorkerInfo` contains information about the worker that is running tests. It is available to @@ -1506,7 +1506,7 @@ export interface TestInfo { * ``` * */ - expectedStatus: "passed"|"failed"|"timedOut"|"skipped"; + expectedStatus: "passed"|"failed"|"timedOut"|"skipped"|"interrupted"; /** * Marks the currently running test as "should fail". Playwright Test runs this test and ensures that it is actually @@ -1714,7 +1714,7 @@ export interface TestInfo { * ``` * */ - status?: "passed"|"failed"|"timedOut"|"skipped"; + status?: "passed"|"failed"|"timedOut"|"skipped"|"interrupted"; /** * Output written to `process.stderr` or `console.error` during the test execution. diff --git a/tests/playwright-test/basic.spec.ts b/tests/playwright-test/basic.spec.ts index 799c7fce02..c6c8bbdab3 100644 --- a/tests/playwright-test/basic.spec.ts +++ b/tests/playwright-test/basic.spec.ts @@ -424,7 +424,7 @@ test('should not reuse worker after unhandled rejection in test.fail', async ({ }, { workers: 1 }); expect(result.exitCode).toBe(1); expect(result.failed).toBe(1); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(result.output).toContain(`Error: Oh my!`); expect(result.output).not.toContain(`Did not teardown test scope`); }); diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index 44858cc90a..fd3e36e548 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -37,6 +37,7 @@ export type RunResult = { failed: number, flaky: number, skipped: number, + interrupted: number, report: JSONReport, results: any[], }; @@ -156,7 +157,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b testProcess.onOutput = () => { if (options.sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= options.sendSIGINTAfter) { didSendSigint = true; - process.kill(testProcess.process.pid, 'SIGINT'); + process.kill(testProcess.process.pid!, 'SIGINT'); } }; const { exitCode } = await testProcess.exited; @@ -176,6 +177,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b const failed = summary(/(\d+) failed/g); const flaky = summary(/(\d+) flaky/g); const skipped = summary(/(\d+) skipped/g); + const interrupted = summary(/(\d+) interrupted/g); let report; try { report = JSON.parse(fs.readFileSync(reportFile).toString()); @@ -205,6 +207,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b failed, flaky, skipped, + interrupted, report, results, }; diff --git a/tests/playwright-test/playwright.expect.text.spec.ts b/tests/playwright-test/playwright.expect.text.spec.ts index 03c1548ebe..fa0a9c2da7 100644 --- a/tests/playwright-test/playwright.expect.text.spec.ts +++ b/tests/playwright-test/playwright.expect.text.spec.ts @@ -657,7 +657,7 @@ test('should print expected/received on Ctrl+C', async ({ runInlineTest }) => { }, { workers: 1 }, {}, { sendSIGINTAfter: 1 }); expect(result.exitCode).toBe(130); expect(result.passed).toBe(0); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(stripAnsi(result.output)).toContain('Expected string: "Text 2"'); expect(stripAnsi(result.output)).toContain('Received string: "Text content"'); }); diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts index 54e251d085..834e603416 100644 --- a/tests/playwright-test/playwright.spec.ts +++ b/tests/playwright-test/playwright.spec.ts @@ -450,7 +450,7 @@ test('should report click error on sigint', async ({ runInlineTest }) => { expect(result.exitCode).toBe(130); expect(result.passed).toBe(0); expect(result.failed).toBe(0); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(stripAnsi(result.output)).toContain(`8 | const promise = page.click('text=Missing');`); }); diff --git a/tests/playwright-test/playwright.unhandled.spec.ts b/tests/playwright-test/playwright.unhandled.spec.ts index 0d2877a5c1..9214001bd2 100644 --- a/tests/playwright-test/playwright.unhandled.spec.ts +++ b/tests/playwright-test/playwright.unhandled.spec.ts @@ -29,7 +29,7 @@ test('should lead in uncaughtException when page.route raises', async ({ runInli }); `, }, { workers: 1 }); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(result.output).toContain('foobar'); }); @@ -46,7 +46,7 @@ test('should lead in unhandledRejection when page.route raises', async ({ runInl }); `, }, { workers: 1 }); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(result.output).toContain('foobar'); }); @@ -63,7 +63,7 @@ test('should lead in uncaughtException when context.route raises', async ({ runI }); `, }, { workers: 1 }); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(result.output).toContain('foobar'); }); @@ -80,6 +80,6 @@ test('should lead in unhandledRejection when context.route raises', async ({ run }); `, }, { workers: 1 }); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(result.output).toContain('foobar'); }); diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts index 4e1f73c10d..10970135fb 100644 --- a/tests/playwright-test/runner.spec.ts +++ b/tests/playwright-test/runner.spec.ts @@ -112,7 +112,8 @@ test('sigint should stop workers', async ({ runInlineTest }) => { expect(result.exitCode).toBe(130); expect(result.passed).toBe(0); expect(result.failed).toBe(0); - expect(result.skipped).toBe(4); + expect(result.skipped).toBe(2); + expect(result.interrupted).toBe(2); expect(result.output).toContain('%%SEND-SIGINT%%1'); expect(result.output).toContain('%%SEND-SIGINT%%2'); expect(result.output).not.toContain('%%skipped1'); @@ -177,7 +178,7 @@ test('worker interrupt should report errors', async ({ runInlineTest }) => { expect(result.exitCode).toBe(130); expect(result.passed).toBe(0); expect(result.failed).toBe(0); - expect(result.skipped).toBe(1); + expect(result.interrupted).toBe(1); expect(result.output).toContain('%%SEND-SIGINT%%'); expect(result.output).toContain('Error: INTERRUPT'); }); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 286b0838dc..507fbc6ec4 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -95,7 +95,7 @@ export interface FullConfig { // [internal] !!! DO NOT ADD TO THIS !!! See prior note. } -export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped'; +export type TestStatus = 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; export interface WorkerInfo { config: FullConfig;