diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index d6f1d87513..00c7a5f0ee 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -1767,118 +1767,63 @@ Whether to box the step in the report. Defaults to `false`. When the step is box Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. +## async method: Test.step.skip +* since: v1.50 +- returns: <[void]> + +Mark a test step as "skip" to temporarily disable its execution, useful for steps that are currently failing and planned for a near-term fix. Playwright will not run the step. + +**Usage** + +You can declare a skipped step, and Playwright will not run it. + +```js +import { test, expect } from '@playwright/test'; + +test('my test', async ({ page }) => { + // ... + await test.step.skip('not yet ready', async () => { + // ... + }); +}); +``` + +### param: Test.step.skip.title +* since: v1.50 +- `title` <[string]> + +Step name. + +### param: Test.step.skip.body +* since: v1.50 +- `body` <[function]\(\):[Promise]<[any]>> + +Step body. + +### option: Test.step.skip.box +* since: v1.50 +- `box` + +Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details. + +### option: Test.step.skip.location +* since: v1.50 +- `location` <[Location]> + +Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. + +### option: Test.step.skip.timeout +* since: v1.50 +- `timeout` <[float]> + +Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout). + ### option: Test.step.timeout * since: v1.50 - `timeout` <[float]> Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout). -## async method: Test.step.fail -* since: v1.50 -- returns: <[void]> - -Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - -:::note -If the step exceeds the timeout, a [TimeoutError] is thrown. This indicates the step did not fail as expected. -::: - -**Usage** - -You can declare a test step as failing, so that Playwright ensures it actually fails. - -```js -import { test, expect } from '@playwright/test'; - -test('my test', async ({ page }) => { - // ... - await test.step.fail('currently failing', async () => { - // ... - }); -}); -``` - -### param: Test.step.fail.title -* since: v1.50 -- `title` <[string]> - -Step name. - -### param: Test.step.fail.body -* since: v1.50 -- `body` <[function]\(\):[Promise]<[any]>> - -Step body. - -### option: Test.step.fail.box -* since: v1.50 -- `box` - -Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details. - -### option: Test.step.fail.location -* since: v1.50 -- `location` <[Location]> - -Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. - -### option: Test.step.fail.timeout -* since: v1.50 -- `timeout` <[float]> - -Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout). - -## async method: Test.step.fixme -* since: v1.50 -- returns: <[void]> - -Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step. - -**Usage** - -You can declare a test step as failing, so that Playwright ensures it actually fails. - -```js -import { test, expect } from '@playwright/test'; - -test('my test', async ({ page }) => { - // ... - await test.step.fixme('not yet ready', async () => { - // ... - }); -}); -``` - -### param: Test.step.fixme.title -* since: v1.50 -- `title` <[string]> - -Step name. - -### param: Test.step.fixme.body -* since: v1.50 -- `body` <[function]\(\):[Promise]<[any]>> - -Step body. - -### option: Test.step.fixme.box -* since: v1.50 -- `box` - -Whether to box the step in the report. Defaults to `false`. When the step is boxed, errors thrown from the step internals point to the step call site. See below for more details. - -### option: Test.step.fixme.location -* since: v1.50 -- `location` <[Location]> - -Specifies a custom location for the step to be shown in test reports and trace viewer. By default, location of the [`method: Test.step`] call is shown. - -### option: Test.step.fixme.timeout -* since: v1.50 -- `timeout` <[float]> - -Maximum time in milliseconds for the step to finish. Defaults to `0` (no timeout). - ## method: Test.use * since: v1.10 diff --git a/packages/html-reporter/src/testResultView.tsx b/packages/html-reporter/src/testResultView.tsx index 4c504a0118..64497b3bcd 100644 --- a/packages/html-reporter/src/testResultView.tsx +++ b/packages/html-reporter/src/testResultView.tsx @@ -176,11 +176,11 @@ const StepTreeItem: React.FC<{ }> = ({ test, step, result, depth }) => { return {msToString(step.duration)} - {statusIcon(step.error || step.duration === -1 ? 'failed' : 'passed')} + {statusIcon(step.error || step.duration === -1 ? 'failed' : (step.skipped ? 'skipped' : 'passed'))} {step.title} {step.count > 1 && <> ✕ {step.count}} {step.location && — {step.location.file}:{step.location.line}} - } loadChildren={step.steps.length + (step.snippet ? 1 : 0) ? () => { + } loadChildren={step.steps.length || step.snippet ? () => { const snippet = step.snippet ? [] : []; const steps = step.steps.map((s, i) => ); const attachments = step.attachments.map(attachmentIndex => ( diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts index 7a99184739..5db8199d34 100644 --- a/packages/html-reporter/src/types.d.ts +++ b/packages/html-reporter/src/types.d.ts @@ -110,4 +110,5 @@ export type TestStep = { steps: TestStep[]; attachments: number[]; count: number; + skipped?: boolean; }; diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index 61f9b36824..4c9e6d8f90 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -57,8 +57,7 @@ export class TestTypeImpl { test.slow = wrapFunctionWithLocation(this._modifier.bind(this, 'slow')); test.setTimeout = wrapFunctionWithLocation(this._setTimeout.bind(this)); test.step = this._step.bind(this, 'pass'); - test.step.fail = this._step.bind(this, 'fail'); - test.step.fixme = this._step.bind(this, 'fixme'); + test.step.skip = this._step.bind(this, 'skip'); test.use = wrapFunctionWithLocation(this._use.bind(this)); test.extend = wrapFunctionWithLocation(this._extend.bind(this)); test.info = () => { @@ -259,40 +258,27 @@ export class TestTypeImpl { suite._use.push({ fixtures, location }); } - async _step(expectation: 'pass'|'fail'|'fixme', title: string, body: () => T | Promise, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise { + async _step(expectation: 'pass'|'skip', title: string, body: () => T | Promise, options: {box?: boolean, location?: Location, timeout?: number } = {}): Promise { const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`test.step() can only be called from a test`); - if (expectation === 'fixme') + if (expectation === 'skip') { + const step = testInfo._addStep({ category: 'test.step.skip', title, location: options.location, box: options.box }); + step.complete({}); return undefined as T; + } const step = testInfo._addStep({ category: 'test.step', title, location: options.location, box: options.box }); return await zones.run('stepZone', step, async () => { - let result; - let error; try { - result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0); - } catch (e) { - error = e; - } - if (result?.timedOut) { - const error = new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`); + const result = await raceAgainstDeadline(async () => body(), options.timeout ? monotonicTime() + options.timeout : 0); + if (result.timedOut) + throw new errors.TimeoutError(`Step timeout ${options.timeout}ms exceeded.`); + step.complete({}); + return result.result; + } catch (error) { step.complete({ error }); throw error; } - const expectedToFail = expectation === 'fail'; - if (error) { - step.complete({ error }); - if (expectedToFail) - return undefined as T; - throw error; - } - if (expectedToFail) { - error = new Error(`Step is expected to fail, but passed`); - step.complete({ error }); - throw error; - } - step.complete({}); - return result!.result; }); } diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index e14be98f63..1fd7695293 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -517,6 +517,7 @@ class HtmlBuilder { private _createTestStep(dedupedStep: DedupedStep, result: api.TestResult): TestStep { const { step, duration, count } = dedupedStep; + const skipped = dedupedStep.step.category === 'test.step.skip'; const testStep: TestStep = { title: step.title, startTime: step.startTime.toISOString(), @@ -530,7 +531,8 @@ class HtmlBuilder { }), location: this._relativeLocation(step.location), error: step.error?.message, - count + count, + skipped }; if (step.location) this._stepsInFile.set(step.location.file, testStep); diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts index 6577e19d0d..569e72c5dd 100644 --- a/packages/playwright/src/worker/testInfo.ts +++ b/packages/playwright/src/worker/testInfo.ts @@ -35,7 +35,7 @@ export interface TestStepInternal { attachmentIndices: number[]; stepId: string; title: string; - category: 'hook' | 'fixture' | 'test.step' | 'expect' | 'attach' | string; + category: 'hook' | 'fixture' | 'test.step' | 'test.step.skip' | 'expect' | 'attach' | string; location?: Location; boxedStack?: StackFrame[]; steps: TestStepInternal[]; diff --git a/packages/playwright/types/test.d.ts b/packages/playwright/types/test.d.ts index 56407fe1e0..501d1bcdb1 100644 --- a/packages/playwright/types/test.d.ts +++ b/packages/playwright/types/test.d.ts @@ -5713,18 +5713,19 @@ export interface TestType { */ (title: string, body: () => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; /** - * Mark a test step as "fixme", with the intention to fix it. Playwright will not run the step. + * Mark a test step as "skip" to temporarily disable its execution, useful for steps that are currently failing and + * planned for a near-term fix. Playwright will not run the step. * * **Usage** * - * You can declare a test step as failing, so that Playwright ensures it actually fails. + * You can declare a skipped step, and Playwright will not run it. * * ```js * import { test, expect } from '@playwright/test'; * * test('my test', async ({ page }) => { * // ... - * await test.step.fixme('not yet ready', async () => { + * await test.step.skip('not yet ready', async () => { * // ... * }); * }); @@ -5734,34 +5735,7 @@ export interface TestType { * @param body Step body. * @param options */ - fixme(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; - /** - * Marks a test step as "should fail". Playwright runs this test step and ensures that it actually fails. This is - * useful for documentation purposes to acknowledge that some functionality is broken until it is fixed. - * - * **NOTE** If the step exceeds the timeout, a [TimeoutError](https://playwright.dev/docs/api/class-timeouterror) is - * thrown. This indicates the step did not fail as expected. - * - * **Usage** - * - * You can declare a test step as failing, so that Playwright ensures it actually fails. - * - * ```js - * import { test, expect } from '@playwright/test'; - * - * test('my test', async ({ page }) => { - * // ... - * await test.step.fail('currently failing', async () => { - * // ... - * }); - * }); - * ``` - * - * @param title Step name. - * @param body Step body. - * @param options - */ - fail(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + skip(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; } /** * `expect` function can be used to create test assertions. Read more about [test assertions](https://playwright.dev/docs/test-assertions). diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index f1e2bb4eda..2640cb61c9 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -757,6 +757,33 @@ for (const useIntermediateMergeReport of [true, false] as const) { ]); }); + test('should show skipped step snippets', async ({ runInlineTest, page, showReport }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + export default { testDir: './tests' }; + `, + 'tests/a.test.ts': ` + import { test, expect } from '@playwright/test'; + + test('example', async ({}) => { + await test.step.skip('skipped step title', async () => { + expect(1).toBe(1); + await test.step('inner step', async () => { + expect(1).toBe(1); + }); + }); + }); + `, + }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + + await showReport(); + await page.click('text=example'); + await page.click('text=skipped step title'); + await expect(page.getByTestId('test-snippet')).toContainText(`await test.step.skip('skipped step title', async () => {`); + }); + test('should render annotations', async ({ runInlineTest, page, showReport }) => { const result = await runInlineTest({ 'playwright.config.js': ` diff --git a/tests/playwright-test/test-step.spec.ts b/tests/playwright-test/test-step.spec.ts index 0e1926e17a..ec04ef19e8 100644 --- a/tests/playwright-test/test-step.spec.ts +++ b/tests/playwright-test/test-step.spec.ts @@ -1499,29 +1499,26 @@ fixture | fixture: context `); }); -test('test.step.fail and test.step.fixme should work', async ({ runInlineTest }) => { +test('test.step.skip should work', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': stepIndentReporter, 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, 'a.test.ts': ` import { test, expect } from '@playwright/test'; test('test', async ({ }) => { - await test.step('outer step 1', async () => { - await test.step.fail('inner step 1.1', async () => { + await test.step.skip('outer step 1', async () => { + await test.step('inner step 1.1', async () => { throw new Error('inner step 1.1 failed'); }); - await test.step.fixme('inner step 1.2', async () => {}); + await test.step.skip('inner step 1.2', async () => {}); await test.step('inner step 1.3', async () => {}); }); await test.step('outer step 2', async () => { - await test.step.fixme('inner step 2.1', async () => {}); + await test.step.skip('inner step 2.1', async () => {}); await test.step('inner step 2.2', async () => { expect(1).toBe(1); }); }); - await test.step.fail('outer step 3', async () => { - throw new Error('outer step 3 failed'); - }); }); ` }, { reporter: '' }); @@ -1531,48 +1528,16 @@ test('test.step.fail and test.step.fixme should work', async ({ runInlineTest }) expect(result.report.stats.unexpected).toBe(0); expect(stripAnsi(result.output)).toBe(` hook |Before Hooks -test.step |outer step 1 @ a.test.ts:4 -test.step | inner step 1.1 @ a.test.ts:5 -test.step | ↪ error: Error: inner step 1.1 failed -test.step | inner step 1.3 @ a.test.ts:9 +test.step.skip|outer step 1 @ a.test.ts:4 test.step |outer step 2 @ a.test.ts:11 +test.step.skip| inner step 2.1 @ a.test.ts:12 test.step | inner step 2.2 @ a.test.ts:13 expect | expect.toBe @ a.test.ts:14 -test.step |outer step 3 @ a.test.ts:17 -test.step |↪ error: Error: outer step 3 failed hook |After Hooks `); }); -test('timeout inside test.step.fail is an error', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'reporter.ts': stepIndentReporter, - 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('test 2', async ({ }) => { - await test.step('outer step 2', async () => { - await test.step.fail('inner step 2', async () => { - await new Promise(() => {}); - }); - }); - }); - ` - }, { reporter: '', timeout: 2500 }); - - expect(result.exitCode).toBe(1); - expect(result.report.stats.unexpected).toBe(1); - expect(stripAnsi(result.output)).toBe(` -hook |Before Hooks -test.step |outer step 2 @ a.test.ts:4 -test.step | inner step 2 @ a.test.ts:5 -hook |After Hooks -hook |Worker Cleanup - |Test timeout of 2500ms exceeded. -`); -}); - -test('skip test.step.fixme body', async ({ runInlineTest }) => { +test('skip test.step.skip body', async ({ runInlineTest }) => { const result = await runInlineTest({ 'reporter.ts': stepIndentReporter, 'playwright.config.ts': `module.exports = { reporter: './reporter' };`, @@ -1581,7 +1546,7 @@ test('skip test.step.fixme body', async ({ runInlineTest }) => { test('test', async ({ }) => { let didRun = false; await test.step('outer step 2', async () => { - await test.step.fixme('inner step 2', async () => { + await test.step.skip('inner step 2', async () => { didRun = true; }); }); @@ -1595,6 +1560,7 @@ test('skip test.step.fixme body', async ({ runInlineTest }) => { expect(stripAnsi(result.output)).toBe(` hook |Before Hooks test.step |outer step 2 @ a.test.ts:5 +test.step.skip| inner step 2 @ a.test.ts:6 expect |expect.toBe @ a.test.ts:10 hook |After Hooks `); diff --git a/tests/playwright-test/types-2.spec.ts b/tests/playwright-test/types-2.spec.ts index 3a06ed0da2..d517d75a7a 100644 --- a/tests/playwright-test/types-2.spec.ts +++ b/tests/playwright-test/types-2.spec.ts @@ -205,21 +205,14 @@ test('step should inherit return type from its callback ', async ({ runTSC }) => expect(result.exitCode).toBe(0); }); -test('step.fail and step.fixme return void ', async ({ runTSC }) => { +test('step.skip returns void ', async ({ runTSC }) => { const result = await runTSC({ 'a.spec.ts': ` import { test, expect } from '@playwright/test'; - test('test step.fail', async ({ }) => { + test('test step.skip', async ({ }) => { // @ts-expect-error - const bad1: string = await test.step.fail('my step', () => { }); - const good: void = await test.step.fail('my step', async () => { - return 2024; - }); - }); - test('test step.fixme', async ({ }) => { - // @ts-expect-error - const bad1: string = await test.step.fixme('my step', () => { }); - const good: void = await test.step.fixme('my step', async () => { + const bad1: string = await test.step.skip('my step', () => { return ''; }); + const good: void = await test.step.skip('my step', async () => { return 2024; }); }); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index 3370103a25..1bc980b42d 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -164,8 +164,7 @@ export interface TestType { use(fixtures: Fixtures<{}, {}, TestArgs, WorkerArgs>): void; step: { (title: string, body: () => T | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; - fixme(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; - fail(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; + skip(title: string, body: () => any | Promise, options?: { box?: boolean, location?: Location, timeout?: number }): Promise; } expect: Expect<{}>; extend(fixtures: Fixtures): TestType;