From ab7e794bf7d0f8580f8c43d98ed41d80c337ff27 Mon Sep 17 00:00:00 2001 From: Dmitry Gozman Date: Thu, 18 May 2023 13:07:22 -0700 Subject: [PATCH] feat(shard): introduce `mode: 'default'` (#23023) This mode allows a suite to opt-out from parallelism. Useful to setup multiple suites running in parallel, with each suite not being sharded. References #22891. --- docs/src/test-api/class-test.md | 20 ++++++- packages/playwright-test/src/common/test.ts | 2 +- .../playwright-test/src/common/testType.ts | 12 ++-- .../src/isomorphic/teleReceiver.ts | 4 +- .../playwright-test/src/runner/testGroups.ts | 10 ++-- .../playwright-test/types/reporterPrivate.ts | 2 +- packages/playwright-test/types/test.d.ts | 20 ++++++- tests/playwright-test/shard.spec.ts | 59 +++++++++++++++++++ utils/generate_types/overrides-test.d.ts | 2 +- 9 files changed, 115 insertions(+), 16 deletions(-) diff --git a/docs/src/test-api/class-test.md b/docs/src/test-api/class-test.md index 1ec8b4b64b..23a457c765 100644 --- a/docs/src/test-api/class-test.md +++ b/docs/src/test-api/class-test.md @@ -284,9 +284,27 @@ Learn more about the execution modes [here](../test-parallel.md). test('runs second', async ({ page }) => {}); ``` +* Run multiple describes in parallel, but tests inside each describe in order. + + ```js + test.describe.configure({ mode: 'parallel' }); + + test.describe('A, runs in parallel with B', () => { + test.describe.configure({ mode: 'default' }); + test('in order A1', async ({ page }) => {}); + test('in order A2', async ({ page }) => {}); + }); + + test.describe('B, runs in parallel with A', () => { + test.describe.configure({ mode: 'default' }); + test('in order B1', async ({ page }) => {}); + test('in order B2', async ({ page }) => {}); + }); + ``` + ### option: Test.describe.configure.mode * since: v1.10 -- `mode` <[TestMode]<"parallel"|"serial">> +- `mode` <[TestMode]<"default"|"parallel"|"serial">> Execution mode. Learn more about the execution modes [here](../test-parallel.md). diff --git a/packages/playwright-test/src/common/test.ts b/packages/playwright-test/src/common/test.ts index ce05cb897a..de9e6a8111 100644 --- a/packages/playwright-test/src/common/test.ts +++ b/packages/playwright-test/src/common/test.ts @@ -50,7 +50,7 @@ export class Suite extends Base implements SuitePrivate { _retries: number | undefined; _staticAnnotations: Annotation[] = []; _modifiers: Modifier[] = []; - _parallelMode: 'default' | 'serial' | 'parallel' = 'default'; + _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; _fullProject: FullProjectInternal | undefined; _fileId: string | undefined; readonly _type: 'root' | 'project' | 'file' | 'describe'; diff --git a/packages/playwright-test/src/common/testType.ts b/packages/playwright-test/src/common/testType.ts index fe7cd4b5e9..5f8d429691 100644 --- a/packages/playwright-test/src/common/testType.ts +++ b/packages/playwright-test/src/common/testType.ts @@ -124,6 +124,8 @@ export class TestTypeImpl { for (let parent: Suite | undefined = suite; parent; parent = parent.parent) { if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel') throw new Error('describe.parallel cannot be nested inside describe.serial'); + if (parent._parallelMode === 'default' && child._parallelMode === 'parallel') + throw new Error('describe.parallel cannot be nested inside describe with default mode'); } setCurrentlyLoadingFileSuite(child); @@ -138,7 +140,7 @@ export class TestTypeImpl { suite._hooks.push({ type: name, fn, location }); } - private _configure(location: Location, options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) { + private _configure(location: Location, options: { mode?: 'default' | 'parallel' | 'serial', retries?: number, timeout?: number }) { throwIfRunningInsideJest(); const suite = this._currentSuite(location, `test.describe.configure()`); if (!suite) @@ -151,12 +153,14 @@ export class TestTypeImpl { suite._retries = options.retries; if (options.mode !== undefined) { - if (suite._parallelMode !== 'default') - throw new Error('Parallel mode is already assigned for the enclosing scope.'); + if (suite._parallelMode !== 'none') + throw new Error(`"${suite._parallelMode}" mode is already assigned for the enclosing scope.`); suite._parallelMode = options.mode; for (let parent: Suite | undefined = suite.parent; parent; parent = parent.parent) { if (parent._parallelMode === 'serial' && suite._parallelMode === 'parallel') - throw new Error('describe.parallel cannot be nested inside describe.serial'); + throw new Error('describe with parallel mode cannot be nested inside describe with serial mode'); + if (parent._parallelMode === 'default' && suite._parallelMode === 'parallel') + throw new Error('describe with parallel mode cannot be nested inside describe with default mode'); } } } diff --git a/packages/playwright-test/src/isomorphic/teleReceiver.ts b/packages/playwright-test/src/isomorphic/teleReceiver.ts index b84416be0d..40515f6469 100644 --- a/packages/playwright-test/src/isomorphic/teleReceiver.ts +++ b/packages/playwright-test/src/isomorphic/teleReceiver.ts @@ -61,7 +61,7 @@ export type JsonSuite = { suites: JsonSuite[]; tests: JsonTestCase[]; fileId: string | undefined; - parallelMode: 'default' | 'serial' | 'parallel'; + parallelMode: 'none' | 'default' | 'serial' | 'parallel'; }; export type JsonTestCase = { @@ -383,7 +383,7 @@ export class TeleSuite implements SuitePrivate { _timeout: number | undefined; _retries: number | undefined; _fileId: string | undefined; - _parallelMode: 'default' | 'serial' | 'parallel' = 'default'; + _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; readonly _type: 'root' | 'project' | 'file' | 'describe'; constructor(title: string, type: 'root' | 'project' | 'file' | 'describe') { diff --git a/packages/playwright-test/src/runner/testGroups.ts b/packages/playwright-test/src/runner/testGroups.ts index 9066cfc6d3..5a70bc84cc 100644 --- a/packages/playwright-test/src/runner/testGroups.ts +++ b/packages/playwright-test/src/runner/testGroups.ts @@ -79,20 +79,20 @@ export function createTestGroups(projectSuite: Suite, workers: number): TestGrou // Note that a parallel suite cannot be inside a serial suite. This is enforced in TestType. let insideParallel = false; - let outerMostSerialSuite: Suite | undefined; + let outerMostSequentialSuite: Suite | undefined; let hasAllHooks = false; for (let parent: Suite | undefined = test.parent; parent; parent = parent.parent) { - if (parent._parallelMode === 'serial') - outerMostSerialSuite = parent; + if (parent._parallelMode === 'serial' || parent._parallelMode === 'default') + outerMostSequentialSuite = parent; insideParallel = insideParallel || parent._parallelMode === 'parallel'; hasAllHooks = hasAllHooks || parent._hooks.some(hook => hook.type === 'beforeAll' || hook.type === 'afterAll'); } if (insideParallel) { - if (hasAllHooks && !outerMostSerialSuite) { + if (hasAllHooks && !outerMostSequentialSuite) { withRequireFile.parallelWithHooks.tests.push(test); } else { - const key = outerMostSerialSuite || test; + const key = outerMostSequentialSuite || test; let group = withRequireFile.parallel.get(key); if (!group) { group = createGroup(test); diff --git a/packages/playwright-test/types/reporterPrivate.ts b/packages/playwright-test/types/reporterPrivate.ts index ca2105c5d8..198b2fe9ea 100644 --- a/packages/playwright-test/types/reporterPrivate.ts +++ b/packages/playwright-test/types/reporterPrivate.ts @@ -18,5 +18,5 @@ import type { Suite } from './testReporter'; export interface SuitePrivate extends Suite { _fileId: string | undefined; - _parallelMode: 'default' | 'serial' | 'parallel'; + _parallelMode: 'none' | 'default' | 'serial' | 'parallel'; } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 32d96c846e..95434fcd8b 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -2626,9 +2626,27 @@ export interface TestType {}); * ``` * + * - Run multiple describes in parallel, but tests inside each describe in order. + * + * ```js + * test.describe.configure({ mode: 'parallel' }); + * + * test.describe('A, runs in parallel with B', () => { + * test.describe.configure({ mode: 'default' }); + * test('in order A1', async ({ page }) => {}); + * test('in order A2', async ({ page }) => {}); + * }); + * + * test.describe('B, runs in parallel with A', () => { + * test.describe.configure({ mode: 'default' }); + * test('in order B1', async ({ page }) => {}); + * test('in order B2', async ({ page }) => {}); + * }); + * ``` + * * @param options */ - configure: (options: { mode?: 'parallel' | 'serial', retries?: number, timeout?: number }) => void; + configure: (options: { mode?: 'default' | 'parallel' | 'serial', retries?: number, timeout?: number }) => void; }; /** * Declares a skipped test, similarly to diff --git a/tests/playwright-test/shard.spec.ts b/tests/playwright-test/shard.spec.ts index 70095abcca..838e8fb222 100644 --- a/tests/playwright-test/shard.spec.ts +++ b/tests/playwright-test/shard.spec.ts @@ -225,3 +225,62 @@ test('should skip dependency when project is sharded out', async ({ runInlineTes 'test in tests2', ]); }); + +test('should not shard mode:default suites', async ({ runInlineTest }) => { + test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/22891' }); + + const tests = { + 'a1.spec.ts': ` + import { test } from '@playwright/test'; + test('test0', async ({ }) => { + console.log('\\n%%test0'); + }); + test('test1', async ({ }) => { + console.log('\\n%%test1'); + }); + `, + 'a2.spec.ts': ` + import { test } from '@playwright/test'; + test.describe.configure({ mode: 'parallel' }); + + test.describe(() => { + test.describe.configure({ mode: 'default' }); + test.beforeAll(() => { + console.log('\\n%%beforeAll1'); + }); + test('test2', async ({ }) => { + console.log('\\n%%test2'); + }); + test('test3', async ({ }) => { + console.log('\\n%%test3'); + }); + }); + + test.describe(() => { + test.describe.configure({ mode: 'default' }); + test.beforeAll(() => { + console.log('\\n%%beforeAll2'); + }); + test('test4', async ({ }) => { + console.log('\\n%%test4'); + }); + test('test5', async ({ }) => { + console.log('\\n%%test5'); + }); + }); + `, + }; + + { + const result = await runInlineTest(tests, { shard: '2/3', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.outputLines).toEqual(['beforeAll1', 'test2', 'test3']); + } + { + const result = await runInlineTest(tests, { shard: '3/3', workers: 1 }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.outputLines).toEqual(['beforeAll2', 'test4', 'test5']); + } +}); diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index f9a9e6b56c..15f32665a5 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -131,7 +131,7 @@ export interface TestType void; + configure: (options: { mode?: 'default' | 'parallel' | 'serial', retries?: number, timeout?: number }) => void; }; skip(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise | void): void; skip(): void;