yujunjun/playwright
1
0
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-06-26 21:40:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(parallel): allow setting enclosing scope parallel mode (#11822)

Browse Source
This commit is contained in:
Pavel Feldman 2022-02-02 20:44:11 -08:00 committed by GitHub
parent ba0c7e679b
commit fdda759a9d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 363 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
docs/src/test-api/class-test.md
View File

@ -248,6 +248,46 @@ Group title.
A callback that is run immediately when calling [`method: Test.describe`]. Any tests added in this callback will belong to the group.
## method: Test.describe.configure
Set execution mode of execution for the enclosing scope. Can be executed either on the top level or inside a describe. Configuration applies to the entire scope, regardless of whether it run before or after the test
declaration.
Learn more about the execution modes [here](./test-parallel-js.md).
```js js-flavor=js
// Run all the tests in the file concurrently using parallel workers.
test.describe.configure({ mode: 'parallel' });
test('runs in parallel 1', async ({ page }) => {});
test('runs in parallel 2', async ({ page }) => {});
```
```js js-flavor=ts
// Run all the tests in the file concurrently using parallel workers.
test.describe.configure({ mode: 'parallel' });
test('runs in parallel 1', async ({ page }) => {});
test('runs in parallel 2', async ({ page }) => {});
```
```js js-flavor=js
// Annotate tests as inter-dependent.
test.describe.configure({ mode: 'serial' });
test('runs first', async ({ page }) => {});
test('runs second', async ({ page }) => {});
```
```js js-flavor=ts
// Annotate tests as inter-dependent.
test.describe.configure({ mode: 'serial' });
test('runs first', async ({ page }) => {});
test('runs second', async ({ page }) => {});
```
### option: Test.describe.configure.mode
- `mode` <"parallel"|"serial">
## method: Test.describe.only
Declares a focused group of tests. If there are some focused tests or suites, all of them will be run but nothing else.
@ -290,21 +330,19 @@ A callback that is run immediately when calling [`method: Test.describe.only`].
Declares a group of tests that could be run in parallel. By default, tests in a single test file run one after another, but using [`method: Test.describe.parallel`] allows them to run in parallel.
See [`method: Test.describe.configure`] for the preferred way of configuring the execution mode.
```js js-flavor=js
test.describe.parallel('group', () => {
test('runs in parallel 1', async ({ page }) => {
});
test('runs in parallel 2', async ({ page }) => {
});
test('runs in parallel 1', async ({ page }) => {});
test('runs in parallel 2', async ({ page }) => {});
});
```
```js js-flavor=ts
test.describe.parallel('group', () => {
test('runs in parallel 1', async ({ page }) => {
});
test('runs in parallel 2', async ({ page }) => {
});
test('runs in parallel 1', async ({ page }) => {});
test('runs in parallel 2', async ({ page }) => {});
});
```
@ -342,25 +380,23 @@ A callback that is run immediately when calling [`method: Test.describe.parallel
Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are skipped. All tests in a group are retried together.
See [`method: Test.describe.configure`] for the preferred way of configuring the execution mode.
:::note
Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently.
:::
```js js-flavor=js
test.describe.serial('group', () => {
test('runs first', async ({ page }) => {
});
test('runs second', async ({ page }) => {
});
test('runs first', async ({ page }) => {});
test('runs second', async ({ page }) => {});
});
```
```js js-flavor=ts
test.describe.serial('group', () => {
test('runs first', async ({ page }) => {
});
test('runs second', async ({ page }) => {
});
test('runs first', async ({ page }) => {});
test('runs second', async ({ page }) => {});
});
```

78
docs/src/test-auth-js.md
View File

@ -293,30 +293,30 @@ order to achieve that:
const { test } = require('@playwright/test');
test.describe.serial('use the same page', () => {
/** @type {import('@playwright/test').Page} */
let page;
test.describe.configure({ mode: 'serial' });
test.beforeAll(async ({ browser }) => {
// Create page yourself and sign in.
page = await browser.newPage();
await page.goto('https://github.com/login');
await page.fill('input[name="user"]', 'user');
await page.fill('input[name="password"]', 'password');
await page.click('text=Sign in');
});
/** @type {import('@playwright/test').Page} */
let page;
test.afterAll(async () => {
await page.close();
});
test.beforeAll(async ({ browser }) => {
// Create page yourself and sign in.
page = await browser.newPage();
await page.goto('https://github.com/login');
await page.fill('input[name="user"]', 'user');
await page.fill('input[name="password"]', 'password');
await page.click('text=Sign in');
});
test('first test', async () => {
// page is signed in.
});
test.afterAll(async () => {
await page.close();
});
test('second test', async () => {
// page is signed in.
});
test('first test', async () => {
// page is signed in.
});
test('second test', async () => {
// page is signed in.
});
```
@ -325,29 +325,29 @@ test.describe.serial('use the same page', () => {
import { test, Page } from '@playwright/test';
test.describe.serial('use the same page', () => {
let page: Page;
test.describe.configure({ mode: 'serial' });
test.beforeAll(async ({ browser }) => {
// Create page once and sign in.
page = await browser.newPage();
await page.goto('https://github.com/login');
await page.fill('input[name="user"]', 'user');
await page.fill('input[name="password"]', 'password');
await page.click('text=Sign in');
});
let page: Page;
test.afterAll(async () => {
await page.close();
});
test.beforeAll(async ({ browser }) => {
// Create page once and sign in.
page = await browser.newPage();
await page.goto('https://github.com/login');
await page.fill('input[name="user"]', 'user');
await page.fill('input[name="password"]', 'password');
await page.click('text=Sign in');
});
test('first test', async () => {
// page is signed in.
});
test.afterAll(async () => {
await page.close();
});
test('second test', async () => {
// page is signed in.
});
test('first test', async () => {
// page is signed in.
});
test('second test', async () => {
// page is signed in.
});
```

77
docs/src/test-parallel-js.md
View File

@ -72,18 +72,81 @@ Note that parallel tests are executed in separate worker processes and cannot sh
```js js-flavor=js
const { test } = require('@playwright/test');
test.describe.parallel('suite', () => {
test('runs in parallel 1', async ({ page }) => { /* ... */ });
test('runs in parallel 2', async ({ page }) => { /* ... */ });
});
test.describe.configure({ mode: 'parallel' });
test('runs in parallel 1', async ({ page }) => { /* ... */ });
test('runs in parallel 2', async ({ page }) => { /* ... */ });
```
```js js-flavor=ts
import { test } from '@playwright/test';
test.describe.parallel('suite', () => {
test('runs in parallel 1', async ({ page }) => { /* ... */ });
test('runs in parallel 2', async ({ page }) => { /* ... */ });
test.describe.configure({ mode: 'parallel' });
test('runs in parallel 1', async ({ page }) => { /* ... */ });
test('runs in parallel 2', async ({ page }) => { /* ... */ });
```
## Serial mode
You can annotate inter-dependent tests as serial. If one of the serial tests
fails, all subsequent tests are skipped. All tests in a group are retried together.
:::note
Using serial is not recommended. It is usually better to make your tests isolated, so they can be run independently.
:::
```js js-flavor=js
// @ts-check
const { test } = require('@playwright/test');
test.describe.configure({ mode: 'serial' });
/** @type {import('@playwright/test').Page} */
let page;
test.beforeAll(async ({ browser }) => {
page = await browser.newPage();
});
test.afterAll(async () => {
await page.close();
});
test('runs first', async () => {
await page.goto('https://playwright.dev/');
});
test('runs second', async () => {
await page.click('text=Get Started');
});
```
```js js-flavor=ts
// example.spec.ts
import { test, Page } from '@playwright/test';
// Annotate entire file as serial.
test.describe.configure({ mode: 'serial' });
let page: Page;
test.beforeAll(async ({ browser }) => {
page = await browser.newPage();
});
test.afterAll(async () => {
await page.close();
});
test('runs first', async () => {
await page.goto('https://playwright.dev/');
});
test('runs second', async () => {
await page.click('text=Get Started');
});
```

82
docs/src/test-retries-js.md
View File

@ -143,23 +143,23 @@ Consider the following snippet that uses `test.describe.serial`:
```js js-flavor=js
const { test } = require('@playwright/test');
test.describe.serial('suite', () => {
test.beforeAll(async () => { /* ... */ });
test('first good', async ({ page }) => { /* ... */ });
test('second flaky', async ({ page }) => { /* ... */ });
test('third good', async ({ page }) => { /* ... */ });
});
test.describe.configure({ mode: 'serial' });
test.beforeAll(async () => { /* ... */ });
test('first good', async ({ page }) => { /* ... */ });
test('second flaky', async ({ page }) => { /* ... */ });
test('third good', async ({ page }) => { /* ... */ });
```
```js js-flavor=ts
import { test } from '@playwright/test';
test.describe.serial('suite', () => {
test.beforeAll(async () => { /* ... */ });
test('first good', async ({ page }) => { /* ... */ });
test('second flaky', async ({ page }) => { /* ... */ });
test('third good', async ({ page }) => { /* ... */ });
});
test.describe.configure({ mode: 'serial' });
test.beforeAll(async () => { /* ... */ });
test('first good', async ({ page }) => { /* ... */ });
test('second flaky', async ({ page }) => { /* ... */ });
test('third good', async ({ page }) => { /* ... */ });
```
When running without [retries](#retries), all tests after the failure are skipped:
@ -195,25 +195,25 @@ Playwright Test creates an isolated [Page] object for each test. However, if you
const { test } = require('@playwright/test');
test.describe.serial('use the same page', () => {
/** @type {import('@playwright/test').Page} */
let page;
test.describe.configure({ mode: 'serial' });
test.beforeAll(async ({ browser }) => {
page = await browser.newPage();
});
/** @type {import('@playwright/test').Page} */
let page;
test.afterAll(async () => {
await page.close();
});
test.beforeAll(async ({ browser }) => {
page = await browser.newPage();
});
test('runs first', async () => {
await page.goto('https://playwright.dev/');
});
test.afterAll(async () => {
await page.close();
});
test('runs second', async () => {
await page.click('text=Get Started');
});
test('runs first', async () => {
await page.goto('https://playwright.dev/');
});
test('runs second', async () => {
await page.click('text=Get Started');
});
```
@ -222,23 +222,23 @@ test.describe.serial('use the same page', () => {
import { test, Page } from '@playwright/test';
test.describe.serial('use the same page', () => {
let page: Page;
test.describe.configure({ mode: 'serial' });
test.beforeAll(async ({ browser }) => {
page = await browser.newPage();
});
let page: Page;
test.afterAll(async () => {
await page.close();
});
test.beforeAll(async ({ browser }) => {
page = await browser.newPage();
});
test('runs first', async () => {
await page.goto('https://playwright.dev/');
});
test.afterAll(async () => {
await page.close();
});
test('runs second', async () => {
await page.click('text=Get Started');
});
test('runs first', async () => {
await page.goto('https://playwright.dev/');
});
test('runs second', async () => {
await page.click('text=Get Started');
});
```

21
packages/playwright-test/src/testType.ts
View File

@ -36,6 +36,7 @@ export class TestTypeImpl {
test.only = wrapFunctionWithLocation(this._createTest.bind(this, 'only'));
test.describe = wrapFunctionWithLocation(this._describe.bind(this, 'default'));
test.describe.only = wrapFunctionWithLocation(this._describe.bind(this, 'only'));
test.describe.configure = wrapFunctionWithLocation(this._configure.bind(this));
test.describe.parallel = wrapFunctionWithLocation(this._describe.bind(this, 'parallel'));
test.describe.parallel.only = wrapFunctionWithLocation(this._describe.bind(this, 'parallel.only'));
test.describe.serial = wrapFunctionWithLocation(this._describe.bind(this, 'serial'));
@ -133,6 +134,26 @@ export class TestTypeImpl {
}
}
private _configure(location: Location, options: { mode?: 'parallel' | 'serial' }) {
throwIfRunningInsideJest();
const suite = currentlyLoadingFileSuite();
if (!suite)
throw errorWithLocation(location, `describe.configure() can only be called in a test file`);
if (!options.mode)
return;
if (suite._parallelMode !== 'default')
throw errorWithLocation(location, 'Parallel mode is already assigned for the enclosing scope.');
suite._parallelMode = options.mode;
for (let parent: Suite | undefined = suite.parent; parent; parent = parent.parent) {
if (parent._parallelMode === 'serial' && suite._parallelMode === 'parallel')
throw errorWithLocation(location, 'describe.parallel cannot be nested inside describe.serial');
if (parent._parallelMode === 'parallel' && suite._parallelMode === 'serial')
throw errorWithLocation(location, 'describe.serial cannot be nested inside describe.parallel');
}
}
private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, ...modifierArgs: [arg?: any | Function, description?: string]) {
const suite = currentlyLoadingFileSuite();
if (suite) {

41
packages/playwright-test/types/test.d.ts vendored
View File

@ -1640,15 +1640,16 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
* Declares a group of tests that should always be run serially. If one of the tests fails, all subsequent tests are
* skipped. All tests in a group are retried together.
*
* See [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure) for the
* preferred way of configuring the execution mode.
*
* > NOTE: Using serial is not recommended. It is usually better to make your tests isolated, so they can be run
* independently.
*
* ```ts
* test.describe.serial('group', () => {
* test('runs first', async ({ page }) => {
* });
* test('runs second', async ({ page }) => {
* });
* test('runs first', async ({ page }) => {});
* test('runs second', async ({ page }) => {});
* });
* ```
*
@ -1685,12 +1686,13 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
* but using [test.describe.parallel(title, callback)](https://playwright.dev/docs/api/class-test#test-describe-parallel)
* allows them to run in parallel.
*
* See [test.describe.configure([options])](https://playwright.dev/docs/api/class-test#test-describe-configure) for the
* preferred way of configuring the execution mode.
*
* ```ts
* test.describe.parallel('group', () => {
* test('runs in parallel 1', async ({ page }) => {
* });
* test('runs in parallel 2', async ({ page }) => {
* });
* test('runs in parallel 1', async ({ page }) => {});
* test('runs in parallel 2', async ({ page }) => {});
* });
* ```
*
@ -1711,6 +1713,29 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
*/
only: SuiteFunction;
};
/**
* Set execution mode of execution for the enclosing scope. Can be executed either on the top level or inside a describe.
* Configuration applies to the entire scope, regardless of whether it run before or after the test declaration.
*
* Learn more about the execution modes [here](https://playwright.dev/docs/test-parallel-js).
*
* ```ts
* // Run all the tests in the file concurrently using parallel workers.
* test.describe.configure({ mode: 'parallel' });
* test('runs in parallel 1', async ({ page }) => {});
* test('runs in parallel 2', async ({ page }) => {});
* ```
*
* ```ts
* // Annotate tests as inter-dependent.
* test.describe.configure({ mode: 'serial' });
* test('runs first', async ({ page }) => {});
* test('runs second', async ({ page }) => {});
* ```
*
* @param options
*/
configure: (options: { mode?: 'parallel' | 'serial' }) => void;
};
/**
* Declares a skipped test, similarly to

58
tests/playwright-test/test-parallel.spec.ts
View File

@ -58,3 +58,61 @@ test('test.describe.parallel should work', async ({ runInlineTest }) => {
expect(result.output).toContain('%% worker=1');
expect(result.output).toContain('%% worker=2');
});
test('test.describe.parallel should work in file', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test.describe.configure({ mode: 'parallel' });
test('test1', async ({}, testInfo) => {
console.log('\\n%% worker=' + testInfo.workerIndex);
await new Promise(f => setTimeout(f, 1000));
});
test('test2', async ({}, testInfo) => {
console.log('\\n%% worker=' + testInfo.workerIndex);
await new Promise(f => setTimeout(f, 1000));
});
test.describe('inner suite', () => {
test('test3', async ({}, testInfo) => {
console.log('\\n%% worker=' + testInfo.workerIndex);
await new Promise(f => setTimeout(f, 1000));
});
});
`,
}, { workers: 3 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.output).toContain('%% worker=0');
expect(result.output).toContain('%% worker=1');
expect(result.output).toContain('%% worker=2');
});
test('test.describe.parallel should work in describe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test.describe('parallel suite', () => {
test.describe.configure({ mode: 'parallel' });
test('test1', async ({}, testInfo) => {
console.log('\\n%% worker=' + testInfo.workerIndex);
await new Promise(f => setTimeout(f, 1000));
});
test('test2', async ({}, testInfo) => {
console.log('\\n%% worker=' + testInfo.workerIndex);
await new Promise(f => setTimeout(f, 1000));
});
test.describe('inner suite', () => {
test('test3', async ({}, testInfo) => {
console.log('\\n%% worker=' + testInfo.workerIndex);
await new Promise(f => setTimeout(f, 1000));
});
});
});
`,
}, { workers: 3 });
expect(result.exitCode).toBe(0);
expect(result.passed).toBe(3);
expect(result.output).toContain('%% worker=0');
expect(result.output).toContain('%% worker=1');
expect(result.output).toContain('%% worker=2');
});

40
tests/playwright-test/test-serial.spec.ts
View File

@ -55,6 +55,46 @@ test('test.describe.serial should work', async ({ runInlineTest }) => {
]);
});
test('test.describe.serial should work in describe', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
const { test } = pwt;
test.describe('serial suite', () => {
test.describe.configure({ mode: 'serial' });
test('test1', async ({}) => {
console.log('\\n%%test1');
});
test('test2', async ({}) => {
console.log('\\n%%test2');
});
test.describe('inner suite', () => {
test('test3', async ({}) => {
console.log('\\n%%test3');
expect(1).toBe(2);
});
test('test4', async ({}) => {
console.log('\\n%%test4');
});
});
test('test5', async ({}) => {
console.log('\\n%%test5');
});
});
`,
});
expect(result.exitCode).toBe(1);
expect(result.passed).toBe(2);
expect(result.failed).toBe(1);
expect(result.skipped).toBe(2);
expect(result.output.split('\n').filter(line => line.startsWith('%%'))).toEqual([
'%%test1',
'%%test2',
'%%test3',
]);
});
test('test.describe.serial should work with retry', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `

1
utils/generate_types/overrides-test.d.ts vendored
View File

@ -244,6 +244,7 @@ export interface TestType<TestArgs extends KeyValue, WorkerArgs extends KeyValue
parallel: SuiteFunction & {
only: SuiteFunction;
};
configure: (options: { mode?: 'parallel' | 'serial' }) => void;
};
skip(title: string, testFunction: (args: TestArgs & WorkerArgs, testInfo: TestInfo) => Promise<void> | void): void;
skip(): void;

10
utils/generate_types/parseOverrides.js
View File

@ -101,8 +101,8 @@ async function parseOverrides(filePath, commentForClass, commentForMethod, extra
* @param {ts.Node} node
*/
function visitProperties(className, prefix, node) {
// This function supports structs like "a: { b: string; c: number }"
// and inserts comments for "a.b" and "a.c"
// This function supports structs like "a: { b: string; c: number, (): void }"
// and inserts comments for "a.b", "a.c", a.
if (ts.isPropertySignature(node)) {
const name = checker.getSymbolAtLocation(node.name).getName();
const pos = node.getStart(file, false);
@ -111,6 +111,12 @@ async function parseOverrides(filePath, commentForClass, commentForMethod, extra
text: commentForMethod(className, `${prefix}.${name}`, 0),
});
ts.forEachChild(node, child => visitProperties(className, `${prefix}.${name}`, child));
} else if (ts.isCallSignatureDeclaration(node)) {
const pos = node.getStart(file, false);
replacers.push({
pos,
text: commentForMethod(className, `${prefix}`, 0),
});
} else if (!ts.isMethodSignature(node)) {
ts.forEachChild(node, child => visitProperties(className, prefix, child));
}