yujunjun/playwright
1
0
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-06-26 21:40:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "feat(test runner): shuffle order of tests with sharding seed … (#31260)

Browse Source 
…(#30817)"
This commit is contained in:
Pavel Feldman 2024-06-11 16:05:35 -07:00 committed by GitHub
parent 732e7393d3
commit cf400a6080
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 0 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
docs/src/test-api/class-testconfig.md
View File

@ -482,27 +482,6 @@ export default defineConfig({
```
## property: TestConfig.shardingSeed
* since: v1.45
- type: ?<[string]>
Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests in a deterministic way can help to distribute the load more evenly.
The sharding seed is a string that is used to initialize a random number generator.
Learn more about [parallelism and sharding](../test-parallel.md) with Playwright Test.
**Usage**
```js title="playwright.config.ts"
import { defineConfig } from '@playwright/test';
export default defineConfig({
shardingSeed: 'string value'
});
```
## property: TestConfig.testDir
* since: v1.10
- type: ?<[string]>

6
docs/src/test-sharding-js.md
View File

@ -22,12 +22,6 @@ Now, if you run these shards in parallel on different computers, your test suite
Note that Playwright can only shard tests that can be run in parallel. By default, this means Playwright will shard test files. Learn about other options in the [parallelism guide](./test-parallel.md).
## Randomizing test order in a deterministic way
By default tests are run in the order they are discovered, which is mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. For example, if the first half of your tests are slower than the rest of your tests and you are using 4 shards it means that shard 1 and 2 will take significantly more time then shard 3 and 4.
To aid with this problem you can pass `--sharding-seed=string-value` to randomize the order of tests in a deterministic way, which could yield better distribution of slow and fast tests across all shards.
## Merging reports from multiple shards
In the previous example, each test shard has its own test report. If you want to have a combined report showing all the test results from all the shards, you can merge them.

2
packages/playwright/src/common/config.ts
View File

@ -55,7 +55,6 @@ export class FullConfigInternal {
cliFailOnFlakyTests?: boolean;
testIdMatcher?: Matcher;
defineConfigWasUsed = false;
shardingSeed: string | null;
constructor(location: ConfigLocation, userConfig: Config, configCLIOverrides: ConfigCLIOverrides) {
if (configCLIOverrides.projects && userConfig.projects)
@ -93,7 +92,6 @@ export class FullConfigInternal {
workers: 0,
webServer: null,
};
this.shardingSeed = takeFirst(configCLIOverrides.shardingSeed, userConfig.shardingSeed, null);
for (const key in userConfig) {
if (key.startsWith('@'))
(this.config as any)[key] = (userConfig as any)[key];

1
packages/playwright/src/common/ipc.ts
View File

@ -32,7 +32,6 @@ export type ConfigCLIOverrides = {
reporter?: ReporterDescription[];
additionalReporters?: ReporterDescription[];
shard?: { current: number, total: number };
shardingSeed?: string;
timeout?: number;
ignoreSnapshots?: boolean;
updateSnapshots?: 'all'|'none'|'missing';

2
packages/playwright/src/program.ts
View File

@ -281,7 +281,6 @@ function overridesFromOptions(options: { [key: string]: any }): ConfigCLIOverrid
retries: options.retries ? parseInt(options.retries, 10) : undefined,
reporter: resolveReporterOption(options.reporter),
shard: shardPair ? { current: shardPair[0], total: shardPair[1] } : undefined,
shardingSeed: options.shardingSeed ? options.shardingSeed : undefined,
timeout: options.timeout ? parseInt(options.timeout, 10) : undefined,
ignoreSnapshots: options.ignoreSnapshots ? !!options.ignoreSnapshots : undefined,
updateSnapshots: options.updateSnapshots ? 'all' as const : undefined,
@ -359,7 +358,6 @@ const testOptions: [string, string][] = [
['--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${defaultReporter}")`],
['--retries <retries>', `Maximum retry count for flaky tests, zero for no retries (default: no retries)`],
['--shard <shard>', `Shard tests and execute only the selected shard, specify in the form "current/all", 1-based, for example "3/5"`],
['--sharding-seed <seed>', `Seed string for randomizing the test order before sharding. Defaults to not randomizing the order.`],
['--timeout <timeout>', `Specify test timeout threshold in milliseconds, zero for unlimited (default: ${defaultTimeout})`],
['--trace <mode>', `Force tracing mode, can be ${kTraceModes.map(mode => `"${mode}"`).join(', ')}`],
['--ui', `Run tests in interactive UI mode`],

4
packages/playwright/src/runner/loadUtils.ts
View File

@ -31,7 +31,6 @@ import { createTestGroups, filterForShard, type TestGroup } from './testGroups';
import { dependenciesForTestFile } from '../transform/compilationCache';
import { sourceMapSupport } from '../utilsBundle';
import type { RawSourceMap } from 'source-map';
import { shuffleWithSeed } from './shuffle';
export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher?: Matcher) {
const config = testRun.config;
@ -180,9 +179,6 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
for (const projectSuite of rootSuite.suites)
testGroups.push(...createTestGroups(projectSuite, config.config.workers));
if (config.shardingSeed)
shuffleWithSeed(testGroups, config.shardingSeed);
// Shard test groups.
const testGroupsInThisShard = filterForShard(config.config.shard, testGroups);
const testsInThisShard = new Set<TestCase>();

59
packages/playwright/src/runner/shuffle.ts
View File

@ -1,59 +0,0 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Shuffles the given array of items using the given seed.
*
* @param items The array of items to shuffle.
* @param seed The seed to use for shuffling.
*/
export function shuffleWithSeed(items: any[], seed: string): void {
const random = rng(cyrb32(seed));
for (let i = items.length - 1; i > 0; i--) {
const j = Math.floor(random() * (i + 1));
[items[i], items[j]] = [items[j], items[i]];
}
}
/**
* Returns a random number generator seeded with the given seed.
*
* @param seed The seed for the random number generator.
* @returns The random number generator.
*/
function rng(seed: number) {
const m = 2 ** 35 - 31;
const a = 185852;
let s = seed % m;
return function() {
return (s = s * a % m) / m;
};
}
/**
* Return a 32-bit hash from a string.
*
* @param str The string to hash.
* @returns The 32-bit hash.
*/
function cyrb32(str: string) {
let h = 0x2323;
for (let i = 0; i < str.length; i++) {
h = h ^ str.charCodeAt(i);
h = Math.imul(h, 2654435761);
}
return h >>> 0;
}

23
packages/playwright/types/test.d.ts vendored
View File

@ -1426,29 +1426,6 @@ interface TestConfig<TestArgs = {}, WorkerArgs = {}> {
total: number;
};
/**
* Shuffle the order of test groups with a seed. By default tests are run in the order they are discovered, which is
* mostly alphabetical. This could lead to an uneven distribution of slow and fast tests. Shuffling the order of tests
* in a deterministic way can help to distribute the load more evenly.
*
* The sharding seed is a string that is used to initialize a random number generator.
*
* Learn more about [parallelism and sharding](https://playwright.dev/docs/test-parallel) with Playwright Test.
*
* **Usage**
*
* ```js
* // playwright.config.ts
* import { defineConfig } from '@playwright/test';
*
* export default defineConfig({
* shardingSeed: 'string value'
* });
* ```
*
*/
shardingSeed?: string;
/**
* **NOTE** Use
* [testConfig.snapshotPathTemplate](https://playwright.dev/docs/api/class-testconfig#test-config-snapshot-path-template)