mirror of
https://github.com/microsoft/playwright.git
synced 2025-06-26 21:40:17 +00:00
367 lines
12 KiB
TypeScript
367 lines
12 KiB
TypeScript
/**
|
|
* Copyright (c) Microsoft Corporation.
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
import type { PlaywrightTestConfig, TestInfo, PlaywrightTestProject } from '@playwright/test';
|
|
import path from 'path';
|
|
import { test, expect } from './playwright-test-fixtures';
|
|
|
|
function createConfigWithProjects(names: string[], testInfo: TestInfo, groups: PlaywrightTestConfig['groups'], projectTemplates?: { [name: string]: PlaywrightTestProject }): Record<string, string> {
|
|
const config: PlaywrightTestConfig = {
|
|
projects: names.map(name => ({ ...projectTemplates?.[name], name, testDir: testInfo.outputPath(name) })),
|
|
groups
|
|
};
|
|
const files = {};
|
|
for (const name of names) {
|
|
files[`${name}/${name}.spec.ts`] = `
|
|
const { test } = pwt;
|
|
test('${name} test', async () => {
|
|
await new Promise(f => setTimeout(f, 100));
|
|
});`;
|
|
}
|
|
function replacer(key, value) {
|
|
if (value instanceof RegExp)
|
|
return `RegExp(${value.toString()})`;
|
|
else
|
|
return value;
|
|
}
|
|
files['playwright.config.ts'] = `
|
|
import * as path from 'path';
|
|
module.exports = ${JSON.stringify(config, replacer, 2)};
|
|
`.replace(/"RegExp\((.*)\)"/g, '$1');
|
|
return files;
|
|
}
|
|
|
|
type Timeline = { titlePath: string[], event: 'begin' | 'end' }[];
|
|
|
|
function formatTimeline(timeline: Timeline) {
|
|
return timeline.map(e => `${e.titlePath.slice(1).join(' > ')} [${e.event}]`).join('\n');
|
|
}
|
|
|
|
function projectNames(timeline: Timeline) {
|
|
const projectNames = Array.from(new Set(timeline.map(({ titlePath }) => titlePath[1])).keys());
|
|
projectNames.sort();
|
|
return projectNames;
|
|
}
|
|
|
|
function expectRunBefore(timeline: Timeline, before: string[], after: string[]) {
|
|
const begin = new Map<string, number>();
|
|
const end = new Map<string, number>();
|
|
for (let i = 0; i < timeline.length; i++) {
|
|
const projectName = timeline[i].titlePath[1];
|
|
const map = timeline[i].event === 'begin' ? begin : end;
|
|
const oldIndex = map.get(projectName) ?? i;
|
|
const newIndex = (timeline[i].event === 'begin') ? Math.min(i, oldIndex) : Math.max(i, oldIndex);
|
|
map.set(projectName, newIndex);
|
|
}
|
|
for (const b of before) {
|
|
for (const a of after) {
|
|
const bEnd = end.get(b) as number;
|
|
expect(bEnd === undefined, `Unknown project ${b}`).toBeFalsy();
|
|
const aBegin = begin.get(a) as number;
|
|
expect(aBegin === undefined, `Unknown project ${a}`).toBeFalsy();
|
|
if (bEnd < aBegin)
|
|
continue;
|
|
throw new Error(`Project '${b}' expected to finish before '${a}'\nTest run order was:\n${formatTimeline(timeline)}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
test('should work', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: ['a']
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(1);
|
|
expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
|
|
a > a${path.sep}a.spec.ts > a test [end]`);
|
|
});
|
|
|
|
test('should order two projects', async ({ runGroups }, testInfo) => {
|
|
await test.step(`order a then b`, async () => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
'a',
|
|
'b'
|
|
]
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(2);
|
|
expect(formatTimeline(timeline)).toEqual(`a > a${path.sep}a.spec.ts > a test [begin]
|
|
a > a${path.sep}a.spec.ts > a test [end]
|
|
b > b${path.sep}b.spec.ts > b test [begin]
|
|
b > b${path.sep}b.spec.ts > b test [end]`);
|
|
});
|
|
await test.step(`order b then a`, async () => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
'b',
|
|
'a'
|
|
]
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(2);
|
|
expect(formatTimeline(timeline)).toEqual(`b > b${path.sep}b.spec.ts > b test [begin]
|
|
b > b${path.sep}b.spec.ts > b test [end]
|
|
a > a${path.sep}a.spec.ts > a test [begin]
|
|
a > a${path.sep}a.spec.ts > a test [end]`);
|
|
});
|
|
});
|
|
|
|
test('should order 1-3-1 projects', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
'e',
|
|
['d', 'c', 'b'],
|
|
'a',
|
|
]
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expectRunBefore(timeline, ['e'], ['d', 'c', 'b']);
|
|
expectRunBefore(timeline, ['d', 'c', 'b'], ['a']);
|
|
expect(passed).toBe(5);
|
|
});
|
|
|
|
test('should order 2-2-2 projects', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
['a', 'b'],
|
|
['d', 'c'],
|
|
['e', 'f'],
|
|
]
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expectRunBefore(timeline, ['a', 'b'], ['c', 'd']);
|
|
expectRunBefore(timeline, ['c', 'd'], ['e', 'f']);
|
|
expect(passed).toBe(6);
|
|
});
|
|
|
|
test('should run parallel groups sequentially without overlaps', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
['a', 'b', 'c', 'd'],
|
|
['a', 'b', 'c', 'd'],
|
|
['a', 'b', 'c', 'd'],
|
|
]
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
|
|
const expectedEndOfFirstPhase = events => {
|
|
const firstProjectEndIndex = project => events.findIndex(e => e.event === 'end' && e.titlePath[1] === project);
|
|
return Math.max(...['a', 'b', 'c', 'd'].map(firstProjectEndIndex));
|
|
};
|
|
const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
|
|
|
|
let remainingTimeline = timeline;
|
|
for (let i = 0; i < 3; i++) {
|
|
const phaseEndIndex = expectedEndOfFirstPhase(remainingTimeline);
|
|
const firstPhase = formatPhaseEvents(remainingTimeline.slice(0, phaseEndIndex + 1));
|
|
firstPhase.sort();
|
|
expect(firstPhase, `check phase ${i}`).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end', 'c:begin', 'c:end', 'd:begin', 'd:end']);
|
|
remainingTimeline = remainingTimeline.slice(phaseEndIndex + 1);
|
|
}
|
|
expect(remainingTimeline.length).toBe(0);
|
|
|
|
expect(passed).toBe(12);
|
|
});
|
|
|
|
test('should support phase with multiple project names', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
[
|
|
{ project: ['a', 'b', 'c'] }
|
|
],
|
|
[
|
|
{ project: ['d'] },
|
|
{ project: ['e', 'f'] }
|
|
],
|
|
]
|
|
});
|
|
|
|
const { exitCode, passed } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(6);
|
|
});
|
|
|
|
test('should support varios syntax', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
'a',
|
|
['a', 'b'],
|
|
[
|
|
{ project: ['a', 'b'] }
|
|
],
|
|
[
|
|
{ project: ['a', 'b'] },
|
|
'c',
|
|
{ project: 'd' },
|
|
],
|
|
[{ project: 'e' }],
|
|
'f'
|
|
]
|
|
});
|
|
const { exitCode, passed } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(11);
|
|
});
|
|
|
|
test('should support --group option', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
'a', 'b'
|
|
],
|
|
foo: [
|
|
['b', 'c']
|
|
],
|
|
bar: [
|
|
'd', 'e'
|
|
]
|
|
});
|
|
const formatPhaseEvents = events => events.map(e => e.titlePath[1] + ':' + e.event);
|
|
{
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'default' });
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(2);
|
|
expect(formatPhaseEvents(timeline)).toEqual(['a:begin', 'a:end', 'b:begin', 'b:end']);
|
|
}
|
|
{
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'foo' });
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(2);
|
|
const formatted = formatPhaseEvents(timeline);
|
|
formatted.sort();
|
|
expect(formatted).toEqual(['b:begin', 'b:end', 'c:begin', 'c:end']);
|
|
}
|
|
{
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles, { group: 'bar' });
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(2);
|
|
expect(formatPhaseEvents(timeline)).toEqual(['d:begin', 'd:end', 'e:begin', 'e:end']);
|
|
}
|
|
});
|
|
|
|
test('should throw when unknown --group is passed', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
'a', 'b'
|
|
],
|
|
foo: [
|
|
['b', 'c']
|
|
]
|
|
});
|
|
const { exitCode, output } = await runGroups(configWithFiles, { group: 'bar' });
|
|
expect(exitCode).toBe(1);
|
|
expect(output).toContain(`Cannot find project group 'bar' in the config`);
|
|
});
|
|
|
|
test('should support testMatch and testIgnore', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
[
|
|
{ project: ['a', 'b'], testMatch: ['**/a.spec.ts'] },
|
|
{ project: ['c', 'd'], testMatch: [/.*c.spec.ts/, '**/*d*'] },
|
|
{ project: ['e'], testIgnore: [/.*e.spec.ts/] },
|
|
{ project: ['f'], testMatch: /does not match/ },
|
|
],
|
|
]
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(3);
|
|
expect(projectNames(timeline)).toEqual(['a', 'c', 'd']);
|
|
});
|
|
|
|
test('should support grep and grepInvert', async ({ runGroups }, testInfo) => {
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
[
|
|
{ project: ['a', 'b'], grep: /.*a test/ },
|
|
{ project: ['c', 'd'], grepInvert: [/.*c test/] },
|
|
{ project: ['e', 'f'], grep: /.*(e|f) test/, grepInvert: [/.*f test/] },
|
|
],
|
|
]
|
|
});
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(3);
|
|
expect(projectNames(timeline)).toEqual(['a', 'd', 'e']);
|
|
});
|
|
|
|
test('should intercect gpoup and project level grep and grepInvert', async ({ runGroups }, testInfo) => {
|
|
const projectTemplates = {
|
|
'a': {
|
|
grep: /a test/,
|
|
grepInvert: [/no test/],
|
|
},
|
|
'b': {
|
|
grep: /.*b te.*/,
|
|
grepInvert: [/.*a test/],
|
|
},
|
|
'c': {
|
|
grepInvert: [/.*test/],
|
|
},
|
|
'd': {
|
|
grep: [/.*unkwnown test/],
|
|
},
|
|
};
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
[
|
|
{ project: ['a', 'b', 'c', 'd', 'e'], grep: /.*(b|c|d|e) test/, grepInvert: /.*d test/ },
|
|
],
|
|
]
|
|
}, projectTemplates);
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(2);
|
|
expect(projectNames(timeline)).toEqual(['b', 'e']);
|
|
});
|
|
|
|
test('should intercect gpoup and project level testMatch and testIgnore', async ({ runGroups }, testInfo) => {
|
|
const projectTemplates = {
|
|
'a': {
|
|
testMatch: /.*a.spec.ts/,
|
|
testIgnore: [/no test/],
|
|
},
|
|
'b': {
|
|
testMatch: '**/b.spec.ts',
|
|
testIgnore: [/.*a.spec.ts/],
|
|
},
|
|
'c': {
|
|
testIgnore: [/.*no-match.spec.ts/],
|
|
},
|
|
'd': {
|
|
testMatch: [/.*unkwnown/],
|
|
},
|
|
};
|
|
const configWithFiles = createConfigWithProjects(['a', 'b', 'c', 'd', 'e', 'f'], testInfo, {
|
|
default: [
|
|
[
|
|
{ project: ['a', 'b', 'c', 'd'], testMatch: /.*(b|c|d).spec.ts/, testIgnore: /.*c.spec.ts/ },
|
|
{ project: ['c', 'd', 'e', 'f'], testIgnore: /.*[^ef].spec.ts/ },
|
|
],
|
|
]
|
|
}, projectTemplates);
|
|
const { exitCode, passed, timeline } = await runGroups(configWithFiles);
|
|
expect(exitCode).toBe(0);
|
|
expect(passed).toBe(3);
|
|
expect(projectNames(timeline)).toEqual(['b', 'e', 'f']);
|
|
});
|