From cb9ace60350e08b5cb90596c001f20258f23f075 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Wed, 1 Feb 2023 15:25:26 -0800 Subject: [PATCH] chore(test): move run options into config (#20568) --- docs/src/test-api/class-testproject.md | 5 +- packages/playwright-test/src/cli.ts | 20 +++--- .../src/common/configLoader.ts | 68 +++++++++++-------- .../playwright-test/src/common/globals.ts | 2 +- packages/playwright-test/src/common/ipc.ts | 4 +- .../playwright-test/src/common/poolBuilder.ts | 2 +- .../playwright-test/src/common/suiteUtils.ts | 6 +- .../playwright-test/src/common/testInfo.ts | 6 +- packages/playwright-test/src/common/types.ts | 45 ++++++++---- .../src/matchers/toMatchSnapshot.ts | 8 +-- .../src/plugins/webServerPlugin.ts | 2 +- .../playwright-test/src/reporters/base.ts | 2 +- .../playwright-test/src/reporters/html.ts | 4 +- .../playwright-test/src/runner/loadUtils.ts | 35 ++++------ .../src/runner/projectUtils.ts | 17 +++-- packages/playwright-test/src/runner/runner.ts | 23 ++----- packages/playwright-test/src/runner/tasks.ts | 32 +++------ .../playwright-test/src/worker/workerMain.ts | 2 +- packages/playwright-test/types/test.d.ts | 10 ++- 19 files changed, 150 insertions(+), 143 deletions(-) diff --git a/docs/src/test-api/class-testproject.md b/docs/src/test-api/class-testproject.md index 9432217e8d..503cc1d266 100644 --- a/docs/src/test-api/class-testproject.md +++ b/docs/src/test-api/class-testproject.md @@ -90,7 +90,10 @@ export default defineConfig({ - type: ?<[Array]<[string]>> List of projects that need to run before any test in this project runs. Dependencies can -be useful for configuring the global setup actions in a way that every action is a test. +be useful for configuring the global setup actions in a way that every action is +in a form of a test. That way one can record traces and other artifacts for the +global setup routine, see the setup steps in the test report, etc. + For example: ```js diff --git a/packages/playwright-test/src/cli.ts b/packages/playwright-test/src/cli.ts index 807dbab8b6..8b7b94e1d3 100644 --- a/packages/playwright-test/src/cli.ts +++ b/packages/playwright-test/src/cli.ts @@ -22,7 +22,6 @@ import path from 'path'; import { Runner } from './runner/runner'; import { stopProfiling, startProfiling } from './common/profiler'; import { experimentalLoaderOption, fileIsModule } from './util'; -import type { TestFileFilter } from './util'; import { createTitleMatcher } from './util'; import { showHTMLReport } from './reporters/html'; import { baseFullConfig, builtInReporters, ConfigLoader, defaultTimeout, kDefaultConfigFiles, resolveConfigFile } from './common/configLoader'; @@ -155,9 +154,9 @@ async function runTests(args: string[], opts: { [key: string]: any }) { await configLoader.loadConfigFile(resolvedConfigFile); else await configLoader.loadEmptyConfig(configFileOrDirectory); - const runner = new Runner(configLoader.fullConfig()); - const testFileFilters: TestFileFilter[] = args.map(arg => { + const config = configLoader.fullConfig(); + config._internal.testFileFilters = args.map(arg => { const match = /^(.*?):(\d+):?(\d+)?$/.exec(arg); return { re: forceRegExp(match ? match[1] : arg), @@ -165,18 +164,15 @@ async function runTests(args: string[], opts: { [key: string]: any }) { column: match?.[3] ? parseInt(match[3], 10) : null, }; }); - const grepMatcher = opts.grep ? createTitleMatcher(forceRegExp(opts.grep)) : () => true; const grepInvertMatcher = opts.grepInvert ? createTitleMatcher(forceRegExp(opts.grepInvert)) : () => false; - const testTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title); + config._internal.testTitleMatcher = (title: string) => !grepInvertMatcher(title) && grepMatcher(title); + config._internal.listOnly = !!opts.list; + config._internal.projectFilter = opts.project || undefined; + config._internal.passWithNoTests = !!opts.passWithNoTests; - const status = await runner.runAllTests({ - listOnly: !!opts.list, - testFileFilters, - testTitleMatcher, - projectFilter: opts.project || undefined, - passWithNoTests: opts.passWithNoTests, - }); + const runner = new Runner(config); + const status = await runner.runAllTests(); await stopProfiling(undefined); if (status === 'interrupted') diff --git a/packages/playwright-test/src/common/configLoader.ts b/packages/playwright-test/src/common/configLoader.ts index 9687d71942..8611a85223 100644 --- a/packages/playwright-test/src/common/configLoader.ts +++ b/packages/playwright-test/src/common/configLoader.ts @@ -30,7 +30,7 @@ export class ConfigLoader { constructor(configCLIOverrides?: ConfigCLIOverrides) { this._fullConfig = { ...baseFullConfig }; - this._fullConfig._configCLIOverrides = configCLIOverrides || {}; + this._fullConfig._internal.configCLIOverrides = configCLIOverrides || {}; } static async deserialize(data: SerializedConfig): Promise { @@ -60,7 +60,7 @@ export class ConfigLoader { validateConfig(configFile || '', config); // 2. Override settings from CLI. - const configCLIOverrides = this._fullConfig._configCLIOverrides; + const configCLIOverrides = this._fullConfig._internal.configCLIOverrides; config.forbidOnly = takeFirst(configCLIOverrides.forbidOnly, config.forbidOnly); config.fullyParallel = takeFirst(configCLIOverrides.fullyParallel, config.fullyParallel); config.globalTimeout = takeFirst(configCLIOverrides.globalTimeout, config.globalTimeout); @@ -101,11 +101,11 @@ export class ConfigLoader { if (config.snapshotDir !== undefined) config.snapshotDir = path.resolve(configDir, config.snapshotDir); - this._fullConfig._configDir = configDir; - this._fullConfig._storeDir = path.resolve(configDir, '.playwright-store'); + this._fullConfig._internal.configDir = configDir; + this._fullConfig._internal.storeDir = path.resolve(configDir, '.playwright-store'); this._fullConfig.configFile = configFile; this._fullConfig.rootDir = config.testDir || configDir; - this._fullConfig._globalOutputDir = takeFirst(config.outputDir, throwawayArtifactsPath, baseFullConfig._globalOutputDir); + this._fullConfig._internal.globalOutputDir = takeFirst(config.outputDir, throwawayArtifactsPath, baseFullConfig._internal.globalOutputDir); this._fullConfig.forbidOnly = takeFirst(config.forbidOnly, baseFullConfig.forbidOnly); this._fullConfig.fullyParallel = takeFirst(config.fullyParallel, baseFullConfig.fullyParallel); this._fullConfig.globalSetup = takeFirst(config.globalSetup, baseFullConfig.globalSetup); @@ -119,9 +119,9 @@ export class ConfigLoader { this._fullConfig.reportSlowTests = takeFirst(config.reportSlowTests, baseFullConfig.reportSlowTests); this._fullConfig.quiet = takeFirst(config.quiet, baseFullConfig.quiet); this._fullConfig.shard = takeFirst(config.shard, baseFullConfig.shard); - this._fullConfig._ignoreSnapshots = takeFirst(config.ignoreSnapshots, baseFullConfig._ignoreSnapshots); + this._fullConfig._internal.ignoreSnapshots = takeFirst(config.ignoreSnapshots, baseFullConfig._internal.ignoreSnapshots); this._fullConfig.updateSnapshots = takeFirst(config.updateSnapshots, baseFullConfig.updateSnapshots); - this._fullConfig._pluginRegistrations = (config as any)._plugins || []; + this._fullConfig._internal.pluginRegistrations = (config as any)._plugins || []; const workers = takeFirst(config.workers, '50%'); if (typeof workers === 'string') { @@ -139,10 +139,10 @@ export class ConfigLoader { if (Array.isArray(webServers)) { // multiple web server mode // Due to previous choices, this value shows up to the user in globalSetup as part of FullConfig. Arrays are not supported by the old type. this._fullConfig.webServer = null; - this._fullConfig._webServers = webServers; + this._fullConfig._internal.webServers = webServers; } else if (webServers) { // legacy singleton mode this._fullConfig.webServer = webServers; - this._fullConfig._webServers = [webServers]; + this._fullConfig._internal.webServers = [webServers]; } this._fullConfig.metadata = takeFirst(config.metadata, baseFullConfig.metadata); this._fullConfig.projects = (config.projects || [config]).map(p => this._resolveProject(config, this._fullConfig, p, throwawayArtifactsPath)); @@ -159,7 +159,7 @@ export class ConfigLoader { const candidate = name + (i ? i : ''); if (usedNames.has(candidate)) continue; - p._id = candidate; + p._internal.id = candidate; usedNames.add(candidate); break; } @@ -171,7 +171,7 @@ export class ConfigLoader { } private _applyCLIOverridesToProject(projectConfig: Project) { - const configCLIOverrides = this._fullConfig._configCLIOverrides; + const configCLIOverrides = this._fullConfig._internal.configCLIOverrides; projectConfig.fullyParallel = takeFirst(configCLIOverrides.fullyParallel, projectConfig.fullyParallel); projectConfig.outputDir = takeFirst(configCLIOverrides.outputDir, projectConfig.outputDir); projectConfig.repeatEach = takeFirst(configCLIOverrides.repeatEach, projectConfig.repeatEach); @@ -183,13 +183,13 @@ export class ConfigLoader { private _resolveProject(config: Config, fullConfig: FullConfigInternal, projectConfig: Project, throwawayArtifactsPath: string): FullProjectInternal { // Resolve all config dirs relative to configDir. if (projectConfig.testDir !== undefined) - projectConfig.testDir = path.resolve(fullConfig._configDir, projectConfig.testDir); + projectConfig.testDir = path.resolve(fullConfig._internal.configDir, projectConfig.testDir); if (projectConfig.outputDir !== undefined) - projectConfig.outputDir = path.resolve(fullConfig._configDir, projectConfig.outputDir); + projectConfig.outputDir = path.resolve(fullConfig._internal.configDir, projectConfig.outputDir); if (projectConfig.snapshotDir !== undefined) - projectConfig.snapshotDir = path.resolve(fullConfig._configDir, projectConfig.snapshotDir); + projectConfig.snapshotDir = path.resolve(fullConfig._internal.configDir, projectConfig.snapshotDir); - const testDir = takeFirst(projectConfig.testDir, config.testDir, fullConfig._configDir); + const testDir = takeFirst(projectConfig.testDir, config.testDir, fullConfig._internal.configDir); const respectGitIgnore = !projectConfig.testDir && !config.testDir; const outputDir = takeFirst(projectConfig.outputDir, config.outputDir, path.join(throwawayArtifactsPath, 'test-results')); @@ -199,11 +199,15 @@ export class ConfigLoader { const defaultSnapshotPathTemplate = '{snapshotDir}/{testFileDir}/{testFileName}-snapshots/{arg}{-projectName}{-snapshotSuffix}{ext}'; const snapshotPathTemplate = takeFirst(projectConfig.snapshotPathTemplate, config.snapshotPathTemplate, defaultSnapshotPathTemplate); return { - _id: '', - _fullConfig: fullConfig, - _fullyParallel: takeFirst(projectConfig.fullyParallel, config.fullyParallel, undefined), - _expect: takeFirst(projectConfig.expect, config.expect, {}), - _deps: [], + _internal: { + id: '', + type: 'top-level', + fullConfig: fullConfig, + fullyParallel: takeFirst(projectConfig.fullyParallel, config.fullyParallel, undefined), + expect: takeFirst(projectConfig.expect, config.expect, {}), + deps: [], + respectGitIgnore: respectGitIgnore, + }, grep: takeFirst(projectConfig.grep, config.grep, baseFullConfig.grep), grepInvert: takeFirst(projectConfig.grepInvert, config.grepInvert, baseFullConfig.grepInvert), outputDir, @@ -212,7 +216,6 @@ export class ConfigLoader { metadata: takeFirst(projectConfig.metadata, config.metadata, undefined), name, testDir, - _respectGitIgnore: respectGitIgnore, snapshotDir, snapshotPathTemplate, testIgnore: takeFirst(projectConfig.testIgnore, config.testIgnore, []), @@ -433,14 +436,19 @@ export const baseFullConfig: FullConfigInternal = { version: require('../../package.json').version, workers: 0, webServer: null, - _webServers: [], - _globalOutputDir: path.resolve(process.cwd()), - _configDir: '', - _configCLIOverrides: {}, - _storeDir: '', - _maxConcurrentTestGroups: 0, - _ignoreSnapshots: false, - _pluginRegistrations: [], + _internal: { + webServers: [], + globalOutputDir: path.resolve(process.cwd()), + configDir: '', + configCLIOverrides: {}, + storeDir: '', + maxConcurrentTestGroups: 0, + ignoreSnapshots: false, + pluginRegistrations: [], + testTitleMatcher: () => true, + testFileFilters: [], + listOnly: false, + } }; function resolveReporters(reporters: Config['reporter'], rootDir: string): ReporterDescription[]|undefined { @@ -466,7 +474,7 @@ function resolveProjectDependencies(projects: FullProjectInternal[]) { throw new Error(`Project '${project.name}' depends on unknown project '${dependencyName}'`); if (dependencies.length > 1) throw new Error(`Project dependencies should have unique names, reading ${dependencyName}`); - project._deps.push(...dependencies); + project._internal.deps.push(...dependencies); } } } diff --git a/packages/playwright-test/src/common/globals.ts b/packages/playwright-test/src/common/globals.ts index d39c58f373..c3c2830481 100644 --- a/packages/playwright-test/src/common/globals.ts +++ b/packages/playwright-test/src/common/globals.ts @@ -37,7 +37,7 @@ export function currentExpectTimeout(options: { timeout?: number }) { const testInfo = currentTestInfo(); if (options.timeout !== undefined) return options.timeout; - let defaultExpectTimeout = testInfo?.project._expect?.timeout; + let defaultExpectTimeout = testInfo?.project._internal.expect?.timeout; if (typeof defaultExpectTimeout === 'undefined') defaultExpectTimeout = 5000; return defaultExpectTimeout; diff --git a/packages/playwright-test/src/common/ipc.ts b/packages/playwright-test/src/common/ipc.ts index 69111868b4..04ff4c5c0f 100644 --- a/packages/playwright-test/src/common/ipc.ts +++ b/packages/playwright-test/src/common/ipc.ts @@ -124,8 +124,8 @@ export type TeardownErrorsPayload = { export function serializeConfig(config: FullConfigInternal): SerializedConfig { const result: SerializedConfig = { configFile: config.configFile, - configDir: config._configDir, - configCLIOverrides: config._configCLIOverrides, + configDir: config._internal.configDir, + configCLIOverrides: config._internal.configCLIOverrides, }; return result; } diff --git a/packages/playwright-test/src/common/poolBuilder.ts b/packages/playwright-test/src/common/poolBuilder.ts index 4a487e9d8e..589a3ffd45 100644 --- a/packages/playwright-test/src/common/poolBuilder.ts +++ b/packages/playwright-test/src/common/poolBuilder.ts @@ -103,7 +103,7 @@ export class PoolBuilder { if (Object.entries(optionsFromConfig).length) { // Add config options immediately after original option definition, // so that any test.use() override it. - result.push({ fixtures: optionsFromConfig, location: { file: `project#${project._id}`, line: 1, column: 1 }, fromConfig: true }); + result.push({ fixtures: optionsFromConfig, location: { file: `project#${project._internal.id}`, line: 1, column: 1 }, fromConfig: true }); } } return result; diff --git a/packages/playwright-test/src/common/suiteUtils.ts b/packages/playwright-test/src/common/suiteUtils.ts index eb48419e38..a46c0b3b46 100644 --- a/packages/playwright-test/src/common/suiteUtils.ts +++ b/packages/playwright-test/src/common/suiteUtils.ts @@ -54,11 +54,11 @@ export function buildFileSuiteForProject(project: FullProjectInternal, suite: Su const repeatEachIndexSuffix = repeatEachIndex ? ` (repeat:${repeatEachIndex})` : ''; // At the point of the query, suite is not yet attached to the project, so we only get file, describe and test titles. - const testIdExpression = `[project=${project._id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`; + const testIdExpression = `[project=${project._internal.id}]${test.titlePath().join('\x1e')}${repeatEachIndexSuffix}`; const testId = fileId + '-' + calculateSha1(testIdExpression).slice(0, 20); test.id = testId; test.repeatEachIndex = repeatEachIndex; - test._projectId = project._id; + test._projectId = project._internal.id; // Inherit properties from parent suites. let inheritedRetries: number | undefined; @@ -79,7 +79,7 @@ export function buildFileSuiteForProject(project: FullProjectInternal, suite: Su // We only compute / set digest in the runner. if (test._poolDigest) - test._workerHash = `${project._id}-${test._poolDigest}-${repeatEachIndex}`; + test._workerHash = `${project._internal.id}-${test._poolDigest}-${repeatEachIndex}`; }); return result; diff --git a/packages/playwright-test/src/common/testInfo.ts b/packages/playwright-test/src/common/testInfo.ts index 5fb6bd9cd3..bbf4c2b9a1 100644 --- a/packages/playwright-test/src/common/testInfo.ts +++ b/packages/playwright-test/src/common/testInfo.ts @@ -117,8 +117,8 @@ export class TestInfoImpl implements TestInfo { const fullTitleWithoutSpec = test.titlePath().slice(1).join(' '); let testOutputDir = trimLongString(sanitizedRelativePath + '-' + sanitizeForFilePath(fullTitleWithoutSpec)); - if (project._id) - testOutputDir += '-' + sanitizeForFilePath(project._id); + if (project._internal.id) + testOutputDir += '-' + sanitizeForFilePath(project._internal.id); if (this.retry) testOutputDir += '-retry' + this.retry; if (this.repeatEachIndex) @@ -302,7 +302,7 @@ export class TestInfoImpl implements TestInfo { .replace(/\{(.)?arg\}/g, '$1' + path.join(parsedSubPath.dir, parsedSubPath.name)) .replace(/\{(.)?ext\}/g, parsedSubPath.ext ? '$1' + parsedSubPath.ext : ''); - return path.normalize(path.resolve(this.config._configDir, snapshotPath)); + return path.normalize(path.resolve(this.config._internal.configDir, snapshotPath)); } skip(...args: [arg?: any, description?: string]) { diff --git a/packages/playwright-test/src/common/types.ts b/packages/playwright-test/src/common/types.ts index cc76a038d0..0c2b61b14a 100644 --- a/packages/playwright-test/src/common/types.ts +++ b/packages/playwright-test/src/common/types.ts @@ -17,6 +17,7 @@ import type { Fixtures, TestInfoError, Project } from '../../types/test'; import type { Location } from '../../types/testReporter'; import type { TestRunnerPluginRegistration } from '../plugins'; +import type { Matcher, TestFileFilter } from '../util'; import type { ConfigCLIOverrides } from './ipc'; import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types'; export * from '../../types/test'; @@ -39,40 +40,54 @@ export interface TestStepInternal { refinedTitle?: string; } +type ConfigInternal = { + globalOutputDir: string; + configDir: string; + configCLIOverrides: ConfigCLIOverrides; + storeDir: string; + maxConcurrentTestGroups: number; + ignoreSnapshots: boolean; + webServers: Exclude[]; + pluginRegistrations: TestRunnerPluginRegistration[]; + listOnly: boolean; + testFileFilters: TestFileFilter[]; + testTitleMatcher: Matcher; + projectFilter?: string[]; + passWithNoTests?: boolean; +}; + /** * FullConfigInternal allows the plumbing of configuration details throughout the Test Runner without * increasing the surface area of the public API type called FullConfig. */ export interface FullConfigInternal extends FullConfigPublic { - _globalOutputDir: string; - _configDir: string; - _configCLIOverrides: ConfigCLIOverrides; - _storeDir: string; - _maxConcurrentTestGroups: number; - _ignoreSnapshots: boolean; + _internal: ConfigInternal; + /** * If populated, this should also be the first/only entry in _webServers. Legacy singleton `webServer` as well as those provided via an array in the user-facing playwright.config.{ts,js} will be in `_webServers`. The legacy field (`webServer`) field additionally stores the backwards-compatible singleton `webServer` since it had been showing up in globalSetup to the user. */ webServer: FullConfigPublic['webServer']; - _webServers: Exclude[]; // Overrides the public field. projects: FullProjectInternal[]; - - _pluginRegistrations: TestRunnerPluginRegistration[]; } +type ProjectInternal = { + id: string; + type: 'top-level' | 'dependency'; + fullConfig: FullConfigInternal; + fullyParallel: boolean; + expect: Project['expect']; + respectGitIgnore: boolean; + deps: FullProjectInternal[]; +}; + /** * FullProjectInternal allows the plumbing of configuration details throughout the Test Runner without * increasing the surface area of the public API type called FullProject. */ export interface FullProjectInternal extends FullProjectPublic { - _id: string; - _fullConfig: FullConfigInternal; - _fullyParallel: boolean; - _expect: Project['expect']; - _respectGitIgnore: boolean; - _deps: FullProjectInternal[]; + _internal: ProjectInternal; snapshotPathTemplate: string; } diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index 7f40079f4e..b20c8d46d4 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -253,12 +253,12 @@ export function toMatchSnapshot( if (received instanceof Promise) throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.'); - if (testInfo.config._ignoreSnapshots) + if (testInfo.config._internal.ignoreSnapshots) return { pass: !this.isNot, message: () => '' }; const helper = new SnapshotHelper( testInfo, testInfo.snapshotPath.bind(testInfo), determineFileExtension(received), - testInfo.project._expect?.toMatchSnapshot || {}, + testInfo.project._internal.expect?.toMatchSnapshot || {}, nameOrOptions, optOptions); if (this.isNot) { @@ -298,10 +298,10 @@ export async function toHaveScreenshot( if (!testInfo) throw new Error(`toHaveScreenshot() must be called during the test`); - if (testInfo.config._ignoreSnapshots) + if (testInfo.config._internal.ignoreSnapshots) return { pass: !this.isNot, message: () => '' }; - const config = (testInfo.project._expect as any)?.toHaveScreenshot; + const config = (testInfo.project._internal.expect as any)?.toHaveScreenshot; const snapshotPathResolver = testInfo.snapshotPath.bind(testInfo); const helper = new SnapshotHelper( testInfo, snapshotPathResolver, 'png', diff --git a/packages/playwright-test/src/plugins/webServerPlugin.ts b/packages/playwright-test/src/plugins/webServerPlugin.ts index 67ead61ae6..60bee7e54e 100644 --- a/packages/playwright-test/src/plugins/webServerPlugin.ts +++ b/packages/playwright-test/src/plugins/webServerPlugin.ts @@ -209,7 +209,7 @@ export const webServer = (options: WebServerPluginOptions): TestRunnerPlugin => export const webServerPluginsForConfig = (config: FullConfigInternal): TestRunnerPlugin[] => { const shouldSetBaseUrl = !!config.webServer; const webServerPlugins = []; - for (const webServerConfig of config._webServers) { + for (const webServerConfig of config._internal.webServers) { if ((!webServerConfig.port && !webServerConfig.url) || (webServerConfig.port && webServerConfig.url)) throw new Error(`Exactly one of 'port' or 'url' is required in config.webServer.`); diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 975cdf59bb..93e209f5c1 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -121,7 +121,7 @@ export class BaseReporter implements Reporter { } protected generateStartingMessage() { - const jobs = Math.min(this.config.workers, this.config._maxConcurrentTestGroups); + const jobs = Math.min(this.config.workers, this.config._internal.maxConcurrentTestGroups); const shardDetails = this.config.shard ? `, shard ${this.config.shard.current} of ${this.config.shard.total}` : ''; return `\nRunning ${this.totalTestCount} test${this.totalTestCount !== 1 ? 's' : ''} using ${jobs} worker${jobs !== 1 ? 's' : ''}${shardDetails}`; } diff --git a/packages/playwright-test/src/reporters/html.ts b/packages/playwright-test/src/reporters/html.ts index b85bdff2d1..af6007e436 100644 --- a/packages/playwright-test/src/reporters/html.ts +++ b/packages/playwright-test/src/reporters/html.ts @@ -92,9 +92,9 @@ class HtmlReporter implements Reporter { _resolveOptions(): { outputFolder: string, open: HtmlReportOpenOption } { let { outputFolder } = this._options; if (outputFolder) - outputFolder = path.resolve(this.config._configDir, outputFolder); + outputFolder = path.resolve(this.config._internal.configDir, outputFolder); return { - outputFolder: reportFolderFromEnv() ?? outputFolder ?? defaultReportFolder(this.config._configDir), + outputFolder: reportFolderFromEnv() ?? outputFolder ?? defaultReportFolder(this.config._internal.configDir), open: process.env.PW_TEST_HTML_REPORT_OPEN as any || this._options.open || 'on-failure', }; } diff --git a/packages/playwright-test/src/runner/loadUtils.ts b/packages/playwright-test/src/runner/loadUtils.ts index da70835cba..e0acf4695e 100644 --- a/packages/playwright-test/src/runner/loadUtils.ts +++ b/packages/playwright-test/src/runner/loadUtils.ts @@ -23,21 +23,13 @@ import type { TestCase } from '../common/test'; import type { FullConfigInternal, FullProjectInternal } from '../common/types'; import { createFileMatcherFromFilters, createTitleMatcher, errorWithFile } from '../util'; import type { Matcher, TestFileFilter } from '../util'; -import { collectFilesForProject, filterProjects, projectsThatAreDependencies } from './projectUtils'; +import { buildProjectsClosure, collectFilesForProject, filterProjects } from './projectUtils'; import { requireOrImport } from '../common/transform'; import { buildFileSuiteForProject, filterByFocusedLine, filterOnly, filterTestsRemoveEmptySuites } from '../common/suiteUtils'; import { filterForShard } from './testGroups'; -type LoadOptions = { - listOnly: boolean; - testFileFilters: TestFileFilter[]; - testTitleMatcher?: Matcher; - projectFilter?: string[]; - passWithNoTests?: boolean; -}; - -export async function loadAllTests(config: FullConfigInternal, options: LoadOptions, errors: TestError[]): Promise { - const projects = filterProjects(config.projects, options.projectFilter); +export async function loadAllTests(config: FullConfigInternal, errors: TestError[]): Promise { + const projects = filterProjects(config.projects, config._internal.projectFilter); let filesToRunByProject = new Map(); let topLevelProjects: FullProjectInternal[]; @@ -54,14 +46,16 @@ export async function loadAllTests(config: FullConfigInternal, options: LoadOpti } // Filter files based on the file filters, eliminate the empty projects. - const commandLineFileMatcher = options.testFileFilters.length ? createFileMatcherFromFilters(options.testFileFilters) : null; + const commandLineFileMatcher = config._internal.testFileFilters.length ? createFileMatcherFromFilters(config._internal.testFileFilters) : null; for (const [project, files] of allFilesForProject) { const filteredFiles = commandLineFileMatcher ? files.filter(commandLineFileMatcher) : files; if (filteredFiles.length) filesToRunByProject.set(project, filteredFiles); } - // Remove dependency projects, they'll be added back later. - for (const project of projectsThatAreDependencies([...filesToRunByProject.keys()])) + + const projectClosure = buildProjectsClosure([...filesToRunByProject.keys()]); + // Remove files for dependency projects, they'll be added back later. + for (const project of projectClosure.filter(p => p._internal.type === 'dependency')) filesToRunByProject.delete(project); // Shard only the top-level projects. @@ -69,8 +63,9 @@ export async function loadAllTests(config: FullConfigInternal, options: LoadOpti filesToRunByProject = filterForShard(config.shard, filesToRunByProject); // Re-build the closure, project set might have changed. - topLevelProjects = [...filesToRunByProject.keys()]; - dependencyProjects = projectsThatAreDependencies(topLevelProjects); + const filteredProjectClosure = buildProjectsClosure([...filesToRunByProject.keys()]); + topLevelProjects = filteredProjectClosure.filter(p => p._internal.type === 'top-level'); + dependencyProjects = filteredProjectClosure.filter(p => p._internal.type === 'dependency'); // (Re-)add all files for dependent projects, disregard filters. for (const project of dependencyProjects) { @@ -101,7 +96,7 @@ export async function loadAllTests(config: FullConfigInternal, options: LoadOpti // First iterate leaf projects to focus only, then add all other projects. for (const project of topLevelProjects) { - const projectSuite = await createProjectSuite(fileSuits, project, options, filesToRunByProject.get(project)!); + const projectSuite = await createProjectSuite(fileSuits, project, config._internal, filesToRunByProject.get(project)!); if (projectSuite) rootSuite._addSuite(projectSuite); } @@ -118,7 +113,7 @@ export async function loadAllTests(config: FullConfigInternal, options: LoadOpti // Prepend the projects that are dependencies. for (const project of dependencyProjects) { - const projectSuite = await createProjectSuite(fileSuits, project, { ...options, testFileFilters: [], testTitleMatcher: undefined }, filesToRunByProject.get(project)!); + const projectSuite = await createProjectSuite(fileSuits, project, { testFileFilters: [], testTitleMatcher: undefined }, filesToRunByProject.get(project)!); if (projectSuite) rootSuite._prependSuite(projectSuite); } @@ -126,14 +121,14 @@ export async function loadAllTests(config: FullConfigInternal, options: LoadOpti return rootSuite; } -async function createProjectSuite(fileSuits: Suite[], project: FullProjectInternal, options: LoadOptions, files: string[]): Promise { +async function createProjectSuite(fileSuits: Suite[], project: FullProjectInternal, options: { testFileFilters: TestFileFilter[], testTitleMatcher?: Matcher }, files: string[]): Promise { const fileSuitesMap = new Map(); for (const fileSuite of fileSuits) fileSuitesMap.set(fileSuite._requireFile, fileSuite); const projectSuite = new Suite(project.name, 'project'); projectSuite._projectConfig = project; - if (project._fullyParallel) + if (project._internal.fullyParallel) projectSuite._parallelMode = 'parallel'; for (const file of files) { const fileSuite = fileSuitesMap.get(file); diff --git a/packages/playwright-test/src/runner/projectUtils.ts b/packages/playwright-test/src/runner/projectUtils.ts index 94d7ff75fc..886bd75f8d 100644 --- a/packages/playwright-test/src/runner/projectUtils.ts +++ b/packages/playwright-test/src/runner/projectUtils.ts @@ -49,7 +49,7 @@ export function filterProjects(projects: FullProjectInternal[], projectNames?: s return result; } -export function projectsThatAreDependencies(projects: FullProjectInternal[]): FullProjectInternal[] { +export function buildProjectsClosure(projects: FullProjectInternal[]): FullProjectInternal[] { const result = new Set(); const visit = (depth: number, project: FullProjectInternal) => { if (depth > 100) { @@ -57,19 +57,22 @@ export function projectsThatAreDependencies(projects: FullProjectInternal[]): Fu error.stack = ''; throw error; } - if (result.has(project)) - return; - project._deps.map(visit.bind(undefined, depth + 1)); - project._deps.forEach(dep => result.add(dep)); + if (depth) + project._internal.type = 'dependency'; + result.add(project); + project._internal.deps.map(visit.bind(undefined, depth + 1)); }; - projects.forEach(visit.bind(undefined, 0)); + for (const p of projects) + p._internal.type = 'top-level'; + for (const p of projects) + visit(0, p); return [...result]; } export async function collectFilesForProject(project: FullProjectInternal, fsCache = new Map()): Promise { const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx']; const testFileExtension = (file: string) => extensions.includes(path.extname(file)); - const allFiles = await cachedCollectFiles(project.testDir, project._respectGitIgnore, fsCache); + const allFiles = await cachedCollectFiles(project.testDir, project._internal.respectGitIgnore, fsCache); const testMatch = createFileMatcher(project.testMatch); const testIgnore = createFileMatcher(project.testIgnore); const testFiles = allFiles.filter(file => { diff --git a/packages/playwright-test/src/runner/runner.ts b/packages/playwright-test/src/runner/runner.ts index e5cd3b2ed7..1850cccd7d 100644 --- a/packages/playwright-test/src/runner/runner.ts +++ b/packages/playwright-test/src/runner/runner.ts @@ -24,17 +24,8 @@ import { createReporter } from './reporters'; import { createTaskRunner, createTaskRunnerForList } from './tasks'; import type { TaskRunnerState } from './tasks'; import type { FullConfigInternal } from '../common/types'; -import type { Matcher, TestFileFilter } from '../util'; import { colors } from 'playwright-core/lib/utilsBundle'; -export type RunOptions = { - listOnly: boolean; - testFileFilters: TestFileFilter[]; - testTitleMatcher: Matcher; - projectFilter?: string[]; - passWithNoTests?: boolean; -}; - export class Runner { private _config: FullConfigInternal; @@ -56,22 +47,22 @@ export class Runner { return report; } - async runAllTests(options: RunOptions): Promise { + async runAllTests(): Promise { const config = this._config; + const listOnly = config._internal.listOnly; const deadline = config.globalTimeout ? monotonicTime() + config.globalTimeout : 0; // Legacy webServer support. - config._pluginRegistrations.push(...webServerPluginsForConfig(config)); + config._internal.pluginRegistrations.push(...webServerPluginsForConfig(config)); // Docker support. - config._pluginRegistrations.push(dockerPlugin); + config._internal.pluginRegistrations.push(dockerPlugin); - const reporter = await createReporter(config, options.listOnly); - const taskRunner = options.listOnly ? createTaskRunnerForList(config, reporter) + const reporter = await createReporter(config, listOnly); + const taskRunner = listOnly ? createTaskRunnerForList(config, reporter) : createTaskRunner(config, reporter); const context: TaskRunnerState = { config, - options, reporter, plugins: [], phases: [], @@ -79,7 +70,7 @@ export class Runner { reporter.onConfigure(config); - if (!options.listOnly && config._ignoreSnapshots) { + if (!listOnly && config._internal.ignoreSnapshots) { reporter.onStdOut(colors.dim([ 'NOTE: running with "ignoreSnapshots" option. All of the following asserts are silently ignored:', '- expect().toMatchSnapshot()', diff --git a/packages/playwright-test/src/runner/tasks.ts b/packages/playwright-test/src/runner/tasks.ts index a54e7e5fad..49f83b580e 100644 --- a/packages/playwright-test/src/runner/tasks.ts +++ b/packages/playwright-test/src/runner/tasks.ts @@ -28,19 +28,10 @@ import { TaskRunner } from './taskRunner'; import type { Suite } from '../common/test'; import type { FullConfigInternal, FullProjectInternal } from '../common/types'; import { loadAllTests, loadGlobalHook } from './loadUtils'; -import type { Matcher, TestFileFilter } from '../util'; const removeFolderAsync = promisify(rimraf); const readDirAsync = promisify(fs.readdir); -type TaskRunnerOptions = { - listOnly: boolean; - testFileFilters: TestFileFilter[]; - testTitleMatcher: Matcher; - projectFilter?: string[]; - passWithNoTests?: boolean; -}; - type ProjectWithTestGroups = { project: FullProjectInternal; projectSuite: Suite; @@ -48,7 +39,6 @@ type ProjectWithTestGroups = { }; export type TaskRunnerState = { - options: TaskRunnerOptions; reporter: Multiplexer; config: FullConfigInternal; plugins: TestRunnerPlugin[]; @@ -62,7 +52,7 @@ export type TaskRunnerState = { export function createTaskRunner(config: FullConfigInternal, reporter: Multiplexer): TaskRunner { const taskRunner = new TaskRunner(reporter, config.globalTimeout); - for (const plugin of config._pluginRegistrations) + for (const plugin of config._internal.pluginRegistrations) taskRunner.addTask('plugin setup', createPluginSetupTask(plugin)); if (config.globalSetup || config.globalTeardown) taskRunner.addTask('global setup', createGlobalSetupTask()); @@ -102,7 +92,7 @@ function createPluginSetupTask(pluginRegistration: TestRunnerPluginRegistration) else plugin = pluginRegistration; plugins.push(plugin); - await plugin.setup?.(config, config._configDir, reporter); + await plugin.setup?.(config, config._internal.configDir, reporter); return () => plugin.teardown?.(); }; } @@ -121,10 +111,10 @@ function createGlobalSetupTask(): Task { } function createRemoveOutputDirsTask(): Task { - return async ({ config, options }) => { + return async ({ config }) => { const outputDirs = new Set(); for (const p of config.projects) { - if (!options.projectFilter || options.projectFilter.includes(p.name)) + if (!config._internal.projectFilter || config._internal.projectFilter.includes(p.name)) outputDirs.add(p.outputDir); } @@ -144,10 +134,10 @@ function createRemoveOutputDirsTask(): Task { function createLoadTask(): Task { return async (context, errors) => { - const { config, options } = context; - context.rootSuite = await loadAllTests(config, options, errors); + const { config } = context; + context.rootSuite = await loadAllTests(config, errors); // Fail when no tests. - if (!context.rootSuite.allTests().length && !context.options.passWithNoTests && !config.shard) + if (!context.rootSuite.allTests().length && !config._internal.passWithNoTests && !config.shard) throw new Error(`No tests found`); }; } @@ -170,7 +160,7 @@ function createTestGroupsTask(): Task { const testGroupsInPhase = projects.reduce((acc, project) => acc + project.testGroups.length, 0); debug('pw:test:task')(`running phase with ${projects.map(p => p.project.name).sort()} projects, ${testGroupsInPhase} testGroups`); context.phases.push({ dispatcher: new Dispatcher(config, reporter), projects }); - context.config._maxConcurrentTestGroups = Math.max(context.config._maxConcurrentTestGroups, testGroupsInPhase); + context.config._internal.maxConcurrentTestGroups = Math.max(context.config._internal.maxConcurrentTestGroups, testGroupsInPhase); } return async () => { @@ -191,7 +181,7 @@ function createRunTestsTask(): Task { // that depend on the projects that failed previously. const phaseTestGroups: TestGroup[] = []; for (const { project, testGroups } of projects) { - const hasFailedDeps = project._deps.some(p => !successfulProjects.has(p)); + const hasFailedDeps = project._internal.deps.some(p => !successfulProjects.has(p)); if (!hasFailedDeps) { phaseTestGroups.push(...testGroups); } else { @@ -211,7 +201,7 @@ function createRunTestsTask(): Task { // projects failed. if (!dispatcher.hasWorkerErrors()) { for (const { project, projectSuite } of projects) { - const hasFailedDeps = project._deps.some(p => !successfulProjects.has(p)); + const hasFailedDeps = project._internal.deps.some(p => !successfulProjects.has(p)); if (!hasFailedDeps && !projectSuite.allTests().some(test => !test.ok())) successfulProjects.add(project); } @@ -228,7 +218,7 @@ function buildPhases(projectSuites: Suite[]): Suite[][] { for (const projectSuite of projectSuites) { if (processed.has(projectSuite._projectConfig!)) continue; - if (projectSuite._projectConfig!._deps.find(p => !processed.has(p))) + if (projectSuite._projectConfig!._internal.deps.find(p => !processed.has(p))) continue; phase.push(projectSuite); } diff --git a/packages/playwright-test/src/worker/workerMain.ts b/packages/playwright-test/src/worker/workerMain.ts index c852204494..d8c90fa5bd 100644 --- a/packages/playwright-test/src/worker/workerMain.ts +++ b/packages/playwright-test/src/worker/workerMain.ts @@ -194,7 +194,7 @@ export class WorkerMain extends ProcessRunner { const configLoader = await ConfigLoader.deserialize(this._params.config); this._config = configLoader.fullConfig(); - this._project = this._config.projects.find(p => p._id === this._params.projectId)!; + this._project = this._config.projects.find(p => p._internal.id === this._params.projectId)!; this._poolBuilder = PoolBuilder.createForWorker(this._project); } diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index d20382ef7b..ce0f8f6498 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -187,7 +187,10 @@ export interface FullProject { name: string; /** * List of projects that need to run before any test in this project runs. Dependencies can be useful for configuring - * the global setup actions in a way that every action is a test. For example: + * the global setup actions in a way that every action is in a form of a test. That way one can record traces and + * other artifacts for the global setup routine, see the setup steps in the test report, etc. + * + * For example: * * ```js * // playwright.config.ts @@ -5036,7 +5039,10 @@ export interface TestInfoError { interface TestProject { /** * List of projects that need to run before any test in this project runs. Dependencies can be useful for configuring - * the global setup actions in a way that every action is a test. For example: + * the global setup actions in a way that every action is in a form of a test. That way one can record traces and + * other artifacts for the global setup routine, see the setup steps in the test report, etc. + * + * For example: * * ```js * // playwright.config.ts