diff --git a/packages/playwright-test/src/loader.ts b/packages/playwright-test/src/loader.ts index 9d44c3c6bd..1f00ab307b 100644 --- a/packages/playwright-test/src/loader.ts +++ b/packages/playwright-test/src/loader.ts @@ -222,8 +222,8 @@ export class Loader { const name = takeFirst(this._configOverrides.name, projectConfig.name, config.name, ''); const screenshotsDir = takeFirst((this._configOverrides as any).screenshotsDir, (projectConfig as any).screenshotsDir, (config as any).screenshotsDir, path.join(testDir, '__screenshots__', process.platform, name)); const fullProject: FullProjectInternal = { - fullyParallel: takeFirst(this._configOverrides.fullyParallel, projectConfig.fullyParallel, config.fullyParallel, undefined), - expect: takeFirst(this._configOverrides.expect, projectConfig.expect, config.expect, undefined), + _fullyParallel: takeFirst(this._configOverrides.fullyParallel, projectConfig.fullyParallel, config.fullyParallel, undefined), + _expect: takeFirst(this._configOverrides.expect, projectConfig.expect, config.expect, undefined), grep: takeFirst(this._configOverrides.grep, projectConfig.grep, config.grep, baseFullConfig.grep), grepInvert: takeFirst(this._configOverrides.grepInvert, projectConfig.grepInvert, config.grepInvert, baseFullConfig.grepInvert), outputDir, diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index 2f27cd1423..dd33de3d79 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -253,7 +253,7 @@ export function toMatchSnapshot( throw new Error('An unresolved Promise was passed to toMatchSnapshot(), make sure to resolve it by adding await to it.'); const helper = new SnapshotHelper( testInfo, testInfo.snapshotPath.bind(testInfo), determineFileExtension(received), - testInfo.project.expect?.toMatchSnapshot || {}, + testInfo.project._expect?.toMatchSnapshot || {}, nameOrOptions, optOptions); if (this.isNot) { @@ -294,7 +294,7 @@ export async function toHaveScreenshot( const testInfo = currentTestInfo(); if (!testInfo) throw new Error(`toHaveScreenshot() must be called during the test`); - const config = (testInfo.project.expect as any)?.toHaveScreenshot; + const config = (testInfo.project._expect as any)?.toHaveScreenshot; const helper = new SnapshotHelper( testInfo, testInfo._screenshotPath.bind(testInfo), 'png', { diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 305d82f175..d8454f87c6 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -294,7 +294,7 @@ export class Runner { const grepInvertMatcher = project.config.grepInvert ? createTitleMatcher(project.config.grepInvert) : null; const projectSuite = new Suite(project.config.name); projectSuite._projectConfig = project.config; - if (project.config.fullyParallel) + if (project.config._fullyParallel) projectSuite._parallelMode = 'parallel'; rootSuite._addSuite(projectSuite); for (const file of files) { diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 06307ada48..bb85b30e25 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { Fixtures, TestError } from '../types/test'; +import type { Fixtures, TestError, Project } from '../types/test'; import type { Location } from '../types/testReporter'; import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types'; export * from '../types/test'; @@ -58,5 +58,7 @@ export interface FullConfigInternal extends FullConfigPublic { * increasing the surface area of the public API type called FullProject. */ export interface FullProjectInternal extends FullProjectPublic { + _fullyParallel: boolean; + _expect: Project['expect']; _screenshotsDir: string; } diff --git a/packages/playwright-test/src/util.ts b/packages/playwright-test/src/util.ts index 49d5b2b268..458a7911dd 100644 --- a/packages/playwright-test/src/util.ts +++ b/packages/playwright-test/src/util.ts @@ -251,7 +251,7 @@ export function currentExpectTimeout(options: { timeout?: number }) { const testInfo = currentTestInfo(); if (options.timeout !== undefined) return options.timeout; - let defaultExpectTimeout = testInfo?.project.expect?.timeout; + let defaultExpectTimeout = testInfo?.project._expect?.timeout; if (typeof defaultExpectTimeout === 'undefined') defaultExpectTimeout = 5000; return defaultExpectTimeout; diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 94f01637d5..c5a6da4430 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -385,7 +385,250 @@ export interface Project extends TestProject { use?: UseOptions; } -export type FullProject = Required>; +/** + * Playwright Test supports running multiple test projects at the same time. This is useful for running tests in multiple + * configurations. For example, consider running tests against multiple browsers. + * + * `TestProject` encapsulates configuration specific to a single project. Projects are configured in + * [testConfig.projects](https://playwright.dev/docs/api/class-testconfig#test-config-projects) specified in the + * [configuration file](https://playwright.dev/docs/test-configuration). Note that all properties of [TestProject] are available in the top-level + * [TestConfig], in which case they are shared between all projects. + * + * Here is an example configuration that runs every test in Chromium, Firefox and WebKit, both Desktop and Mobile versions. + * + * ```ts + * // playwright.config.ts + * import { PlaywrightTestConfig, devices } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * // Options shared for all projects. + * timeout: 30000, + * use: { + * ignoreHTTPSErrors: true, + * }, + * + * // Options specific to each project. + * projects: [ + * { + * name: 'Desktop Chromium', + * use: { + * browserName: 'chromium', + * viewport: { width: 1280, height: 720 }, + * }, + * }, + * { + * name: 'Desktop Safari', + * use: { + * browserName: 'webkit', + * viewport: { width: 1280, height: 720 }, + * } + * }, + * { + * name: 'Desktop Firefox', + * use: { + * browserName: 'firefox', + * viewport: { width: 1280, height: 720 }, + * } + * }, + * { + * name: 'Mobile Chrome', + * use: devices['Pixel 5'], + * }, + * { + * name: 'Mobile Safari', + * use: devices['iPhone 12'], + * }, + * ], + * }; + * export default config; + * ``` + * + */ +export interface FullProject { + /** + * Filter to only run tests with a title matching one of the patterns. For example, passing `grep: /cart/` should only run + * tests with "cart" in the title. Also available globally and in the [command line](https://playwright.dev/docs/test-cli) with the `-g` option. + * + * `grep` option is also useful for [tagging tests](https://playwright.dev/docs/test-annotations#tag-tests). + */ + grep: RegExp | RegExp[]; + /** + * Filter to only run tests with a title **not** matching one of the patterns. This is the opposite of + * [testProject.grep](https://playwright.dev/docs/api/class-testproject#test-project-grep). Also available globally and in + * the [command line](https://playwright.dev/docs/test-cli) with the `--grep-invert` option. + * + * `grepInvert` option is also useful for [tagging tests](https://playwright.dev/docs/test-annotations#tag-tests). + */ + grepInvert: RegExp | RegExp[] | null; + /** + * Any JSON-serializable metadata that will be put directly to the test report. + */ + metadata: any; + /** + * Project name is visible in the report and during test execution. + */ + name: string; + /** + * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to + * [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir). + * + * The directory for each test can be accessed by + * [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) and + * [testInfo.snapshotPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-path). + * + * This path will serve as the base directory for each test file snapshot directory. Setting `snapshotDir` to + * `'snapshots'`, the [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) would + * resolve to `snapshots/a.spec.js-snapshots`. + */ + snapshotDir: string; + /** + * The output directory for files created during test execution. Defaults to `/test-results`. + * + * This directory is cleaned at the start. When running a test, a unique subdirectory inside the + * [testProject.outputDir](https://playwright.dev/docs/api/class-testproject#test-project-output-dir) is created, + * guaranteeing that test running in parallel do not conflict. This directory can be accessed by + * [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) and + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path). + * + * Here is an example that uses + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path) to create a + * temporary file. + * + * ```ts + * import { test, expect } from '@playwright/test'; + * import fs from 'fs'; + * + * test('example test', async ({}, testInfo) => { + * const file = testInfo.outputPath('temporary-file.txt'); + * await fs.promises.writeFile(file, 'Put some data to the file', 'utf8'); + * }); + * ``` + * + * Use [testConfig.outputDir](https://playwright.dev/docs/api/class-testconfig#test-config-output-dir) to change this + * option for all projects. + */ + outputDir: string; + /** + * The number of times to repeat each test, useful for debugging flaky tests. + * + * Use [testConfig.repeatEach](https://playwright.dev/docs/api/class-testconfig#test-config-repeat-each) to change this + * option for all projects. + */ + repeatEach: number; + /** + * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + * + * Use [testConfig.retries](https://playwright.dev/docs/api/class-testconfig#test-config-retries) to change this option for + * all projects. + */ + retries: number; + /** + * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. + * + * Each project can use a different directory. Here is an example that runs smoke tests in three browsers and all other + * tests in stable Chrome browser. + * + * ```ts + * // playwright.config.ts + * import { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * projects: [ + * { + * name: 'Smoke Chromium', + * testDir: './smoke-tests', + * use: { + * browserName: 'chromium', + * } + * }, + * { + * name: 'Smoke WebKit', + * testDir: './smoke-tests', + * use: { + * browserName: 'webkit', + * } + * }, + * { + * name: 'Smoke Firefox', + * testDir: './smoke-tests', + * use: { + * browserName: 'firefox', + * } + * }, + * { + * name: 'Chrome Stable', + * testDir: './', + * use: { + * browserName: 'chromium', + * channel: 'chrome', + * } + * }, + * ], + * }; + * export default config; + * ``` + * + * Use [testConfig.testDir](https://playwright.dev/docs/api/class-testconfig#test-config-test-dir) to change this option + * for all projects. + */ + testDir: string; + /** + * Files matching one of these patterns are not executed as test files. Matching is performed against the absolute file + * path. Strings are treated as glob patterns. + * + * For example, `'**\/test-assets/**'` will ignore any files in the `test-assets` directory. + * + * Use [testConfig.testIgnore](https://playwright.dev/docs/api/class-testconfig#test-config-test-ignore) to change this + * option for all projects. + */ + testIgnore: string | RegExp | (string | RegExp)[]; + /** + * Only the files matching one of these patterns are executed as test files. Matching is performed against the absolute + * file path. Strings are treated as glob patterns. + * + * By default, Playwright Test looks for files matching `.*(test|spec)\.(js|ts|mjs)`. + * + * Use [testConfig.testMatch](https://playwright.dev/docs/api/class-testconfig#test-config-test-match) to change this + * option for all projects. + */ + testMatch: string | RegExp | (string | RegExp)[]; + /** + * Timeout for each test in milliseconds. Defaults to 30 seconds. + * + * This is a base timeout for all tests. In addition, each test can configure its own timeout with + * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). + * + * Use [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout) to change this option for + * all projects. + */ + timeout: number; + /** + * Options for all tests in this project, for example + * [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more about + * [configuration](https://playwright.dev/docs/test-configuration) and see [available options][TestOptions]. + * + * ```ts + * // playwright.config.ts + * import { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * projects: [ + * { + * name: 'Chromium', + * use: { + * browserName: 'chromium', + * }, + * }, + * ], + * }; + * export default config; + * ``` + * + * Use [testConfig.use](https://playwright.dev/docs/api/class-testconfig#test-config-use) to change this option for all + * projects. + */ + use: UseOptions; +} export type WebServerConfig = { /** diff --git a/tests/config/experimental.d.ts b/tests/config/experimental.d.ts index cd7d4a4b7c..3ada8dd771 100644 --- a/tests/config/experimental.d.ts +++ b/tests/config/experimental.d.ts @@ -16596,7 +16596,250 @@ export interface Project extends TestProject { use?: UseOptions; } -export type FullProject = Required>; +/** + * Playwright Test supports running multiple test projects at the same time. This is useful for running tests in multiple + * configurations. For example, consider running tests against multiple browsers. + * + * `TestProject` encapsulates configuration specific to a single project. Projects are configured in + * [testConfig.projects](https://playwright.dev/docs/api/class-testconfig#test-config-projects) specified in the + * [configuration file](https://playwright.dev/docs/test-configuration). Note that all properties of [TestProject] are available in the top-level + * [TestConfig], in which case they are shared between all projects. + * + * Here is an example configuration that runs every test in Chromium, Firefox and WebKit, both Desktop and Mobile versions. + * + * ```ts + * // playwright.config.ts + * import { PlaywrightTestConfig, devices } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * // Options shared for all projects. + * timeout: 30000, + * use: { + * ignoreHTTPSErrors: true, + * }, + * + * // Options specific to each project. + * projects: [ + * { + * name: 'Desktop Chromium', + * use: { + * browserName: 'chromium', + * viewport: { width: 1280, height: 720 }, + * }, + * }, + * { + * name: 'Desktop Safari', + * use: { + * browserName: 'webkit', + * viewport: { width: 1280, height: 720 }, + * } + * }, + * { + * name: 'Desktop Firefox', + * use: { + * browserName: 'firefox', + * viewport: { width: 1280, height: 720 }, + * } + * }, + * { + * name: 'Mobile Chrome', + * use: devices['Pixel 5'], + * }, + * { + * name: 'Mobile Safari', + * use: devices['iPhone 12'], + * }, + * ], + * }; + * export default config; + * ``` + * + */ +export interface FullProject { + /** + * Filter to only run tests with a title matching one of the patterns. For example, passing `grep: /cart/` should only run + * tests with "cart" in the title. Also available globally and in the [command line](https://playwright.dev/docs/test-cli) with the `-g` option. + * + * `grep` option is also useful for [tagging tests](https://playwright.dev/docs/test-annotations#tag-tests). + */ + grep: RegExp | RegExp[]; + /** + * Filter to only run tests with a title **not** matching one of the patterns. This is the opposite of + * [testProject.grep](https://playwright.dev/docs/api/class-testproject#test-project-grep). Also available globally and in + * the [command line](https://playwright.dev/docs/test-cli) with the `--grep-invert` option. + * + * `grepInvert` option is also useful for [tagging tests](https://playwright.dev/docs/test-annotations#tag-tests). + */ + grepInvert: RegExp | RegExp[] | null; + /** + * Any JSON-serializable metadata that will be put directly to the test report. + */ + metadata: any; + /** + * Project name is visible in the report and during test execution. + */ + name: string; + /** + * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to + * [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir). + * + * The directory for each test can be accessed by + * [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) and + * [testInfo.snapshotPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-path). + * + * This path will serve as the base directory for each test file snapshot directory. Setting `snapshotDir` to + * `'snapshots'`, the [testInfo.snapshotDir](https://playwright.dev/docs/api/class-testinfo#test-info-snapshot-dir) would + * resolve to `snapshots/a.spec.js-snapshots`. + */ + snapshotDir: string; + /** + * The output directory for files created during test execution. Defaults to `/test-results`. + * + * This directory is cleaned at the start. When running a test, a unique subdirectory inside the + * [testProject.outputDir](https://playwright.dev/docs/api/class-testproject#test-project-output-dir) is created, + * guaranteeing that test running in parallel do not conflict. This directory can be accessed by + * [testInfo.outputDir](https://playwright.dev/docs/api/class-testinfo#test-info-output-dir) and + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path). + * + * Here is an example that uses + * [testInfo.outputPath(...pathSegments)](https://playwright.dev/docs/api/class-testinfo#test-info-output-path) to create a + * temporary file. + * + * ```ts + * import { test, expect } from '@playwright/test'; + * import fs from 'fs'; + * + * test('example test', async ({}, testInfo) => { + * const file = testInfo.outputPath('temporary-file.txt'); + * await fs.promises.writeFile(file, 'Put some data to the file', 'utf8'); + * }); + * ``` + * + * Use [testConfig.outputDir](https://playwright.dev/docs/api/class-testconfig#test-config-output-dir) to change this + * option for all projects. + */ + outputDir: string; + /** + * The number of times to repeat each test, useful for debugging flaky tests. + * + * Use [testConfig.repeatEach](https://playwright.dev/docs/api/class-testconfig#test-config-repeat-each) to change this + * option for all projects. + */ + repeatEach: number; + /** + * The maximum number of retry attempts given to failed tests. Learn more about [test retries](https://playwright.dev/docs/test-retries#retries). + * + * Use [testConfig.retries](https://playwright.dev/docs/api/class-testconfig#test-config-retries) to change this option for + * all projects. + */ + retries: number; + /** + * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file. + * + * Each project can use a different directory. Here is an example that runs smoke tests in three browsers and all other + * tests in stable Chrome browser. + * + * ```ts + * // playwright.config.ts + * import { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * projects: [ + * { + * name: 'Smoke Chromium', + * testDir: './smoke-tests', + * use: { + * browserName: 'chromium', + * } + * }, + * { + * name: 'Smoke WebKit', + * testDir: './smoke-tests', + * use: { + * browserName: 'webkit', + * } + * }, + * { + * name: 'Smoke Firefox', + * testDir: './smoke-tests', + * use: { + * browserName: 'firefox', + * } + * }, + * { + * name: 'Chrome Stable', + * testDir: './', + * use: { + * browserName: 'chromium', + * channel: 'chrome', + * } + * }, + * ], + * }; + * export default config; + * ``` + * + * Use [testConfig.testDir](https://playwright.dev/docs/api/class-testconfig#test-config-test-dir) to change this option + * for all projects. + */ + testDir: string; + /** + * Files matching one of these patterns are not executed as test files. Matching is performed against the absolute file + * path. Strings are treated as glob patterns. + * + * For example, `'**\/test-assets/**'` will ignore any files in the `test-assets` directory. + * + * Use [testConfig.testIgnore](https://playwright.dev/docs/api/class-testconfig#test-config-test-ignore) to change this + * option for all projects. + */ + testIgnore: string | RegExp | (string | RegExp)[]; + /** + * Only the files matching one of these patterns are executed as test files. Matching is performed against the absolute + * file path. Strings are treated as glob patterns. + * + * By default, Playwright Test looks for files matching `.*(test|spec)\.(js|ts|mjs)`. + * + * Use [testConfig.testMatch](https://playwright.dev/docs/api/class-testconfig#test-config-test-match) to change this + * option for all projects. + */ + testMatch: string | RegExp | (string | RegExp)[]; + /** + * Timeout for each test in milliseconds. Defaults to 30 seconds. + * + * This is a base timeout for all tests. In addition, each test can configure its own timeout with + * [test.setTimeout(timeout)](https://playwright.dev/docs/api/class-test#test-set-timeout). + * + * Use [testConfig.timeout](https://playwright.dev/docs/api/class-testconfig#test-config-timeout) to change this option for + * all projects. + */ + timeout: number; + /** + * Options for all tests in this project, for example + * [testOptions.browserName](https://playwright.dev/docs/api/class-testoptions#test-options-browser-name). Learn more about + * [configuration](https://playwright.dev/docs/test-configuration) and see [available options][TestOptions]. + * + * ```ts + * // playwright.config.ts + * import { PlaywrightTestConfig } from '@playwright/test'; + * + * const config: PlaywrightTestConfig = { + * projects: [ + * { + * name: 'Chromium', + * use: { + * browserName: 'chromium', + * }, + * }, + * ], + * }; + * export default config; + * ``` + * + * Use [testConfig.use](https://playwright.dev/docs/api/class-testconfig#test-config-use) to change this option for all + * projects. + */ + use: UseOptions; +} export type WebServerConfig = { /** diff --git a/utils/generate_types/index.js b/utils/generate_types/index.js index d2c9089a27..491464c3ed 100644 --- a/utils/generate_types/index.js +++ b/utils/generate_types/index.js @@ -559,6 +559,7 @@ class TypesGenerator { ['Config', 'TestConfig'], ['FullConfig', 'TestConfig'], ['Project', 'TestProject'], + ['FullProject', 'TestProject'], ['PlaywrightWorkerOptions', 'TestOptions'], ['PlaywrightTestOptions', 'TestOptions'], ['PlaywrightWorkerArgs', 'Fixtures'], diff --git a/utils/generate_types/overrides-test.d.ts b/utils/generate_types/overrides-test.d.ts index cfe7044126..9c13b2019b 100644 --- a/utils/generate_types/overrides-test.d.ts +++ b/utils/generate_types/overrides-test.d.ts @@ -55,7 +55,25 @@ export interface Project extends TestProject { use?: UseOptions; } -export type FullProject = Required>; +// [internal] !!! DO NOT ADD TO THIS !!! +// [internal] It is part of the public API and is computed from the user's config. +// [internal] If you need new fields internally, add them to FullConfigInternal instead. +export interface FullProject { + grep: RegExp | RegExp[]; + grepInvert: RegExp | RegExp[] | null; + metadata: any; + name: string; + snapshotDir: string; + outputDir: string; + repeatEach: number; + retries: number; + testDir: string; + testIgnore: string | RegExp | (string | RegExp)[]; + testMatch: string | RegExp | (string | RegExp)[]; + timeout: number; + use: UseOptions; + // [internal] !!! DO NOT ADD TO THIS !!! See prior note. +} export type WebServerConfig = { /**