From f23d02a2116947369ff6206f9c73050dd36fe95c Mon Sep 17 00:00:00 2001 From: Simon Knott Date: Tue, 23 Jul 2024 18:04:17 +0200 Subject: [PATCH] feat(test runner): `--only-changed` option (#31727) Introduces an `--only-changed [base ref]` option. `playwright test --only-changed` filters the test run to only run test suites that have uncommitted changes. `playwright test --only-changed=foo` runs only tests that were changed since commit `foo`. In pull request CI, this can be used to run changed tests first and fail fast: `--only-changed=$GITHUB_BASE_REF`. During local development, it can be used to quickly filter down to the touched set of tests suites. In some rare usecases, this can also help to cut down on CI usage for pull requests. Tread with caution though. File dependencies are taken into account to ensure that if you touched a utility file, all relevant tests are still executed. Closes https://github.com/microsoft/playwright/issues/15075 --- docs/src/ci-intro.md | 37 ++ docs/src/test-cli-js.md | 1 + packages/playwright-ct-core/src/vitePlugin.ts | 6 +- packages/playwright/src/common/config.ts | 1 + packages/playwright/src/plugins/index.ts | 1 + packages/playwright/src/program.ts | 6 +- packages/playwright/src/runner/loadUtils.ts | 6 +- packages/playwright/src/runner/reporters.ts | 2 + packages/playwright/src/runner/tasks.ts | 22 +- packages/playwright/src/runner/vcs.ts | 45 +++ tests/playwright-test/only-changed.spec.ts | 367 ++++++++++++++++++ .../playwright-test-fixtures.ts | 4 +- tests/playwright-test/watch.spec.ts | 46 +-- 13 files changed, 509 insertions(+), 35 deletions(-) create mode 100644 packages/playwright/src/runner/vcs.ts create mode 100644 tests/playwright-test/only-changed.spec.ts diff --git a/docs/src/ci-intro.md b/docs/src/ci-intro.md index 4652c31133..4e4b2bb19c 100644 --- a/docs/src/ci-intro.md +++ b/docs/src/ci-intro.md @@ -383,6 +383,43 @@ jobs: PLAYWRIGHT_TEST_BASE_URL: ${{ github.event.deployment_status.target_url }} ``` +### Fail-Fast +* langs: js + +Even with sharding enabled, large test suites can take very long to execute. Running changed tests first on PRs will give you a faster feedback loop and use less CI resources. + +```yml js title=".github/workflows/playwright.yml" +name: Playwright Tests +on: + push: + branches: [ main, master ] + pull_request: + branches: [ main, master ] +jobs: + test: + timeout-minutes: 60 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Install dependencies + run: npm ci + - name: Install Playwright Browsers + run: npx playwright install --with-deps + - name: Run changed Playwright tests + run: npx playwright test --only-changed=$GITHUB_BASE_REF + if: github.event_name == 'pull_request' + - name: Run Playwright tests + run: npx playwright test + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: playwright-report + path: playwright-report/ + retention-days: 30 +``` ## Create a Repo and Push to GitHub diff --git a/docs/src/test-cli-js.md b/docs/src/test-cli-js.md index 290fb7df96..052c0abf1f 100644 --- a/docs/src/test-cli-js.md +++ b/docs/src/test-cli-js.md @@ -93,6 +93,7 @@ Complete set of Playwright Test options is available in the [configuration file] | `--max-failures ` or `-x`| Stop after the first `N` test failures. Passing `-x` stops after the first failure.| | `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. | | `--output ` | Directory for artifacts produced by tests, defaults to `test-results`. | +| `--only-changed [ref]` | Only run tests that have been changed between working tree and "ref". Defaults to running all uncommitted changes with ref=HEAD. Only supports Git. | | `--pass-with-no-tests` | Allows the test suite to pass when no files are found. | | `--project ` | Only run tests from the specified [projects](./test-projects.md), supports '*' wildcard. Defaults to running all projects defined in the configuration file.| | `--quiet` | Whether to suppress stdout and stderr from the tests. | diff --git a/packages/playwright-ct-core/src/vitePlugin.ts b/packages/playwright-ct-core/src/vitePlugin.ts index dd028c0f06..e4a0a43266 100644 --- a/packages/playwright-ct-core/src/vitePlugin.ts +++ b/packages/playwright-ct-core/src/vitePlugin.ts @@ -69,6 +69,10 @@ export function createPlugin(): TestRunnerPlugin { if (stoppableServer) await new Promise(f => stoppableServer.stop(f)); }, + + populateDependencies: async () => { + await buildBundle(config, configDir); + }, }; } @@ -157,7 +161,7 @@ export async function buildBundle(config: FullConfig, configDir: string): Promis const viteConfig = await createConfig(dirs, config, frameworkPluginFactory, jsxInJS); if (sourcesDirty) { - // Only add out own plugin when we actually build / transform. + // Only add our own plugin when we actually build / transform. log('build'); const depsCollector = new Map(); const buildConfig = mergeConfig(viteConfig, { diff --git a/packages/playwright/src/common/config.ts b/packages/playwright/src/common/config.ts index 32d00d2751..610382e175 100644 --- a/packages/playwright/src/common/config.ts +++ b/packages/playwright/src/common/config.ts @@ -49,6 +49,7 @@ export class FullConfigInternal { cliArgs: string[] = []; cliGrep: string | undefined; cliGrepInvert: string | undefined; + cliOnlyChanged: string | undefined; cliProjectFilter?: string[]; cliListOnly = false; cliPassWithNoTests?: boolean; diff --git a/packages/playwright/src/plugins/index.ts b/packages/playwright/src/plugins/index.ts index 58a1c2477e..3502d90966 100644 --- a/packages/playwright/src/plugins/index.ts +++ b/packages/playwright/src/plugins/index.ts @@ -20,6 +20,7 @@ import type { ReporterV2 } from '../reporters/reporterV2'; export interface TestRunnerPlugin { name: string; setup?(config: FullConfig, configDir: string, reporter: ReporterV2): Promise; + populateDependencies?(): Promise; begin?(suite: Suite): Promise; end?(): Promise; teardown?(): Promise; diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts index 6e49a02b05..7cbe7e5461 100644 --- a/packages/playwright/src/program.ts +++ b/packages/playwright/src/program.ts @@ -153,12 +153,14 @@ Examples: $ npx playwright merge-reports playwright-report`); } - async function runTests(args: string[], opts: { [key: string]: any }) { await startProfiling(); const cliOverrides = overridesFromOptions(opts); if (opts.ui || opts.uiHost || opts.uiPort) { + if (opts.onlyChanged) + throw new Error(`--only-changed is not supported in UI mode. If you'd like that to change, see https://github.com/microsoft/playwright/issues/15075 for more details.`); + const status = await testServer.runUIMode(opts.config, { host: opts.uiHost, port: opts.uiPort ? +opts.uiPort : undefined, @@ -192,6 +194,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) { config.cliArgs = args; config.cliGrep = opts.grep as string | undefined; + config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged; config.cliGrepInvert = opts.grepInvert as string | undefined; config.cliListOnly = !!opts.list; config.cliProjectFilter = opts.project || undefined; @@ -352,6 +355,7 @@ const testOptions: [string, string][] = [ ['--max-failures ', `Stop after the first N failures`], ['--no-deps', 'Do not run project dependencies'], ['--output ', `Folder for output artifacts (default: "test-results")`], + ['--only-changed [ref]', `Only run tests that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git.`], ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`], ['--project ', `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)`], ['--quiet', `Suppress stdio`], diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts index d7d1c7c6c8..4cac21cd08 100644 --- a/packages/playwright/src/runner/loadUtils.ts +++ b/packages/playwright/src/runner/loadUtils.ts @@ -32,6 +32,7 @@ import { dependenciesForTestFile } from '../transform/compilationCache'; import { sourceMapSupport } from '../utilsBundle'; import type { RawSourceMap } from 'source-map'; + export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher?: Matcher) { const config = testRun.config; const fsCache = new Map(); @@ -118,7 +119,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' | } } -export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise { +export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean, additionalFileMatcher?: Matcher): Promise { const config = testRun.config; // Create root suite, where each child will be a project suite with cloned file suites inside it. const rootSuite = new Suite('', 'root'); @@ -135,7 +136,8 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho // Filter file suites for all projects. for (const [project, fileSuites] of testRun.projectSuites) { - const projectSuite = createProjectSuite(project, fileSuites); + const filteredFileSuites = additionalFileMatcher ? fileSuites.filter(fileSuite => additionalFileMatcher(fileSuite.location!.file)) : fileSuites; + const projectSuite = createProjectSuite(project, filteredFileSuites); projectSuites.set(project, projectSuite); const filteredProjectSuite = filterProjectSuite(projectSuite, { cliFileFilters, cliTitleMatcher, testIdMatcher: config.testIdMatcher }); filteredProjectSuites.set(project, filteredProjectSuite); diff --git a/packages/playwright/src/runner/reporters.ts b/packages/playwright/src/runner/reporters.ts index cbad53e892..1748292cc0 100644 --- a/packages/playwright/src/runner/reporters.ts +++ b/packages/playwright/src/runner/reporters.ts @@ -107,6 +107,8 @@ function computeCommandHash(config: FullConfigInternal) { command.cliGrep = config.cliGrep; if (config.cliGrepInvert) command.cliGrepInvert = config.cliGrepInvert; + if (config.cliOnlyChanged) + command.cliOnlyChanged = config.cliOnlyChanged; if (Object.keys(command).length) parts.push(calculateSha1(JSON.stringify(command)).substring(0, 7)); return parts.join('-'); diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts index 5a69b94e8b..624b6e1d9f 100644 --- a/packages/playwright/src/runner/tasks.ts +++ b/packages/playwright/src/runner/tasks.ts @@ -31,6 +31,7 @@ import type { Matcher } from '../util'; import { Suite } from '../common/test'; import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils'; import { FailureTracker } from './failureTracker'; +import { detectChangedTests } from './vcs'; const readDirAsync = promisify(fs.readdir); @@ -64,7 +65,7 @@ export class TestRun { export function createTaskRunner(config: FullConfigInternal, reporter: ReporterV2): TaskRunner { const taskRunner = new TaskRunner(reporter, config.config.globalTimeout); addGlobalSetupTasks(taskRunner, config); - taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true })); + taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: true })); addRunTasks(taskRunner, config); return taskRunner; } @@ -77,14 +78,14 @@ export function createTaskRunnerForWatchSetup(config: FullConfigInternal, report export function createTaskRunnerForWatch(config: FullConfigInternal, reporter: ReporterV2, additionalFileMatcher?: Matcher): TaskRunner { const taskRunner = new TaskRunner(reporter, 0); - taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true, additionalFileMatcher })); + taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true, additionalFileMatcher })); addRunTasks(taskRunner, config); return taskRunner; } export function createTaskRunnerForTestServer(config: FullConfigInternal, reporter: ReporterV2): TaskRunner { const taskRunner = new TaskRunner(reporter, 0); - taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true })); + taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true })); addRunTasks(taskRunner, config); return taskRunner; } @@ -109,7 +110,7 @@ function addRunTasks(taskRunner: TaskRunner, config: FullConfigInternal export function createTaskRunnerForList(config: FullConfigInternal, reporter: ReporterV2, mode: 'in-process' | 'out-of-process', options: { failOnLoadErrors: boolean }): TaskRunner { const taskRunner = new TaskRunner(reporter, config.config.globalTimeout); - taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false })); + taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false, filterOnlyChanged: false })); taskRunner.addTask('report begin', createReportBeginTask()); return taskRunner; } @@ -223,12 +224,21 @@ function createListFilesTask(): Task { }; } -function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task { +function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, filterOnlyChanged: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task { return { setup: async (testRun, errors, softErrors) => { await collectProjectsAndTestFiles(testRun, !!options.doNotRunDepsOutsideProjectFilter, options.additionalFileMatcher); await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors); - testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly); + + let cliOnlyChangedMatcher: Matcher | undefined = undefined; + if (testRun.config.cliOnlyChanged && options.filterOnlyChanged) { + for (const plugin of testRun.config.plugins) + await plugin.instance?.populateDependencies?.(); + const changedFiles = await detectChangedTests(testRun.config.cliOnlyChanged, testRun.config.configDir); + cliOnlyChangedMatcher = file => changedFiles.has(file); + } + + testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly, cliOnlyChangedMatcher); testRun.failureTracker.onRootSuite(testRun.rootSuite); // Fail when no tests. if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard) { diff --git a/packages/playwright/src/runner/vcs.ts b/packages/playwright/src/runner/vcs.ts new file mode 100644 index 0000000000..160ca60840 --- /dev/null +++ b/packages/playwright/src/runner/vcs.ts @@ -0,0 +1,45 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import childProcess from 'child_process'; +import { affectedTestFiles } from '../transform/compilationCache'; +import path from 'path'; + +export async function detectChangedTests(baseCommit: string, configDir: string): Promise> { + function gitFileList(command: string) { + try { + return childProcess.execSync( + `git ${command}`, + { encoding: 'utf-8', stdio: 'pipe', cwd: configDir } + ).split('\n').filter(Boolean); + } catch (_error) { + const error = _error as childProcess.SpawnSyncReturns; + throw new Error([ + `Cannot detect changed files for --only-changed mode:`, + `git ${command}`, + '', + ...error.output, + ].join('\n')); + } + } + + const untrackedFiles = gitFileList(`ls-files --others --exclude-standard`).map(file => path.join(configDir, file)); + + const [gitRoot] = gitFileList('rev-parse --show-toplevel'); + const trackedFilesWithChanges = gitFileList(`diff ${baseCommit} --name-only`).map(file => path.join(gitRoot, file)); + + return new Set(affectedTestFiles([...untrackedFiles, ...trackedFilesWithChanges])); +} \ No newline at end of file diff --git a/tests/playwright-test/only-changed.spec.ts b/tests/playwright-test/only-changed.spec.ts new file mode 100644 index 0000000000..45b750cbe4 --- /dev/null +++ b/tests/playwright-test/only-changed.spec.ts @@ -0,0 +1,367 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test as baseTest, expect, playwrightCtConfigText } from './playwright-test-fixtures'; +import { execSync } from 'node:child_process'; + +const test = baseTest.extend<{ git(command: string): void }>({ + git: async ({}, use, testInfo) => { + const baseDir = testInfo.outputPath(); + + const git = (command: string) => execSync(`git ${command}`, { cwd: baseDir }); + + git(`init --initial-branch=main`); + git(`config --local user.name "Robert Botman"`); + git(`config --local user.email "botty@mcbotface.com"`); + + await use((command: string) => git(command)); + }, +}); + +test.slow(); + +test('should detect untracked files', async ({ runInlineTest, git, writeFiles }) => { + await writeFiles({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }); + + git(`add .`); + git(`commit -m init`); + + const result = await runInlineTest({ + 'c.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + ` + }, { 'only-changed': true }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.passed).toBe(0); + expect(result.output).toContain('c.spec.ts'); +}); + + +test('should detect changed files', async ({ runInlineTest, git, writeFiles }) => { + await writeFiles({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }); + + git(`add .`); + git(`commit -m init`); + + const result = await runInlineTest({ + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(3); }); + `, + }, { 'only-changed': true }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.passed).toBe(0); + expect(result.output).toContain('b.spec.ts'); +}); + +test('should diff based on base commit', async ({ runInlineTest, git, writeFiles }) => { + await writeFiles({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }); + + git(`add .`); + git(`commit -m init`); + + await writeFiles({ + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(3); }); + `, + }); + git('commit -a -m update'); + const result = await runInlineTest({}, { 'only-changed': `HEAD~1` }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.passed).toBe(0); + expect(result.output).toContain('b.spec.ts'); +}); + +test('should understand dependency structure', async ({ runInlineTest, git, writeFiles }) => { + await writeFiles({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + import { answer, question } from './utils'; + test('fails', () => { expect(question).toBe(answer); }); + `, + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + import { answer, question } from './utils'; + test('fails', () => { expect(question).toBe(answer); }); + `, + 'c.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + 'utils.ts': ` + export * from './answer'; + export * from './question'; + `, + 'answer.ts': ` + export const answer = 42; + `, + 'question.ts': ` + export const question = "???"; + `, + }); + + git(`add .`); + git(`commit -m init`); + + await writeFiles({ + 'question.ts': ` + export const question = "what is the answer to life the universe and everything"; + `, + }); + const result = await runInlineTest({}, { 'only-changed': true }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(2); + expect(result.passed).toBe(0); + expect(result.output).toContain('a.spec.ts'); + expect(result.output).toContain('b.spec.ts'); + expect(result.output).not.toContain('c.spec.ts'); +}); + +test('should support watch mode', async ({ git, writeFiles, runWatchTest }) => { + await writeFiles({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }); + + git(`add .`); + git(`commit -m init`); + + await writeFiles({ + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(3); }); + `, + }); + git(`commit -a -m update`); + + const testProcess = await runWatchTest({}, { 'only-changed': `HEAD~1` }); + await testProcess.waitForOutput('Waiting for file changes.'); + testProcess.clearOutput(); + testProcess.write('r'); + + await testProcess.waitForOutput('b.spec.ts:3:13 › fails'); + expect(testProcess.output).not.toContain('a.spec'); +}); + +test('should throw nice error message if git doesnt work', async ({ git, runInlineTest }) => { + const result = await runInlineTest({}, { 'only-changed': `this-commit-does-not-exist` }); + + expect(result.exitCode).toBe(1); + expect(result.output, 'contains our error message').toContain('Cannot detect changed files for --only-changed mode'); + expect(result.output, 'contains command').toContain('git diff this-commit-does-not-exist --name-only'); + expect(result.output, 'contains git command output').toContain('unknown revision or path not in the working tree'); +}); + +test('should suppport component tests', async ({ runInlineTest, git, writeFiles }) => { + await writeFiles({ + 'playwright.config.ts': playwrightCtConfigText, + 'playwright/index.html': ``, + 'playwright/index.ts': ` + `, + 'src/contents.ts': ` + export const content = "Button"; + `, + 'src/button.tsx': ` + import {content} from './contents'; + export const Button = () => ; + `, + 'src/helper.ts': ` + export { Button } from "./button"; + `, + 'src/button.test.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + import { Button } from './helper'; + + test('pass', async ({ mount }) => { + const component = await mount(); + await expect(component).toHaveText('Button'); + }); + `, + 'src/button2.test.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + import { Button } from './helper'; + + test('pass', async ({ mount }) => { + const component = await mount(); + await expect(component).toHaveText('Button'); + }); + `, + 'src/button3.test.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + + test('pass', async ({ mount }) => { + const component = await mount(

Hello World

); + await expect(component).toHaveText('Hello World'); + }); + `, + }); + + git(`add .`); + git(`commit -m "init"`); + + const result = await runInlineTest({}, { 'workers': 1, 'only-changed': true }); + + expect(result.exitCode).toBe(1); + expect(result.passed).toBe(0); + expect(result.failed).toBe(0); + expect(result.output).toContain('No tests found'); + + const result2 = await runInlineTest({ + 'src/button2.test.tsx': ` + import { test, expect } from '@playwright/experimental-ct-react'; + import { Button } from './helper'; + + test('pass', async ({ mount }) => { + const component = await mount(); + await expect(component).toHaveText('Different Button'); + }); + ` + }, { 'workers': 1, 'only-changed': true }); + + expect(result2.exitCode).toBe(1); + expect(result2.failed).toBe(1); + expect(result2.passed).toBe(0); + expect(result2.output).toContain('button2.test.tsx'); + expect(result2.output).not.toContain('button.test.tsx'); + expect(result2.output).not.toContain('button3.test.tsx'); + + git(`commit -am "update button2 test"`); + + const result3 = await runInlineTest({ + 'src/contents.ts': ` + export const content = 'Changed Content'; + ` + }, { 'workers': 1, 'only-changed': true }); + + expect(result3.exitCode).toBe(1); + expect(result3.failed).toBe(2); + expect(result3.passed).toBe(0); +}); + +test.describe('should work the same if being called in subdirectory', () => { + test('tracked file', async ({ runInlineTest, git, writeFiles }) => { + await writeFiles({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }); + + git(`add .`); + git(`commit -m init`); + + await writeFiles({ + 'tests/c.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + ` + }); + git(`add .`); + git(`commit -a -m "add test"`); + + const result = await runInlineTest({ + 'tests/c.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(3); }); + ` + }, { 'only-changed': true }, {}, { cwd: 'tests' }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.passed).toBe(0); + expect(result.output).toContain('c.spec.ts'); + }); + + test('untracked file', async ({ runInlineTest, git, writeFiles }) => { + await writeFiles({ + 'a.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + 'b.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(2); }); + `, + }); + + git(`add .`); + git(`commit -m init`); + + const result = await runInlineTest({ + 'tests/c.spec.ts': ` + import { test, expect } from '@playwright/test'; + test('fails', () => { expect(1).toBe(3); }); + ` + }, { 'only-changed': true }, {}, { cwd: 'tests' }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.passed).toBe(0); + expect(result.output).toContain('c.spec.ts'); + }); +}); + +test('UI mode is not supported', async ({ runInlineTest }) => { + const result = await runInlineTest({}, { 'only-changed': true, 'ui': true }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('--only-changed is not supported in UI mode'); +}); \ No newline at end of file diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index 4f0350cb59..86b07d7a80 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -246,7 +246,7 @@ type Fixtures = { deleteFile: (file: string) => Promise; runInlineTest: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise; runCLICommand: (files: Files, command: string, args?: string[], entryPoint?: string) => Promise<{ stdout: string, stderr: string, exitCode: number }>; - runWatchTest: (files: Files, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise; + runWatchTest: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise; interactWithTestRunner: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise; runTSC: (files: Files) => Promise; mergeReports: (reportFolder: string, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise @@ -288,7 +288,7 @@ export const test = base }, runWatchTest: async ({ interactWithTestRunner }, use, testInfo: TestInfo) => { - await use((files, env, options) => interactWithTestRunner(files, {}, { ...env, PWTEST_WATCH: '1' }, options)); + await use((files, params, env, options) => interactWithTestRunner(files, params, { ...env, PWTEST_WATCH: '1' }, options)); }, interactWithTestRunner: async ({ childProcess }, use, testInfo: TestInfo) => { diff --git a/tests/playwright-test/watch.spec.ts b/tests/playwright-test/watch.spec.ts index 49639ed9ff..946377a357 100644 --- a/tests/playwright-test/watch.spec.ts +++ b/tests/playwright-test/watch.spec.ts @@ -179,20 +179,20 @@ test('should perform initial run', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); }); test('should quit on Q', async ({ runWatchTest }) => { - const testProcess = await runWatchTest({}, {}); + const testProcess = await runWatchTest({}); await testProcess.waitForOutput('Waiting for file changes.'); testProcess.write('q'); await testProcess!.exited; }); test('should print help on H', async ({ runWatchTest }) => { - const testProcess = await runWatchTest({}, {}); + const testProcess = await runWatchTest({}); await testProcess.waitForOutput('Waiting for file changes.'); testProcess.write('h'); await testProcess.waitForOutput('to quit'); @@ -204,7 +204,7 @@ test('should run tests on Enter', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); testProcess.clearOutput(); @@ -220,7 +220,7 @@ test('should run tests on R', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); testProcess.clearOutput(); @@ -244,7 +244,7 @@ test('should run failed tests on F', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('fails', () => { expect(1).toBe(2); }); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:3:11 › passes'); await testProcess.waitForOutput('c.test.ts:3:11 › fails'); @@ -267,7 +267,7 @@ test('should respect file filter P', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -291,7 +291,7 @@ test('should respect project filter C', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes'); await testProcess.waitForOutput('[bar] › a.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -317,7 +317,7 @@ test('should respect file filter P and split files', async ({ runWatchTest }) => import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -341,7 +341,7 @@ test('should respect title filter T', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('title 2', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › title 1'); await testProcess.waitForOutput('b.test.ts:3:11 › title 2'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -369,7 +369,7 @@ test('should re-run failed tests on F > R', async ({ runWatchTest }) => { import { test, expect } from '@playwright/test'; test('fails', () => { expect(1).toBe(2); }); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:3:11 › passes'); await testProcess.waitForOutput('c.test.ts:3:11 › fails'); @@ -401,7 +401,7 @@ test('should run on changed files', async ({ runWatchTest, writeFiles }) => { import { test, expect } from '@playwright/test'; test('fails', () => { expect(1).toBe(2); }); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:3:11 › passes'); await testProcess.waitForOutput('c.test.ts:3:11 › fails'); @@ -434,7 +434,7 @@ test('should run on changed deps', async ({ runWatchTest, writeFiles }) => { 'helper.ts': ` console.log('old helper'); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:4:11 › passes'); await testProcess.waitForOutput('old helper'); @@ -467,7 +467,7 @@ test('should run on changed deps in ESM', async ({ runWatchTest, writeFiles }) = 'helper.ts': ` console.log('old helper'); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:7 › passes'); await testProcess.waitForOutput('b.test.ts:4:7 › passes'); await testProcess.waitForOutput('old helper'); @@ -498,7 +498,7 @@ test('should re-run changed files on R', async ({ runWatchTest, writeFiles }) => import { test, expect } from '@playwright/test'; test('fails', () => { expect(1).toBe(2); }); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:3:11 › passes'); await testProcess.waitForOutput('c.test.ts:3:11 › fails'); @@ -533,7 +533,7 @@ test('should not trigger on changes to non-tests', async ({ runWatchTest, writeF import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('b.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -559,7 +559,7 @@ test('should only watch selected projects', async ({ runWatchTest, writeFiles }) import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}, { additionalArgs: ['--project=foo'] }); + }, undefined, undefined, { additionalArgs: ['--project=foo'] }); await testProcess.waitForOutput('npx playwright test --project foo'); await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes'); expect(testProcess.output).not.toContain('[bar]'); @@ -589,7 +589,7 @@ test('should watch filtered files', async ({ runWatchTest, writeFiles }) => { import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}, { additionalArgs: ['a.test.ts'] }); + }, undefined, undefined, { additionalArgs: ['a.test.ts'] }); await testProcess.waitForOutput('npx playwright test a.test.ts'); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); expect(testProcess.output).not.toContain('b.test'); @@ -617,7 +617,7 @@ test('should not watch unfiltered files', async ({ runWatchTest, writeFiles }) = import { test, expect } from '@playwright/test'; test('passes', () => {}); `, - }, {}, { additionalArgs: ['a.test.ts'] }); + }, undefined, undefined, { additionalArgs: ['a.test.ts'] }); await testProcess.waitForOutput('npx playwright test a.test.ts'); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); expect(testProcess.output).not.toContain('b.test'); @@ -661,7 +661,7 @@ test('should run CT on changed deps', async ({ runWatchTest, writeFiles }) => { await expect(component).toHaveText('hello'); }); `, - }, {}); + }); await testProcess.waitForOutput('button.spec.tsx:4:11 › pass'); await testProcess.waitForOutput('link.spec.tsx:3:11 › pass'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -709,7 +709,7 @@ test('should run CT on indirect deps change', async ({ runWatchTest, writeFiles await expect(component).toHaveText('hello'); }); `, - }, {}); + }); await testProcess.waitForOutput('button.spec.tsx:4:11 › pass'); await testProcess.waitForOutput('link.spec.tsx:3:11 › pass'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -753,7 +753,7 @@ test('should run CT on indirect deps change ESM mode', async ({ runWatchTest, wr await expect(component).toHaveText('hello'); }); `, - }, {}); + }); await testProcess.waitForOutput('button.spec.tsx:4:7 › pass'); await testProcess.waitForOutput('link.spec.tsx:3:7 › pass'); await testProcess.waitForOutput('Waiting for file changes.'); @@ -786,7 +786,7 @@ test('should run global teardown before exiting', async ({ runWatchTest }) => { test('passes', async () => { }); `, - }, {}); + }); await testProcess.waitForOutput('a.test.ts:3:11 › passes'); await testProcess.waitForOutput('Waiting for file changes.'); testProcess.write('\x1B');