feat(test runner): --only-changed option (#31727)

Introduces an `--only-changed [base ref]` option.

`playwright test --only-changed` filters the test run to only run test
suites that have uncommitted changes.
`playwright test --only-changed=foo` runs only tests that were changed
since commit `foo`.

In pull request CI, this can be used to run changed tests first and fail
fast: `--only-changed=$GITHUB_BASE_REF`.
During local development, it can be used to quickly filter down to the
touched set of tests suites.
In some rare usecases, this can also help to cut down on CI usage for
pull requests. Tread with caution though.

File dependencies are taken into account to ensure that if you touched a
utility file, all relevant tests are still executed.

Closes https://github.com/microsoft/playwright/issues/15075

docs/src/ci-intro.md

@@ -383,6 +383,43 @@ jobs:
         PLAYWRIGHT_TEST_BASE_URL: ${{ github.event.deployment_status.target_url }}
 ```
 
+### Fail-Fast
+* langs: js
+
+Even with sharding enabled, large test suites can take very long to execute. Running changed tests first on PRs will give you a faster feedback loop and use less CI resources.
+
+```yml js title=".github/workflows/playwright.yml"
+name: Playwright Tests
+on:
+  push:
+    branches: [ main, master ]
+  pull_request:
+    branches: [ main, master ]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-node@v4
+      with:
+        node-version: 18
+    - name: Install dependencies
+      run: npm ci
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+    - name: Run changed Playwright tests
+      run: npx playwright test --only-changed=$GITHUB_BASE_REF
+      if: github.event_name == 'pull_request'
+    - name: Run Playwright tests
+      run: npx playwright test
+    - uses: actions/upload-artifact@v4
+      if: ${{ !cancelled() }}
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30
+```
 
 ## Create a Repo and Push to GitHub
 

docs/src/test-cli-js.md

@@ -93,6 +93,7 @@ Complete set of Playwright Test options is available in the [configuration file]
 | `--max-failures <N>` or `-x`| Stop after the first `N` test failures. Passing `-x` stops after the first failure.|
 | `--no-deps` | Ignore the dependencies between projects and behave as if they were not specified. |
 | `--output <dir>` | Directory for artifacts produced by tests, defaults to `test-results`. |
+| `--only-changed [ref]` | Only run tests that have been changed between working tree and "ref". Defaults to running all uncommitted changes with ref=HEAD. Only supports Git. |
 | `--pass-with-no-tests` | Allows the test suite to pass when no files are found. |
 | `--project <name>` | Only run tests from the specified [projects](./test-projects.md), supports '*' wildcard. Defaults to running all projects defined in the configuration file.|
 | `--quiet` | Whether to suppress stdout and stderr from the tests. |

packages/playwright-ct-core/src/vitePlugin.ts

@@ -69,6 +69,10 @@ export function createPlugin(): TestRunnerPlugin {
       if (stoppableServer)
         await new Promise(f => stoppableServer.stop(f));
     },
+
+    populateDependencies: async () => {
+      await buildBundle(config, configDir);
+    },
   };
 }
 
@@ -157,7 +161,7 @@ export async function buildBundle(config: FullConfig, configDir: string): Promis
   const viteConfig = await createConfig(dirs, config, frameworkPluginFactory, jsxInJS);
 
   if (sourcesDirty) {
-    // Only add out own plugin when we actually build / transform.
+    // Only add our own plugin when we actually build / transform.
     log('build');
     const depsCollector = new Map<string, string[]>();
     const buildConfig = mergeConfig(viteConfig, {

packages/playwright/src/common/config.ts

@@ -49,6 +49,7 @@ export class FullConfigInternal {
   cliArgs: string[] = [];
   cliGrep: string | undefined;
   cliGrepInvert: string | undefined;
+  cliOnlyChanged: string | undefined;
   cliProjectFilter?: string[];
   cliListOnly = false;
   cliPassWithNoTests?: boolean;

packages/playwright/src/plugins/index.ts

@@ -20,6 +20,7 @@ import type { ReporterV2 } from '../reporters/reporterV2';
 export interface TestRunnerPlugin {
   name: string;
   setup?(config: FullConfig, configDir: string, reporter: ReporterV2): Promise<void>;
+  populateDependencies?(): Promise<void>;
   begin?(suite: Suite): Promise<void>;
   end?(): Promise<void>;
   teardown?(): Promise<void>;

packages/playwright/src/program.ts

@@ -153,12 +153,14 @@ Examples:
   $ npx playwright merge-reports playwright-report`);
 }
 
-
 async function runTests(args: string[], opts: { [key: string]: any }) {
   await startProfiling();
   const cliOverrides = overridesFromOptions(opts);
 
   if (opts.ui || opts.uiHost || opts.uiPort) {
+    if (opts.onlyChanged)
+      throw new Error(`--only-changed is not supported in UI mode. If you'd like that to change, see https://github.com/microsoft/playwright/issues/15075 for more details.`);
+
     const status = await testServer.runUIMode(opts.config, {
       host: opts.uiHost,
       port: opts.uiPort ? +opts.uiPort : undefined,
@@ -192,6 +194,7 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
 
   config.cliArgs = args;
   config.cliGrep = opts.grep as string | undefined;
+  config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
   config.cliGrepInvert = opts.grepInvert as string | undefined;
   config.cliListOnly = !!opts.list;
   config.cliProjectFilter = opts.project || undefined;
@@ -352,6 +355,7 @@ const testOptions: [string, string][] = [
   ['--max-failures <N>', `Stop after the first N failures`],
   ['--no-deps', 'Do not run project dependencies'],
   ['--output <dir>', `Folder for output artifacts (default: "test-results")`],
+  ['--only-changed [ref]', `Only run tests that have been changed between 'HEAD' and 'ref'. Defaults to running all uncommitted changes. Only supports Git.`],
   ['--pass-with-no-tests', `Makes test run succeed even if no tests were found`],
   ['--project <project-name...>', `Only run tests from the specified list of projects, supports '*' wildcard (default: run all projects)`],
   ['--quiet', `Suppress stdio`],

packages/playwright/src/runner/loadUtils.ts

@@ -32,6 +32,7 @@ import { dependenciesForTestFile } from '../transform/compilationCache';
 import { sourceMapSupport } from '../utilsBundle';
 import type { RawSourceMap } from 'source-map';
 
+
 export async function collectProjectsAndTestFiles(testRun: TestRun, doNotRunTestsOutsideProjectFilter: boolean, additionalFileMatcher?: Matcher) {
   const config = testRun.config;
   const fsCache = new Map();
@@ -118,7 +119,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |
   }
 }
 
-export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<Suite> {
+export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean, additionalFileMatcher?: Matcher): Promise<Suite> {
   const config = testRun.config;
   // Create root suite, where each child will be a project suite with cloned file suites inside it.
   const rootSuite = new Suite('', 'root');
@@ -135,7 +136,8 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
 
     // Filter file suites for all projects.
     for (const [project, fileSuites] of testRun.projectSuites) {
-      const projectSuite = createProjectSuite(project, fileSuites);
+      const filteredFileSuites = additionalFileMatcher ? fileSuites.filter(fileSuite => additionalFileMatcher(fileSuite.location!.file)) : fileSuites;
+      const projectSuite = createProjectSuite(project, filteredFileSuites);
       projectSuites.set(project, projectSuite);
       const filteredProjectSuite = filterProjectSuite(projectSuite, { cliFileFilters, cliTitleMatcher, testIdMatcher: config.testIdMatcher });
       filteredProjectSuites.set(project, filteredProjectSuite);

packages/playwright/src/runner/reporters.ts

@@ -107,6 +107,8 @@ function computeCommandHash(config: FullConfigInternal) {
     command.cliGrep = config.cliGrep;
   if (config.cliGrepInvert)
     command.cliGrepInvert = config.cliGrepInvert;
+  if (config.cliOnlyChanged)
+    command.cliOnlyChanged = config.cliOnlyChanged;
   if (Object.keys(command).length)
     parts.push(calculateSha1(JSON.stringify(command)).substring(0, 7));
   return parts.join('-');

packages/playwright/src/runner/tasks.ts

@@ -31,6 +31,7 @@ import type { Matcher } from '../util';
 import { Suite } from '../common/test';
 import { buildDependentProjects, buildTeardownToSetupsMap, filterProjects } from './projectUtils';
 import { FailureTracker } from './failureTracker';
+import { detectChangedTests } from './vcs';
 
 const readDirAsync = promisify(fs.readdir);
 
@@ -64,7 +65,7 @@ export class TestRun {
 export function createTaskRunner(config: FullConfigInternal, reporter: ReporterV2): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, config.config.globalTimeout);
   addGlobalSetupTasks(taskRunner, config);
-  taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, failOnLoadErrors: true }));
+  taskRunner.addTask('load tests', createLoadTask('in-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: true }));
   addRunTasks(taskRunner, config);
   return taskRunner;
 }
@@ -77,14 +78,14 @@ export function createTaskRunnerForWatchSetup(config: FullConfigInternal, report
 
 export function createTaskRunnerForWatch(config: FullConfigInternal, reporter: ReporterV2, additionalFileMatcher?: Matcher): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, 0);
-  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true, additionalFileMatcher }));
+  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true, additionalFileMatcher }));
   addRunTasks(taskRunner, config);
   return taskRunner;
 }
 
 export function createTaskRunnerForTestServer(config: FullConfigInternal, reporter: ReporterV2): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, 0);
-  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }));
+  taskRunner.addTask('load tests', createLoadTask('out-of-process', { filterOnly: true, filterOnlyChanged: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }));
   addRunTasks(taskRunner, config);
   return taskRunner;
 }
@@ -109,7 +110,7 @@ function addRunTasks(taskRunner: TaskRunner<TestRun>, config: FullConfigInternal
 
 export function createTaskRunnerForList(config: FullConfigInternal, reporter: ReporterV2, mode: 'in-process' | 'out-of-process', options: { failOnLoadErrors: boolean }): TaskRunner<TestRun> {
   const taskRunner = new TaskRunner<TestRun>(reporter, config.config.globalTimeout);
-  taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false }));
+  taskRunner.addTask('load tests', createLoadTask(mode, { ...options, filterOnly: false, filterOnlyChanged: false }));
   taskRunner.addTask('report begin', createReportBeginTask());
   return taskRunner;
 }
@@ -223,12 +224,21 @@ function createListFilesTask(): Task<TestRun> {
   };
 }
 
-function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task<TestRun> {
+function createLoadTask(mode: 'out-of-process' | 'in-process', options: { filterOnly: boolean, filterOnlyChanged: boolean, failOnLoadErrors: boolean, doNotRunDepsOutsideProjectFilter?: boolean, additionalFileMatcher?: Matcher }): Task<TestRun> {
   return {
     setup: async (testRun, errors, softErrors) => {
       await collectProjectsAndTestFiles(testRun, !!options.doNotRunDepsOutsideProjectFilter, options.additionalFileMatcher);
       await loadFileSuites(testRun, mode, options.failOnLoadErrors ? errors : softErrors);
-      testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
+
+      let cliOnlyChangedMatcher: Matcher | undefined = undefined;
+      if (testRun.config.cliOnlyChanged && options.filterOnlyChanged) {
+        for (const plugin of testRun.config.plugins)
+          await plugin.instance?.populateDependencies?.();
+        const changedFiles = await detectChangedTests(testRun.config.cliOnlyChanged, testRun.config.configDir);
+        cliOnlyChangedMatcher = file => changedFiles.has(file);
+      }
+
+      testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly, cliOnlyChangedMatcher);
       testRun.failureTracker.onRootSuite(testRun.rootSuite);
       // Fail when no tests.
       if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard) {

packages/playwright/src/runner/vcs.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import childProcess from 'child_process';
+import { affectedTestFiles } from '../transform/compilationCache';
+import path from 'path';
+
+export async function detectChangedTests(baseCommit: string, configDir: string): Promise<Set<string>> {
+  function gitFileList(command: string) {
+    try {
+      return childProcess.execSync(
+          `git ${command}`,
+          { encoding: 'utf-8', stdio: 'pipe', cwd: configDir }
+      ).split('\n').filter(Boolean);
+    } catch (_error) {
+      const error = _error as childProcess.SpawnSyncReturns<string>;
+      throw new Error([
+        `Cannot detect changed files for --only-changed mode:`,
+        `git ${command}`,
+        '',
+        ...error.output,
+      ].join('\n'));
+    }
+  }
+
+  const untrackedFiles = gitFileList(`ls-files --others --exclude-standard`).map(file => path.join(configDir, file));
+
+  const [gitRoot] = gitFileList('rev-parse --show-toplevel');
+  const trackedFilesWithChanges = gitFileList(`diff ${baseCommit} --name-only`).map(file => path.join(gitRoot, file));
+
+  return new Set(affectedTestFiles([...untrackedFiles, ...trackedFilesWithChanges]));
+}

tests/playwright-test/only-changed.spec.ts

@@ -0,0 +1,367 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test as baseTest, expect, playwrightCtConfigText } from './playwright-test-fixtures';
+import { execSync } from 'node:child_process';
+
+const test = baseTest.extend<{ git(command: string): void }>({
+  git: async ({}, use, testInfo) => {
+    const baseDir = testInfo.outputPath();
+
+    const git = (command: string) => execSync(`git ${command}`, { cwd: baseDir });
+
+    git(`init --initial-branch=main`);
+    git(`config --local user.name "Robert Botman"`);
+    git(`config --local user.email "botty@mcbotface.com"`);
+
+    await use((command: string) => git(command));
+  },
+});
+
+test.slow();
+
+test('should detect untracked files', async ({ runInlineTest, git, writeFiles }) => {
+  await writeFiles({
+    'a.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+    'b.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+  });
+
+  git(`add .`);
+  git(`commit -m init`);
+
+  const result = await runInlineTest({
+    'c.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('fails', () => { expect(1).toBe(2); });
+    `
+  }, { 'only-changed': true });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('c.spec.ts');
+});
+
+
+test('should detect changed files', async ({ runInlineTest, git, writeFiles }) => {
+  await writeFiles({
+    'a.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+    'b.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+  });
+
+  git(`add .`);
+  git(`commit -m init`);
+
+  const result = await runInlineTest({
+    'b.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(3); });
+      `,
+  }, { 'only-changed': true });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('b.spec.ts');
+});
+
+test('should diff based on base commit', async ({ runInlineTest, git, writeFiles }) => {
+  await writeFiles({
+    'a.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+    'b.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+  });
+
+  git(`add .`);
+  git(`commit -m init`);
+
+  await writeFiles({
+    'b.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(3); });
+      `,
+  });
+  git('commit -a -m update');
+  const result = await runInlineTest({}, { 'only-changed': `HEAD~1` });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('b.spec.ts');
+});
+
+test('should understand dependency structure', async ({ runInlineTest, git, writeFiles }) => {
+  await writeFiles({
+    'a.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    import { answer, question } from './utils';
+    test('fails', () => { expect(question).toBe(answer); });
+  `,
+    'b.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    import { answer, question } from './utils';
+    test('fails', () => { expect(question).toBe(answer); });
+  `,
+    'c.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+    'utils.ts': `
+    export * from './answer';
+    export * from './question';
+  `,
+    'answer.ts': `
+    export const answer = 42;
+  `,
+    'question.ts': `
+    export const question = "???";
+  `,
+  });
+
+  git(`add .`);
+  git(`commit -m init`);
+
+  await writeFiles({
+    'question.ts': `
+        export const question = "what is the answer to life the universe and everything";
+      `,
+  });
+  const result = await runInlineTest({}, { 'only-changed': true });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(2);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain('a.spec.ts');
+  expect(result.output).toContain('b.spec.ts');
+  expect(result.output).not.toContain('c.spec.ts');
+});
+
+test('should support watch mode', async ({ git, writeFiles, runWatchTest }) => {
+  await writeFiles({
+    'a.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+    'b.spec.ts': `
+    import { test, expect } from '@playwright/test';
+    test('fails', () => { expect(1).toBe(2); });
+  `,
+  });
+
+  git(`add .`);
+  git(`commit -m init`);
+
+  await writeFiles({
+    'b.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(3); });
+      `,
+  });
+  git(`commit -a -m update`);
+
+  const testProcess = await runWatchTest({}, { 'only-changed': `HEAD~1` });
+  await testProcess.waitForOutput('Waiting for file changes.');
+  testProcess.clearOutput();
+  testProcess.write('r');
+
+  await testProcess.waitForOutput('b.spec.ts:3:13 › fails');
+  expect(testProcess.output).not.toContain('a.spec');
+});
+
+test('should throw nice error message if git doesnt work', async ({ git, runInlineTest }) => {
+  const result = await runInlineTest({}, { 'only-changed': `this-commit-does-not-exist` });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.output, 'contains our error message').toContain('Cannot detect changed files for --only-changed mode');
+  expect(result.output, 'contains command').toContain('git diff this-commit-does-not-exist --name-only');
+  expect(result.output, 'contains git command output').toContain('unknown revision or path not in the working tree');
+});
+
+test('should suppport component tests', async ({ runInlineTest, git, writeFiles }) => {
+  await writeFiles({
+    'playwright.config.ts': playwrightCtConfigText,
+    'playwright/index.html': `<script type="module" src="./index.ts"></script>`,
+    'playwright/index.ts': `
+    `,
+    'src/contents.ts': `
+      export const content = "Button";
+    `,
+    'src/button.tsx': `
+      import {content} from './contents';
+      export const Button = () => <button>{content}</button>;
+    `,
+    'src/helper.ts': `
+      export { Button } from "./button";
+    `,
+    'src/button.test.tsx': `
+      import { test, expect } from '@playwright/experimental-ct-react';
+      import { Button } from './helper';
+
+      test('pass', async ({ mount }) => {
+        const component = await mount(<Button></Button>);
+        await expect(component).toHaveText('Button');
+      });
+    `,
+    'src/button2.test.tsx': `
+      import { test, expect } from '@playwright/experimental-ct-react';
+      import { Button } from './helper';
+
+      test('pass', async ({ mount }) => {
+        const component = await mount(<Button></Button>);
+        await expect(component).toHaveText('Button');
+      });
+    `,
+    'src/button3.test.tsx': `
+      import { test, expect } from '@playwright/experimental-ct-react';
+
+      test('pass', async ({ mount }) => {
+        const component = await mount(<p>Hello World</p>);
+        await expect(component).toHaveText('Hello World');
+      });
+    `,
+  });
+
+  git(`add .`);
+  git(`commit -m "init"`);
+
+  const result = await runInlineTest({}, { 'workers': 1, 'only-changed': true });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.failed).toBe(0);
+  expect(result.output).toContain('No tests found');
+
+  const result2 = await runInlineTest({
+    'src/button2.test.tsx': `
+      import { test, expect } from '@playwright/experimental-ct-react';
+      import { Button } from './helper';
+
+      test('pass', async ({ mount }) => {
+        const component = await mount(<Button></Button>);
+        await expect(component).toHaveText('Different Button');
+      });
+    `
+  }, { 'workers': 1, 'only-changed': true });
+
+  expect(result2.exitCode).toBe(1);
+  expect(result2.failed).toBe(1);
+  expect(result2.passed).toBe(0);
+  expect(result2.output).toContain('button2.test.tsx');
+  expect(result2.output).not.toContain('button.test.tsx');
+  expect(result2.output).not.toContain('button3.test.tsx');
+
+  git(`commit -am "update button2 test"`);
+
+  const result3 = await runInlineTest({
+    'src/contents.ts': `
+      export const content = 'Changed Content';
+    `
+  }, { 'workers': 1, 'only-changed': true });
+
+  expect(result3.exitCode).toBe(1);
+  expect(result3.failed).toBe(2);
+  expect(result3.passed).toBe(0);
+});
+
+test.describe('should work the same if being called in subdirectory', () => {
+  test('tracked file', async ({ runInlineTest, git, writeFiles }) => {
+    await writeFiles({
+      'a.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('fails', () => { expect(1).toBe(2); });
+    `,
+      'b.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('fails', () => { expect(1).toBe(2); });
+    `,
+    });
+
+    git(`add .`);
+    git(`commit -m init`);
+
+    await writeFiles({
+      'tests/c.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(2); });
+      `
+    });
+    git(`add .`);
+    git(`commit -a -m "add test"`);
+
+    const result = await runInlineTest({
+      'tests/c.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(3); });
+      `
+    }, { 'only-changed': true }, {}, { cwd: 'tests' });
+
+    expect(result.exitCode).toBe(1);
+    expect(result.failed).toBe(1);
+    expect(result.passed).toBe(0);
+    expect(result.output).toContain('c.spec.ts');
+  });
+
+  test('untracked file', async ({ runInlineTest, git, writeFiles }) => {
+    await writeFiles({
+      'a.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('fails', () => { expect(1).toBe(2); });
+    `,
+      'b.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('fails', () => { expect(1).toBe(2); });
+    `,
+    });
+
+    git(`add .`);
+    git(`commit -m init`);
+
+    const result = await runInlineTest({
+      'tests/c.spec.ts': `
+        import { test, expect } from '@playwright/test';
+        test('fails', () => { expect(1).toBe(3); });
+      `
+    }, { 'only-changed': true }, {}, { cwd: 'tests' });
+
+    expect(result.exitCode).toBe(1);
+    expect(result.failed).toBe(1);
+    expect(result.passed).toBe(0);
+    expect(result.output).toContain('c.spec.ts');
+  });
+});
+
+test('UI mode is not supported', async ({ runInlineTest }) => {
+  const result = await runInlineTest({}, { 'only-changed': true, 'ui': true });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain('--only-changed is not supported in UI mode');
+});

tests/playwright-test/playwright-test-fixtures.ts

@@ -246,7 +246,7 @@ type Fixtures = {
   deleteFile: (file: string) => Promise<void>;
   runInlineTest: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<RunResult>;
   runCLICommand: (files: Files, command: string, args?: string[], entryPoint?: string) => Promise<{ stdout: string, stderr: string, exitCode: number }>;
-  runWatchTest: (files: Files, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<TestChildProcess>;
+  runWatchTest: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<TestChildProcess>;
   interactWithTestRunner: (files: Files, params?: Params, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<TestChildProcess>;
   runTSC: (files: Files) => Promise<TSCResult>;
   mergeReports: (reportFolder: string, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise<CliRunResult>
@@ -288,7 +288,7 @@ export const test = base
       },
 
       runWatchTest: async ({ interactWithTestRunner }, use, testInfo: TestInfo) => {
-        await use((files, env, options) => interactWithTestRunner(files, {}, { ...env, PWTEST_WATCH: '1' }, options));
+        await use((files, params, env, options) => interactWithTestRunner(files, params, { ...env, PWTEST_WATCH: '1' }, options));
       },
 
       interactWithTestRunner: async ({ childProcess }, use, testInfo: TestInfo) => {

tests/playwright-test/watch.spec.ts

@@ -179,20 +179,20 @@ test('should perform initial run', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
 });
 
 test('should quit on Q', async ({ runWatchTest }) => {
-  const testProcess = await runWatchTest({}, {});
+  const testProcess = await runWatchTest({});
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.write('q');
   await testProcess!.exited;
 });
 
 test('should print help on H', async ({ runWatchTest }) => {
-  const testProcess = await runWatchTest({}, {});
+  const testProcess = await runWatchTest({});
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.write('h');
   await testProcess.waitForOutput('to quit');
@@ -204,7 +204,7 @@ test('should run tests on Enter', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
@@ -220,7 +220,7 @@ test('should run tests on R', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.clearOutput();
@@ -244,7 +244,7 @@ test('should run failed tests on F', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('fails', () => { expect(1).toBe(2); });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
@@ -267,7 +267,7 @@ test('should respect file filter P', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -291,7 +291,7 @@ test('should respect project filter C', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('[bar] › a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -317,7 +317,7 @@ test('should respect file filter P and split files', async ({ runWatchTest }) =>
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -341,7 +341,7 @@ test('should respect title filter T', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('title 2', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › title 1');
   await testProcess.waitForOutput('b.test.ts:3:11 › title 2');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -369,7 +369,7 @@ test('should re-run failed tests on F > R', async ({ runWatchTest }) => {
       import { test, expect } from '@playwright/test';
       test('fails', () => { expect(1).toBe(2); });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
@@ -401,7 +401,7 @@ test('should run on changed files', async ({ runWatchTest, writeFiles }) => {
       import { test, expect } from '@playwright/test';
       test('fails', () => { expect(1).toBe(2); });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
@@ -434,7 +434,7 @@ test('should run on changed deps', async ({ runWatchTest, writeFiles }) => {
     'helper.ts': `
       console.log('old helper');
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:4:11 › passes');
   await testProcess.waitForOutput('old helper');
@@ -467,7 +467,7 @@ test('should run on changed deps in ESM', async ({ runWatchTest, writeFiles }) =
     'helper.ts': `
       console.log('old helper');
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:7 › passes');
   await testProcess.waitForOutput('b.test.ts:4:7 › passes');
   await testProcess.waitForOutput('old helper');
@@ -498,7 +498,7 @@ test('should re-run changed files on R', async ({ runWatchTest, writeFiles }) =>
       import { test, expect } from '@playwright/test';
       test('fails', () => { expect(1).toBe(2); });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('c.test.ts:3:11 › fails');
@@ -533,7 +533,7 @@ test('should not trigger on changes to non-tests', async ({ runWatchTest, writeF
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('b.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -559,7 +559,7 @@ test('should only watch selected projects', async ({ runWatchTest, writeFiles })
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {}, { additionalArgs: ['--project=foo'] });
+  }, undefined, undefined, { additionalArgs: ['--project=foo'] });
   await testProcess.waitForOutput('npx playwright test --project foo');
   await testProcess.waitForOutput('[foo] › a.test.ts:3:11 › passes');
   expect(testProcess.output).not.toContain('[bar]');
@@ -589,7 +589,7 @@ test('should watch filtered files', async ({ runWatchTest, writeFiles }) => {
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {}, { additionalArgs: ['a.test.ts'] });
+  }, undefined, undefined, { additionalArgs: ['a.test.ts'] });
   await testProcess.waitForOutput('npx playwright test a.test.ts');
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   expect(testProcess.output).not.toContain('b.test');
@@ -617,7 +617,7 @@ test('should not watch unfiltered files', async ({ runWatchTest, writeFiles }) =
       import { test, expect } from '@playwright/test';
       test('passes', () => {});
     `,
-  }, {}, { additionalArgs: ['a.test.ts'] });
+  }, undefined, undefined, { additionalArgs: ['a.test.ts'] });
   await testProcess.waitForOutput('npx playwright test a.test.ts');
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   expect(testProcess.output).not.toContain('b.test');
@@ -661,7 +661,7 @@ test('should run CT on changed deps', async ({ runWatchTest, writeFiles }) => {
         await expect(component).toHaveText('hello');
       });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('button.spec.tsx:4:11 › pass');
   await testProcess.waitForOutput('link.spec.tsx:3:11 › pass');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -709,7 +709,7 @@ test('should run CT on indirect deps change', async ({ runWatchTest, writeFiles
         await expect(component).toHaveText('hello');
       });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('button.spec.tsx:4:11 › pass');
   await testProcess.waitForOutput('link.spec.tsx:3:11 › pass');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -753,7 +753,7 @@ test('should run CT on indirect deps change ESM mode', async ({ runWatchTest, wr
         await expect(component).toHaveText('hello');
       });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('button.spec.tsx:4:7 › pass');
   await testProcess.waitForOutput('link.spec.tsx:3:7 › pass');
   await testProcess.waitForOutput('Waiting for file changes.');
@@ -786,7 +786,7 @@ test('should run global teardown before exiting', async ({ runWatchTest }) => {
       test('passes', async () => {
       });
     `,
-  }, {});
+  });
   await testProcess.waitForOutput('a.test.ts:3:11 › passes');
   await testProcess.waitForOutput('Waiting for file changes.');
   testProcess.write('\x1B');