fix(runner): run all setup files when there is test.only (#18967)

packages/playwright-test/src/runner.ts

@@ -327,8 +327,13 @@ export class Runner {
     }
 
     // Filter only.
-    if (!options.listOnly)
+    if (!options.listOnly) {
-      filterOnly(preprocessRoot);
+      const onlyItems = preprocessRoot._getOnlyItems();
+      if (onlyItems.length) {
+        const hasOnlyInSetup = onlyItems.some(item => setupFiles.has(item._requireFile));
+        filterOnly(preprocessRoot, hasOnlyInSetup ? new Set() : setupFiles);
+      }
+    }
 
     // Generate projects.
     const fileSuites = new Map<string, Suite>();
@@ -671,13 +676,13 @@ export class Runner {
   }
 }
 
-function filterOnly(suite: Suite) {
+function filterOnly(suite: Suite, doNotFilterFiles: Set<string>) {
   const suiteFilter = (suite: Suite) => suite._only;
-  const testFilter = (test: TestCase) => test._only;
+  const testFilter = (test: TestCase) => doNotFilterFiles.has(test._requireFile) || test._only;
   return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter);
 }
 
-function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[], setupFiles: Set<string>) {
+function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[], doNotFilterFiles: Set<string>) {
   const filterWithLine = !!focusedTestFileLines.find(f => f.line !== null);
   if (!filterWithLine)
     return;
@@ -692,7 +697,7 @@ function filterByFocusedLine(suite: Suite, focusedTestFileLines: TestFileFilter[
     return !!suite.location && testFileLineMatches(suite.location.file, suite.location.line, suite.location.column);
   };
   // Project setup files are always included.
-  const testFilter = (test: TestCase) => setupFiles.has(test._requireFile) || testFileLineMatches(test.location.file, test.location.line, test.location.column);
+  const testFilter = (test: TestCase) => doNotFilterFiles.has(test._requireFile) || testFileLineMatches(test.location.file, test.location.line, test.location.column);
   return filterSuite(suite, suiteFilter, testFilter);
 }
 

tests/playwright-test/project-setup.spec.ts

@@ -461,6 +461,44 @@ test('should allow .only in setup files', async ({ runGroups }, testInfo) => {
   expect(passed).toBe(2);
 });
 
+test('should allow describe.only in setup files', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*.setup.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+      test('test4', async () => { });
+    `,
+    'a.setup.ts': `
+      const { test } = pwt;
+      test.describe.only('main', () => {
+        test('setup1', async () => { });
+        test('setup2', async () => { });
+      });
+      test('setup3', async () => { });
+    `,
+  };
+
+  const { exitCode, passed, timeline, output } =  await runGroups(files);
+  expect(output).toContain('Running 2 tests using 1 worker');
+  expect(output).toContain('[p1] › a.setup.ts:6:9 › main › setup1');
+  expect(output).toContain('[p1] › a.setup.ts:7:9 › main › setup2');
+  expect(fileNames(timeline)).toEqual(['a.setup.ts']);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(2);
+});
+
+
 test('should allow .only in both setup and test files', async ({ runGroups }, testInfo) => {
   const files = {
     'playwright.config.ts': `
@@ -493,6 +531,45 @@ test('should allow .only in both setup and test files', async ({ runGroups }, te
   expect(output).toContain('[p1] › a.test.ts:7:12 › test2');
 });
 
+test('should run full setup when there is test.only', async ({ runGroups }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*.setup.ts/,
+          },
+        ]
+      };`,
+    'a.test.ts': `
+      const { test } = pwt;
+      test.only('test1', async () => { });
+      test('test2', async () => { });
+      test('test3', async () => { });
+      test('test4', async () => { });
+    `,
+    'a.setup.ts': `
+      const { test } = pwt;
+      test('setup1', async () => { });
+      test('setup2', async () => { });
+    `,
+    'b.setup.ts': `
+      const { test } = pwt;
+      test('setup3', async () => { });
+    `,
+  };
+
+  const { exitCode, passed, output } =  await runGroups(files);
+  expect(exitCode).toBe(0);
+  expect(passed).toBe(4);
+  expect(output).toContain('Running 4 tests using 2 workers');
+  expect(output).toContain('[p1] › b.setup.ts:5:7 › setup3');
+  expect(output).toContain('[p1] › a.setup.ts:5:7 › setup1');
+  expect(output).toContain('[p1] › a.setup.ts:6:7 › setup2');
+  expect(output).toContain('[p1] › a.test.ts:6:12 › test1');
+});
+
 test('should allow filtering setup by file:line', async ({ runGroups }, testInfo) => {
   const files = {
     'playwright.config.ts': `