chore: focus by line should override only (#11427)

packages/playwright-test/src/runner.ts

@@ -221,9 +221,15 @@ export class Runner {
     if (config.globalSetup && !list)
       globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig());
     try {
+      // 1. Add all tests.
       const preprocessRoot = new Suite('');
       for (const file of allTestFiles)
         preprocessRoot._addSuite(await this._loader.loadTestFile(file));
+
+      // 2. Filter tests to respect column filter.
+      filterByFocusedLine(preprocessRoot, testFileReFilters);
+
+      // 3. Complain about only.
       if (config.forbidOnly) {
         const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
         if (onlyTestsAndSuites.length > 0) {
@@ -231,13 +237,16 @@ export class Runner {
           return { status: 'failed' };
         }
       }
+
+      // 4. Filter only
+      filterOnly(preprocessRoot);
+
+      // 5. Complain about clashing.
       const clashingTests = getClashingTestsPerSuite(preprocessRoot);
       if (clashingTests.size > 0) {
         this._reporter.onError?.(createDuplicateTitlesError(config, clashingTests));
         return { status: 'failed' };
       }
-      filterOnly(preprocessRoot);
-      filterByFocusedLine(preprocessRoot, testFileReFilters);
 
       const fileSuites = new Map<string, Suite>();
       for (const fileSuite of preprocessRoot.suites)
@@ -305,7 +314,7 @@ export class Runner {
         }
 
         testGroups = shardGroups;
-        filterSuite(rootSuite, () => false, test => shardTests.has(test));
+        filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test));
         total = rootSuite.allTests().length;
       }
       (config as any).__testGroupsCount = testGroups.length;
@@ -374,21 +383,27 @@ export class Runner {
 function filterOnly(suite: Suite) {
   const suiteFilter = (suite: Suite) => suite._only;
   const testFilter = (test: TestCase) => test._only;
-  return filterSuite(suite, suiteFilter, testFilter);
+  return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter);
 }
 
 function filterByFocusedLine(suite: Suite, focusedTestFileLines: FilePatternFilter[]) {
+  const filterWithLine = !!focusedTestFileLines.find(f => f.line !== null);
+  if (!filterWithLine)
+    return;
+
   const testFileLineMatches = (testFileName: string, testLine: number) => focusedTestFileLines.some(({ re, line }) => {
     re.lastIndex = 0;
     return re.test(testFileName) && (line === testLine || line === null);
   });
-  const suiteFilter = (suite: Suite) => !!suite.location && testFileLineMatches(suite.location.file, suite.location.line);
+  const suiteFilter = (suite: Suite) => {
+    return !!suite.location && testFileLineMatches(suite.location.file, suite.location.line);
+  };
   const testFilter = (test: TestCase) => testFileLineMatches(test.location.file, test.location.line);
   return filterSuite(suite, suiteFilter, testFilter);
 }
 
-function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
-  const onlySuites = suite.suites.filter(child => filterSuite(child, suiteFilter, testFilter) || suiteFilter(child));
+function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
+  const onlySuites = suite.suites.filter(child => filterSuiteWithOnlySemantics(child, suiteFilter, testFilter) || suiteFilter(child));
   const onlyTests = suite.tests.filter(testFilter);
   const onlyEntries = new Set([...onlySuites, ...onlyTests]);
   if (onlyEntries.size) {
@@ -400,6 +415,16 @@ function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, test
   return false;
 }
 
+function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) {
+  for (const child of suite.suites) {
+    if (!suiteFilter(child))
+      filterSuite(child, suiteFilter, testFilter);
+  }
+  suite.tests = suite.tests.filter(testFilter);
+  const entries = new Set([...suite.suites, ...suite.tests]);
+  suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order.
+}
+
 async function collectFiles(testDir: string): Promise<string[]> {
   type Rule = {
     dir: string;

tests/config/default.config.ts

@@ -14,6 +14,8 @@
  * limitations under the License.
  */
 
+// @playwright.config
+
 import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test';
 import * as path from 'path';
 import { TestModeWorkerOptions } from './testModeFixtures';
@@ -30,7 +32,9 @@ const getExecutablePath = (browserName: BrowserName) => {
     return process.env.WKPATH;
 };
 
-const mode = (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service');
+const mode = process.env.PW_OUT_OF_PROCESS ?
+  'driver' :
+  (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service');
 const headed = !!process.env.HEADFUL;
 const channel = process.env.PWTEST_CHANNEL as any;
 const video = !!process.env.PWTEST_VIDEO;

tests/playwright-test/command-line-filter.spec.ts

@@ -0,0 +1,147 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+
+test('should filter by file name', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+    'b.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+  }, undefined, undefined, { additionalArgs: ['a.spec.ts'] });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toContain('1) a.spec.ts');
+});
+
+test('should filter by folder', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'foo/x.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+    'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+    'bar/x.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+    'bar/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+  }, undefined, undefined, { additionalArgs: ['bar'] });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(2);
+  expect(result.output).toMatch(/bar[\\/]x.spec.ts/);
+  expect(result.output).toMatch(/bar[\\/]y.spec.ts/);
+});
+
+test('should filter by line', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'foo/x.spec.ts': `
+      pwt.test('one', () => { expect(1).toBe(2); });
+      pwt.test('two', () => { expect(1).toBe(2); });
+      pwt.test('three', () => { expect(1).toBe(2); });
+      `,
+    'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+  }, undefined, undefined, { additionalArgs: ['x.spec.ts:6'] });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toMatch(/x\.spec\.ts.*two/);
+});
+
+test('line should override focused test', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'foo/x.spec.ts': `
+      pwt.test.only('one', () => { expect(1).toBe(2); });
+      pwt.test('two', () => { expect(1).toBe(2); });
+      pwt.test.only('three', () => { expect(1).toBe(2); });
+      `,
+    'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+  }, undefined, undefined, { additionalArgs: ['x.spec.ts:6'] });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.output).toMatch(/x\.spec\.ts.*two/);
+});
+
+test('should merge filtered line and filtered file', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'foo/x.spec.ts': `
+      pwt.test('one', () => { expect(1).toBe(2); });
+      pwt.test('two', () => { expect(1).toBe(2); });
+      pwt.test('three', () => { expect(1).toBe(2); });
+      `,
+    'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+  }, undefined, undefined, { additionalArgs: ['x.spec.ts:6', 'x.spec.ts'] });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(3);
+});
+
+test('should run nothing for missing line', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'foo/x.spec.ts': `
+      pwt.test('one', () => { expect(1).toBe(2); });
+      pwt.test('two', () => { expect(1).toBe(2); });
+      pwt.test('three', () => { expect(1).toBe(2); });
+      `,
+    'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`,
+  }, undefined, undefined, { additionalArgs: ['x.spec.ts:10', 'y.spec.ts'] });
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+});
+
+test('should focus a single nested test spec', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'foo.test.ts': `
+      const { test } = pwt;
+      test('pass1', ({}) => {});
+      test.describe('suite-1', () => {
+        test.describe('suite-2', () => {
+          test('pass2', ({}) => {});
+        });
+      });
+      test('pass3', ({}) => {});
+    `,
+    'bar.test.ts': `
+      const { test } = pwt;
+      test('pass3', ({}) => {});
+    `,
+    'noooo.test.ts': `
+      const { test } = pwt;
+      test('no-pass1', ({}) => {});
+    `,
+  }, {}, {}, { additionalArgs: ['foo.test.ts:9', 'bar.test.ts'] });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  expect(result.skipped).toBe(0);
+  expect(result.report.suites[0].specs[0].title).toEqual('pass3');
+  expect(result.report.suites[1].suites[0].suites[0].specs[0].title).toEqual('pass2');
+});
+
+test('should focus a single test suite', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'foo.test.ts': `
+      const { test } = pwt;
+      test('pass1', ({}) => {});
+      test.describe('suite-1', () => {
+        test.describe('suite-2', () => {
+          test('pass2', ({}) => {});
+          test('pass3', ({}) => {});
+        });
+      });
+      test('pass4', ({}) => {});
+    `,
+    'bar.test.ts': `
+      const { test } = pwt;
+      test('no-pass1', ({}) => {});
+    `,
+  }, {}, {}, { additionalArgs: ['foo.test.ts:8'] });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  expect(result.skipped).toBe(0);
+  expect(result.report.suites[0].suites[0].suites[0].specs[0].title).toEqual('pass2');
+  expect(result.report.suites[0].suites[0].suites[0].specs[1].title).toEqual('pass3');
+});

tests/playwright-test/test-ignore.spec.ts

@@ -248,78 +248,6 @@ test('should match case insensitive', async ({ runInlineTest }) => {
   expect(result.exitCode).toBe(0);
 });
 
-test('should focus a single test spec', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'foo.test.ts': `
-      const { test } = pwt;
-      test('pass1', ({}) => {});
-      test('pass2', ({}) => {});
-      test('pass3', ({}) => {});
-    `,
-    'bar.test.ts': `
-      const { test } = pwt;
-      test('no-pass1', ({}) => {});
-    `,
-  }, {}, {}, { additionalArgs: ['foo.test.ts:7'] });
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(1);
-  expect(result.skipped).toBe(0);
-  expect(result.report.suites[0].specs[0].title).toEqual('pass2');
-});
-
-test('should focus a single nested test spec', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'foo.test.ts': `
-      const { test } = pwt;
-      test('pass1', ({}) => {});
-      test.describe('suite-1', () => {
-        test.describe('suite-2', () => {
-          test('pass2', ({}) => {});
-        });
-      });
-      test('pass3', ({}) => {});
-    `,
-    'bar.test.ts': `
-      const { test } = pwt;
-      test('pass3', ({}) => {});
-    `,
-    'noooo.test.ts': `
-      const { test } = pwt;
-      test('no-pass1', ({}) => {});
-    `,
-  }, {}, {}, { additionalArgs: ['foo.test.ts:9', 'bar.test.ts'] });
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(2);
-  expect(result.skipped).toBe(0);
-  expect(result.report.suites[0].specs[0].title).toEqual('pass3');
-  expect(result.report.suites[1].suites[0].suites[0].specs[0].title).toEqual('pass2');
-});
-
-test('should focus a single test suite', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'foo.test.ts': `
-      const { test } = pwt;
-      test('pass1', ({}) => {});
-      test.describe('suite-1', () => {
-        test.describe('suite-2', () => {
-          test('pass2', ({}) => {});
-          test('pass3', ({}) => {});
-        });
-      });
-      test('pass4', ({}) => {});
-    `,
-    'bar.test.ts': `
-      const { test } = pwt;
-      test('no-pass1', ({}) => {});
-    `,
-  }, {}, {}, { additionalArgs: ['foo.test.ts:8'] });
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(2);
-  expect(result.skipped).toBe(0);
-  expect(result.report.suites[0].suites[0].suites[0].specs[0].title).toEqual('pass2');
-  expect(result.report.suites[0].suites[0].suites[0].specs[1].title).toEqual('pass3');
-});
-
 test('should match by directory', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'dir-a/file.test.ts': `