From 8ecf581b45e02c360b0d26df99f91d37fdf5cf19 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Sun, 16 Jan 2022 08:47:09 -0800 Subject: [PATCH] chore: focus by line should override only (#11427) --- packages/playwright-test/src/runner.ts | 39 ++++- tests/config/default.config.ts | 6 +- .../command-line-filter.spec.ts | 147 ++++++++++++++++++ tests/playwright-test/test-ignore.spec.ts | 72 --------- 4 files changed, 184 insertions(+), 80 deletions(-) create mode 100644 tests/playwright-test/command-line-filter.spec.ts diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index 67cb3e2a60..11713f7d45 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -221,9 +221,15 @@ export class Runner { if (config.globalSetup && !list) globalSetupResult = await (await this._loader.loadGlobalHook(config.globalSetup, 'globalSetup'))(this._loader.fullConfig()); try { + // 1. Add all tests. const preprocessRoot = new Suite(''); for (const file of allTestFiles) preprocessRoot._addSuite(await this._loader.loadTestFile(file)); + + // 2. Filter tests to respect column filter. + filterByFocusedLine(preprocessRoot, testFileReFilters); + + // 3. Complain about only. if (config.forbidOnly) { const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); if (onlyTestsAndSuites.length > 0) { @@ -231,13 +237,16 @@ export class Runner { return { status: 'failed' }; } } + + // 4. Filter only + filterOnly(preprocessRoot); + + // 5. Complain about clashing. const clashingTests = getClashingTestsPerSuite(preprocessRoot); if (clashingTests.size > 0) { this._reporter.onError?.(createDuplicateTitlesError(config, clashingTests)); return { status: 'failed' }; } - filterOnly(preprocessRoot); - filterByFocusedLine(preprocessRoot, testFileReFilters); const fileSuites = new Map(); for (const fileSuite of preprocessRoot.suites) @@ -305,7 +314,7 @@ export class Runner { } testGroups = shardGroups; - filterSuite(rootSuite, () => false, test => shardTests.has(test)); + filterSuiteWithOnlySemantics(rootSuite, () => false, test => shardTests.has(test)); total = rootSuite.allTests().length; } (config as any).__testGroupsCount = testGroups.length; @@ -374,21 +383,27 @@ export class Runner { function filterOnly(suite: Suite) { const suiteFilter = (suite: Suite) => suite._only; const testFilter = (test: TestCase) => test._only; - return filterSuite(suite, suiteFilter, testFilter); + return filterSuiteWithOnlySemantics(suite, suiteFilter, testFilter); } function filterByFocusedLine(suite: Suite, focusedTestFileLines: FilePatternFilter[]) { + const filterWithLine = !!focusedTestFileLines.find(f => f.line !== null); + if (!filterWithLine) + return; + const testFileLineMatches = (testFileName: string, testLine: number) => focusedTestFileLines.some(({ re, line }) => { re.lastIndex = 0; return re.test(testFileName) && (line === testLine || line === null); }); - const suiteFilter = (suite: Suite) => !!suite.location && testFileLineMatches(suite.location.file, suite.location.line); + const suiteFilter = (suite: Suite) => { + return !!suite.location && testFileLineMatches(suite.location.file, suite.location.line); + }; const testFilter = (test: TestCase) => testFileLineMatches(test.location.file, test.location.line); return filterSuite(suite, suiteFilter, testFilter); } -function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { - const onlySuites = suite.suites.filter(child => filterSuite(child, suiteFilter, testFilter) || suiteFilter(child)); +function filterSuiteWithOnlySemantics(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { + const onlySuites = suite.suites.filter(child => filterSuiteWithOnlySemantics(child, suiteFilter, testFilter) || suiteFilter(child)); const onlyTests = suite.tests.filter(testFilter); const onlyEntries = new Set([...onlySuites, ...onlyTests]); if (onlyEntries.size) { @@ -400,6 +415,16 @@ function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, test return false; } +function filterSuite(suite: Suite, suiteFilter: (suites: Suite) => boolean, testFilter: (test: TestCase) => boolean) { + for (const child of suite.suites) { + if (!suiteFilter(child)) + filterSuite(child, suiteFilter, testFilter); + } + suite.tests = suite.tests.filter(testFilter); + const entries = new Set([...suite.suites, ...suite.tests]); + suite._entries = suite._entries.filter(e => entries.has(e)); // Preserve the order. +} + async function collectFiles(testDir: string): Promise { type Rule = { dir: string; diff --git a/tests/config/default.config.ts b/tests/config/default.config.ts index e3d9b5c106..118b472d02 100644 --- a/tests/config/default.config.ts +++ b/tests/config/default.config.ts @@ -14,6 +14,8 @@ * limitations under the License. */ +// @playwright.config + import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test'; import * as path from 'path'; import { TestModeWorkerOptions } from './testModeFixtures'; @@ -30,7 +32,9 @@ const getExecutablePath = (browserName: BrowserName) => { return process.env.WKPATH; }; -const mode = (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service'); +const mode = process.env.PW_OUT_OF_PROCESS ? + 'driver' : + (process.env.PWTEST_MODE || 'default') as ('default' | 'driver' | 'service'); const headed = !!process.env.HEADFUL; const channel = process.env.PWTEST_CHANNEL as any; const video = !!process.env.PWTEST_VIDEO; diff --git a/tests/playwright-test/command-line-filter.spec.ts b/tests/playwright-test/command-line-filter.spec.ts new file mode 100644 index 0000000000..c0a9fe3e72 --- /dev/null +++ b/tests/playwright-test/command-line-filter.spec.ts @@ -0,0 +1,147 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from './playwright-test-fixtures'; + +test('should filter by file name', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + 'b.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + }, undefined, undefined, { additionalArgs: ['a.spec.ts'] }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toContain('1) a.spec.ts'); +}); + +test('should filter by folder', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'foo/x.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + 'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + 'bar/x.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + 'bar/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + }, undefined, undefined, { additionalArgs: ['bar'] }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(2); + expect(result.output).toMatch(/bar[\\/]x.spec.ts/); + expect(result.output).toMatch(/bar[\\/]y.spec.ts/); +}); + +test('should filter by line', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'foo/x.spec.ts': ` + pwt.test('one', () => { expect(1).toBe(2); }); + pwt.test('two', () => { expect(1).toBe(2); }); + pwt.test('three', () => { expect(1).toBe(2); }); + `, + 'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + }, undefined, undefined, { additionalArgs: ['x.spec.ts:6'] }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toMatch(/x\.spec\.ts.*two/); +}); + +test('line should override focused test', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'foo/x.spec.ts': ` + pwt.test.only('one', () => { expect(1).toBe(2); }); + pwt.test('two', () => { expect(1).toBe(2); }); + pwt.test.only('three', () => { expect(1).toBe(2); }); + `, + 'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + }, undefined, undefined, { additionalArgs: ['x.spec.ts:6'] }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + expect(result.output).toMatch(/x\.spec\.ts.*two/); +}); + +test('should merge filtered line and filtered file', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'foo/x.spec.ts': ` + pwt.test('one', () => { expect(1).toBe(2); }); + pwt.test('two', () => { expect(1).toBe(2); }); + pwt.test('three', () => { expect(1).toBe(2); }); + `, + 'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + }, undefined, undefined, { additionalArgs: ['x.spec.ts:6', 'x.spec.ts'] }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(3); +}); + +test('should run nothing for missing line', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'foo/x.spec.ts': ` + pwt.test('one', () => { expect(1).toBe(2); }); + pwt.test('two', () => { expect(1).toBe(2); }); + pwt.test('three', () => { expect(1).toBe(2); }); + `, + 'foo/y.spec.ts': `pwt.test('fails', () => { expect(1).toBe(2); });`, + }, undefined, undefined, { additionalArgs: ['x.spec.ts:10', 'y.spec.ts'] }); + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); +}); + +test('should focus a single nested test spec', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'foo.test.ts': ` + const { test } = pwt; + test('pass1', ({}) => {}); + test.describe('suite-1', () => { + test.describe('suite-2', () => { + test('pass2', ({}) => {}); + }); + }); + test('pass3', ({}) => {}); + `, + 'bar.test.ts': ` + const { test } = pwt; + test('pass3', ({}) => {}); + `, + 'noooo.test.ts': ` + const { test } = pwt; + test('no-pass1', ({}) => {}); + `, + }, {}, {}, { additionalArgs: ['foo.test.ts:9', 'bar.test.ts'] }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.report.suites[0].specs[0].title).toEqual('pass3'); + expect(result.report.suites[1].suites[0].suites[0].specs[0].title).toEqual('pass2'); +}); + +test('should focus a single test suite', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'foo.test.ts': ` + const { test } = pwt; + test('pass1', ({}) => {}); + test.describe('suite-1', () => { + test.describe('suite-2', () => { + test('pass2', ({}) => {}); + test('pass3', ({}) => {}); + }); + }); + test('pass4', ({}) => {}); + `, + 'bar.test.ts': ` + const { test } = pwt; + test('no-pass1', ({}) => {}); + `, + }, {}, {}, { additionalArgs: ['foo.test.ts:8'] }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(2); + expect(result.skipped).toBe(0); + expect(result.report.suites[0].suites[0].suites[0].specs[0].title).toEqual('pass2'); + expect(result.report.suites[0].suites[0].suites[0].specs[1].title).toEqual('pass3'); +}); \ No newline at end of file diff --git a/tests/playwright-test/test-ignore.spec.ts b/tests/playwright-test/test-ignore.spec.ts index 3662592de2..2f72663118 100644 --- a/tests/playwright-test/test-ignore.spec.ts +++ b/tests/playwright-test/test-ignore.spec.ts @@ -248,78 +248,6 @@ test('should match case insensitive', async ({ runInlineTest }) => { expect(result.exitCode).toBe(0); }); -test('should focus a single test spec', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'foo.test.ts': ` - const { test } = pwt; - test('pass1', ({}) => {}); - test('pass2', ({}) => {}); - test('pass3', ({}) => {}); - `, - 'bar.test.ts': ` - const { test } = pwt; - test('no-pass1', ({}) => {}); - `, - }, {}, {}, { additionalArgs: ['foo.test.ts:7'] }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(result.skipped).toBe(0); - expect(result.report.suites[0].specs[0].title).toEqual('pass2'); -}); - -test('should focus a single nested test spec', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'foo.test.ts': ` - const { test } = pwt; - test('pass1', ({}) => {}); - test.describe('suite-1', () => { - test.describe('suite-2', () => { - test('pass2', ({}) => {}); - }); - }); - test('pass3', ({}) => {}); - `, - 'bar.test.ts': ` - const { test } = pwt; - test('pass3', ({}) => {}); - `, - 'noooo.test.ts': ` - const { test } = pwt; - test('no-pass1', ({}) => {}); - `, - }, {}, {}, { additionalArgs: ['foo.test.ts:9', 'bar.test.ts'] }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); - expect(result.skipped).toBe(0); - expect(result.report.suites[0].specs[0].title).toEqual('pass3'); - expect(result.report.suites[1].suites[0].suites[0].specs[0].title).toEqual('pass2'); -}); - -test('should focus a single test suite', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'foo.test.ts': ` - const { test } = pwt; - test('pass1', ({}) => {}); - test.describe('suite-1', () => { - test.describe('suite-2', () => { - test('pass2', ({}) => {}); - test('pass3', ({}) => {}); - }); - }); - test('pass4', ({}) => {}); - `, - 'bar.test.ts': ` - const { test } = pwt; - test('no-pass1', ({}) => {}); - `, - }, {}, {}, { additionalArgs: ['foo.test.ts:8'] }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(2); - expect(result.skipped).toBe(0); - expect(result.report.suites[0].suites[0].suites[0].specs[0].title).toEqual('pass2'); - expect(result.report.suites[0].suites[0].suites[0].specs[1].title).toEqual('pass3'); -}); - test('should match by directory', async ({ runInlineTest }) => { const result = await runInlineTest({ 'dir-a/file.test.ts': `