diff --git a/docs/src/test-api/class-testinfo.md b/docs/src/test-api/class-testinfo.md index 4c9e6e6c18..aceac10f3c 100644 --- a/docs/src/test-api/class-testinfo.md +++ b/docs/src/test-api/class-testinfo.md @@ -149,14 +149,14 @@ The number of milliseconds the test took to finish. Always zero before the test ## property: TestInfo.error * since: v1.10 -- type: ?<[TestError]> +- type: ?<[TestInfoError]> First error thrown during test execution, if any. This is equal to the first element in [`property: TestInfo.errors`]. ## property: TestInfo.errors * since: v1.10 -- type: <[Array]<[TestError]>> +- type: <[Array]<[TestInfoError]>> Errors thrown during test execution, if any. diff --git a/docs/src/test-api/class-testinfoerror.md b/docs/src/test-api/class-testinfoerror.md new file mode 100644 index 0000000000..66e78ecabd --- /dev/null +++ b/docs/src/test-api/class-testinfoerror.md @@ -0,0 +1,23 @@ +# class: TestInfoError +* since: v1.10 +* langs: js + +Information about an error thrown during test execution. + +## property: TestInfoError.message +* since: v1.10 +- type: ?<[string]> + +Error message. Set when [Error] (or its subclass) has been thrown. + +## property: TestInfoError.stack +* since: v1.10 +- type: ?<[string]> + +Error stack. Set when [Error] (or its subclass) has been thrown. + +## property: TestInfoError.value +* since: v1.10 +- type: ?<[string]> + +The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. diff --git a/docs/src/test-api/class-testerror.md b/docs/src/test-reporter-api/class-testerror.md similarity index 83% rename from docs/src/test-api/class-testerror.md rename to docs/src/test-reporter-api/class-testerror.md index 0555598c5d..43eda7e647 100644 --- a/docs/src/test-api/class-testerror.md +++ b/docs/src/test-reporter-api/class-testerror.md @@ -21,3 +21,9 @@ Error stack. Set when [Error] (or its subclass) has been thrown. - type: ?<[string]> The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. + +## property: TestError.location +* since: v1.30 +- type: ?<[Location]> + +Error location in the source code. diff --git a/packages/playwright-test/src/dispatcher.ts b/packages/playwright-test/src/dispatcher.ts index 0ff95225db..6f44cd8c3d 100644 --- a/packages/playwright-test/src/dispatcher.ts +++ b/packages/playwright-test/src/dispatcher.ts @@ -350,7 +350,9 @@ export class Dispatcher { const test = this._testById.get(testId)!.test; return test.titlePath().slice(1).join(' > '); }); - massSkipTestsFromRemaining(new Set(params.fatalUnknownTestIds), [{ message: `Unknown test(s) in worker:\n${titles.join('\n')}` }]); + massSkipTestsFromRemaining(new Set(params.fatalUnknownTestIds), [{ + message: `Internal error: unknown test(s) in worker:\n${titles.join('\n')}` + }]); } if (params.fatalErrors.length) { // In case of fatal errors, report first remaining test as failing with these errors, @@ -423,7 +425,9 @@ export class Dispatcher { worker.on('done', onDone); const onExit = (data: WorkerExitData) => { - const unexpectedExitError = data.unexpectedly ? { value: `Worker process exited unexpectedly (code=${data.code}, signal=${data.signal})` } : undefined; + const unexpectedExitError: TestError | undefined = data.unexpectedly ? { + message: `Internal error: worker process exited unexpectedly (code=${data.code}, signal=${data.signal})` + } : undefined; onDone({ skipTestsDueToSetupFailure: [], fatalErrors: [], unexpectedExitError }); }; worker.on('exit', onExit); diff --git a/packages/playwright-test/src/ipc.ts b/packages/playwright-test/src/ipc.ts index c330cae505..e5b265fddd 100644 --- a/packages/playwright-test/src/ipc.ts +++ b/packages/playwright-test/src/ipc.ts @@ -14,9 +14,8 @@ * limitations under the License. */ -import type { TestError } from '../types/testReporter'; import type { ConfigCLIOverrides } from './runner'; -import type { TestStatus } from './types'; +import type { TestInfoError, TestStatus } from './types'; export type SerializedLoaderData = { configFile: string | undefined; @@ -61,7 +60,7 @@ export type TestEndPayload = { testId: string; duration: number; status: TestStatus; - errors: TestError[]; + errors: TestInfoError[]; expectedStatus: TestStatus; annotations: { type: string, description?: string }[]; timeout: number; @@ -84,7 +83,7 @@ export type StepEndPayload = { stepId: string; refinedTitle?: string; wallTime: number; // milliseconds since unix epoch - error?: TestError; + error?: TestInfoError; }; export type TestEntry = { @@ -100,7 +99,7 @@ export type RunPayload = { }; export type DonePayload = { - fatalErrors: TestError[]; + fatalErrors: TestInfoError[]; skipTestsDueToSetupFailure: string[]; // test ids fatalUnknownTestIds?: string[]; }; @@ -112,5 +111,5 @@ export type TestOutputPayload = { }; export type TeardownErrorsPayload = { - fatalErrors: TestError[]; + fatalErrors: TestInfoError[]; }; diff --git a/packages/playwright-test/src/reporters/base.ts b/packages/playwright-test/src/reporters/base.ts index 7aa6600f06..56782c901d 100644 --- a/packages/playwright-test/src/reporters/base.ts +++ b/packages/playwright-test/src/reporters/base.ts @@ -160,7 +160,7 @@ export class BaseReporter implements ReporterInternal { tokens.push(colors.green(` ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`)); if (this.result.status === 'timedout') tokens.push(colors.red(` Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`)); - if (fatalErrors.length) + if (fatalErrors.length && expected + unexpected.length + interrupted.length + flaky.length > 0) tokens.push(colors.red(` ${fatalErrors.length === 1 ? '1 error was not a part of any test' : fatalErrors.length + ' errors were not a part of any test'}, see above for details`)); return tokens.join('\n'); @@ -377,37 +377,42 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in } export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails { + const message = error.message || error.value || ''; const stack = error.stack; + if (!stack && !error.location) + return { message }; + const tokens = []; - let location: Location | undefined; - if (stack) { - // Now that we filter out internals from our stack traces, we can safely render - // the helper / original exception locations. - const parsed = prepareErrorStack(stack); - tokens.push(parsed.message); - location = parsed.location; - if (location) { - try { - const source = fs.readFileSync(location.file, 'utf8'); - const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode }); - // Convert /var/folders to /private/var/folders on Mac. - if (!file || fs.realpathSync(file) !== location.file) { - tokens.push(''); - tokens.push(colors.gray(` at `) + `${relativeFilePath(config, location.file)}:${location.line}`); - } + + // Now that we filter out internals from our stack traces, we can safely render + // the helper / original exception locations. + const parsedStack = stack ? prepareErrorStack(stack) : undefined; + tokens.push(parsedStack?.message || message); + + let location = error.location; + if (parsedStack && !location) + location = parsedStack.location; + + if (location) { + try { + const source = fs.readFileSync(location.file, 'utf8'); + const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode }); + // Convert /var/folders to /private/var/folders on Mac. + if (!file || fs.realpathSync(file) !== location.file) { tokens.push(''); - tokens.push(codeFrame); - } catch (e) { - // Failed to read the source file - that's ok. + tokens.push(colors.gray(` at `) + `${relativeFilePath(config, location.file)}:${location.line}`); } + tokens.push(''); + tokens.push(codeFrame); + } catch (e) { + // Failed to read the source file - that's ok. } - tokens.push(''); - tokens.push(colors.dim(parsed.stackLines.join('\n'))); - } else if (error.message) { - tokens.push(error.message); - } else if (error.value) { - tokens.push(error.value); } + if (parsedStack) { + tokens.push(''); + tokens.push(colors.dim(parsedStack.stackLines.join('\n'))); + } + return { location, message: tokens.join('\n'), diff --git a/packages/playwright-test/src/runner.ts b/packages/playwright-test/src/runner.ts index cf571a32aa..eee7bcec49 100644 --- a/packages/playwright-test/src/runner.ts +++ b/packages/playwright-test/src/runner.ts @@ -17,7 +17,6 @@ import * as fs from 'fs'; import * as path from 'path'; -import { MultiMap } from 'playwright-core/lib/utils/multimap'; import { raceAgainstTimeout } from 'playwright-core/lib/utils/timeoutRunner'; import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle'; import { promisify } from 'util'; @@ -177,7 +176,8 @@ export class Runner { const result = await raceAgainstTimeout(() => this._run(options), config.globalTimeout); let fullResult: FullResult; if (result.timedOut) { - this._reporter.onError?.(createStacklessError(`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`)); + this._reporter.onError?.(createStacklessError( + `Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`)); fullResult = { status: 'timedout' }; } else { fullResult = result.result; @@ -325,9 +325,7 @@ export class Runner { } // Complain about duplicate titles. - const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot); - if (duplicateTitlesError) - fatalErrors.push(duplicateTitlesError); + fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot)); // Filter tests to respect line/column filter. filterByFocusedLine(preprocessRoot, options.testFileFilters, doNotFilterFiles); @@ -336,7 +334,7 @@ export class Runner { if (config.forbidOnly) { const onlyTestsAndSuites = preprocessRoot._getOnlyItems(); if (onlyTestsAndSuites.length > 0) - fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites)); + fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites)); } // Filter only. @@ -782,7 +780,6 @@ async function collectFiles(testDir: string, respectGitIgnore: boolean): Promise return files; } - function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) { if (!testOrSuite.location) return ''; @@ -927,53 +924,46 @@ class ListModeReporter implements Reporter { } } -function createForbidOnlyError(config: FullConfigInternal, onlyTestsAndSuites: (TestCase | Suite)[]): TestError { - const errorMessage = [ - '=====================================', - ' --forbid-only found a focused test.', - ]; +function createForbidOnlyErrors(config: FullConfigInternal, onlyTestsAndSuites: (TestCase | Suite)[]): TestError[] { + const errors: TestError[] = []; for (const testOrSuite of onlyTestsAndSuites) { // Skip root and file. const title = testOrSuite.titlePath().slice(2).join(' '); - errorMessage.push(` - ${buildItemLocation(config.rootDir, testOrSuite)} > ${title}`); + const error: TestError = { + message: `Error: focused item found in the --forbid-only mode: "${title}"`, + location: testOrSuite.location!, + }; + errors.push(error); } - errorMessage.push('====================================='); - return createStacklessError(errorMessage.join('\n')); + return errors; } -function createDuplicateTitlesError(config: FullConfigInternal, rootSuite: Suite): TestError | undefined { - const lines: string[] = []; +function createDuplicateTitlesErrors(config: FullConfigInternal, rootSuite: Suite): TestError[] { + const errors: TestError[] = []; for (const fileSuite of rootSuite.suites) { - const testsByFullTitle = new MultiMap(); + const testsByFullTitle = new Map(); for (const test of fileSuite.allTests()) { const fullTitle = test.titlePath().slice(2).join('\x1e'); + const existingTest = testsByFullTitle.get(fullTitle); + if (existingTest) { + const error: TestError = { + message: `Error: duplicate test title "${fullTitle}", first declared in ${buildItemLocation(config.rootDir, existingTest)}`, + location: test.location, + }; + errors.push(error); + } testsByFullTitle.set(fullTitle, test); } - for (const fullTitle of testsByFullTitle.keys()) { - const tests = testsByFullTitle.get(fullTitle); - if (tests.length > 1) { - lines.push(` - title: ${fullTitle.replace(/\u001e/g, ' › ')}`); - for (const test of tests) - lines.push(` - ${buildItemLocation(config.rootDir, test)}`); - } - } } - if (!lines.length) - return; - return createStacklessError([ - '========================================', - ' duplicate test titles are not allowed.', - ...lines, - '========================================', - ].join('\n')); + return errors; } function createNoTestsError(): TestError { return createStacklessError(`=================\n no tests found.\n=================`); } -function createStacklessError(message: string): TestError { - return { message, __isNotAFatalError: true } as any; +function createStacklessError(message: string, location?: TestError['location']): TestError { + return { message, location }; } function sanitizeConfigForJSON(object: any, visited: Set): any { diff --git a/packages/playwright-test/src/testInfo.ts b/packages/playwright-test/src/testInfo.ts index 00e52f7f5d..1848f18dd1 100644 --- a/packages/playwright-test/src/testInfo.ts +++ b/packages/playwright-test/src/testInfo.ts @@ -17,7 +17,7 @@ import fs from 'fs'; import path from 'path'; import { monotonicTime } from 'playwright-core/lib/utils'; -import type { TestError, TestInfo, TestStatus } from '../types/test'; +import type { TestInfoError, TestInfo, TestStatus } from '../types/test'; import type { WorkerInitParams } from './ipc'; import type { Loader } from './loader'; import type { TestCase } from './test'; @@ -58,14 +58,14 @@ export class TestInfoImpl implements TestInfo { snapshotSuffix: string = ''; readonly outputDir: string; readonly snapshotDir: string; - errors: TestError[] = []; + errors: TestInfoError[] = []; currentStep: TestStepInternal | undefined; - get error(): TestError | undefined { + get error(): TestInfoError | undefined { return this.errors[0]; } - set error(e: TestError | undefined) { + set error(e: TestInfoError | undefined) { if (e === undefined) throw new Error('Cannot assign testInfo.error undefined value!'); this.errors[0] = e; @@ -168,7 +168,7 @@ export class TestInfoImpl implements TestInfo { this.duration = this._timeoutManager.defaultSlotTimings().elapsed | 0; } - async _runFn(fn: Function, skips?: 'allowSkips'): Promise { + async _runFn(fn: Function, skips?: 'allowSkips'): Promise { try { await fn(); } catch (error) { @@ -187,7 +187,7 @@ export class TestInfoImpl implements TestInfo { return this._addStepImpl(data); } - _failWithError(error: TestError, isHardError: boolean) { + _failWithError(error: TestInfoError, isHardError: boolean) { // Do not overwrite any previous hard errors. // Some (but not all) scenarios include: // - expect() that fails after uncaught exception. diff --git a/packages/playwright-test/src/timeoutManager.ts b/packages/playwright-test/src/timeoutManager.ts index 6d0dc6472b..94f0ee7fb4 100644 --- a/packages/playwright-test/src/timeoutManager.ts +++ b/packages/playwright-test/src/timeoutManager.ts @@ -16,8 +16,7 @@ import { colors } from 'playwright-core/lib/utilsBundle'; import { TimeoutRunner, TimeoutRunnerError } from 'playwright-core/lib/utils/timeoutRunner'; -import type { TestError } from '../types/test'; -import type { Location } from './types'; +import type { Location, TestInfoError } from './types'; export type TimeSlot = { timeout: number; @@ -72,7 +71,7 @@ export class TimeoutManager { this._timeoutRunner.updateTimeout(slot.timeout); } - async runWithTimeout(cb: () => Promise): Promise { + async runWithTimeout(cb: () => Promise): Promise { try { await this._timeoutRunner.run(cb); } catch (error) { @@ -105,7 +104,7 @@ export class TimeoutManager { this._timeoutRunner.updateTimeout(slot.timeout, slot.elapsed); } - private _createTimeoutError(): TestError { + private _createTimeoutError(): TestInfoError { let message = ''; const timeout = this._currentSlot().timeout; switch (this._runnable.type) { @@ -142,7 +141,7 @@ export class TimeoutManager { return { message, // Include location for hooks, modifiers and fixtures to distinguish between them. - stack: location ? message + `\n at ${location.file}:${location.line}:${location.column}` : undefined, + stack: location ? message + `\n at ${location.file}:${location.line}:${location.column}` : undefined }; } } diff --git a/packages/playwright-test/src/types.ts b/packages/playwright-test/src/types.ts index 608fa22559..f9d94baaa0 100644 --- a/packages/playwright-test/src/types.ts +++ b/packages/playwright-test/src/types.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -import type { Fixtures, TestError, Project } from '../types/test'; +import type { Fixtures, TestInfoError, Project } from '../types/test'; import type { Location, Reporter } from '../types/testReporter'; import type { WorkerIsolation } from './ipc'; import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types'; @@ -28,7 +28,7 @@ export type FixturesWithLocation = { export type Annotation = { type: string, description?: string }; export interface TestStepInternal { - complete(result: { error?: Error | TestError }): void; + complete(result: { error?: Error | TestInfoError }): void; title: string; category: string; canHaveChildren: boolean; diff --git a/packages/playwright-test/src/util.ts b/packages/playwright-test/src/util.ts index aed694a4bb..6a2dae38c8 100644 --- a/packages/playwright-test/src/util.ts +++ b/packages/playwright-test/src/util.ts @@ -20,7 +20,7 @@ import util from 'util'; import path from 'path'; import url from 'url'; import { colors, debug, minimatch } from 'playwright-core/lib/utilsBundle'; -import type { TestError, Location } from './types'; +import type { TestInfoError, Location } from './types'; import { calculateSha1, isRegExp, isString } from 'playwright-core/lib/utils'; import { isInternalFileName } from 'playwright-core/lib/utils/stackTrace'; import { currentTestInfo } from './globals'; @@ -86,7 +86,7 @@ export function captureStackTrace(customApiName?: string): ParsedStackTrace { }; } -export function serializeError(error: Error | any): TestError { +export function serializeError(error: Error | any): TestInfoError { if (error instanceof Error) { filterStackTrace(error); return { diff --git a/packages/playwright-test/src/workerRunner.ts b/packages/playwright-test/src/workerRunner.ts index 4017044aa8..0e7e6fbcb1 100644 --- a/packages/playwright-test/src/workerRunner.ts +++ b/packages/playwright-test/src/workerRunner.ts @@ -22,7 +22,7 @@ import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerI import { setCurrentTestInfo } from './globals'; import { Loader } from './loader'; import type { Suite, TestCase } from './test'; -import type { Annotation, FullProjectInternal, TestError, TestStepInternal } from './types'; +import type { Annotation, FullProjectInternal, TestInfoError, TestStepInternal } from './types'; import { FixtureRunner } from './fixtures'; import { ManualPromise } from 'playwright-core/lib/utils/manualPromise'; import { TestInfoImpl } from './testInfo'; @@ -38,7 +38,7 @@ export class WorkerRunner extends EventEmitter { private _fixtureRunner: FixtureRunner; // Accumulated fatal errors that cannot be attributed to a test. - private _fatalErrors: TestError[] = []; + private _fatalErrors: TestInfoError[] = []; // Whether we should skip running remaining tests in this suite because // of a setup error, usually beforeAll hook. private _skipRemainingTestsInSuite: Suite | undefined; @@ -91,7 +91,7 @@ export class WorkerRunner extends EventEmitter { } } - appendWorkerTeardownDiagnostics(error: TestError) { + appendWorkerTeardownDiagnostics(error: TestInfoError) { if (!this._lastRunningTests.length) return; const count = this._totalRunningTests === 1 ? '1 test' : `${this._totalRunningTests} tests`; @@ -408,7 +408,7 @@ export class WorkerRunner extends EventEmitter { canHaveChildren: true, forceNoParent: true }); - let firstAfterHooksError: TestError | undefined; + let firstAfterHooksError: TestInfoError | undefined; let afterHooksSlot: TimeSlot | undefined; if (testInfo._didTimeout) { @@ -561,7 +561,7 @@ export class WorkerRunner extends EventEmitter { if (!this._activeSuites.has(suite)) return; this._activeSuites.delete(suite); - let firstError: TestError | undefined; + let firstError: TestInfoError | undefined; for (const hook of suite._hooks) { if (hook.type !== 'afterAll') continue; diff --git a/packages/playwright-test/types/test.d.ts b/packages/playwright-test/types/test.d.ts index 2ae64ac621..adc148e432 100644 --- a/packages/playwright-test/types/test.d.ts +++ b/packages/playwright-test/types/test.d.ts @@ -1615,12 +1615,12 @@ export interface TestInfo { * First error thrown during test execution, if any. This is equal to the first element in * [testInfo.errors](https://playwright.dev/docs/api/class-testinfo#test-info-errors). */ - error?: TestError; + error?: TestInfoError; /** * Errors thrown during test execution, if any. */ - errors: Array; + errors: Array; /** * Expected status for the currently running test. This is usually `'passed'`, except for a few cases: @@ -4458,7 +4458,7 @@ interface SnapshotAssertions { /** * Information about an error thrown during test execution. */ -export interface TestError { +export interface TestInfoError { /** * Error message. Set when [Error] (or its subclass) has been thrown. */ diff --git a/packages/playwright-test/types/testReporter.d.ts b/packages/playwright-test/types/testReporter.d.ts index f29a4e4b21..a68033cca2 100644 --- a/packages/playwright-test/types/testReporter.d.ts +++ b/packages/playwright-test/types/testReporter.d.ts @@ -15,8 +15,8 @@ * limitations under the License. */ -import type { FullConfig, FullProject, TestStatus, TestError, Metadata } from './test'; -export type { FullConfig, TestStatus, TestError } from './test'; +import type { FullConfig, FullProject, TestStatus, Metadata } from './test'; +export type { FullConfig, TestStatus } from './test'; /** * `Suite` is a group of tests. All tests in Playwright Test form the following hierarchy: @@ -563,6 +563,31 @@ export interface Location { column: number; } +/** + * Information about an error thrown during test execution. + */ +export interface TestError { + /** + * Error message. Set when [Error] (or its subclass) has been thrown. + */ + message?: string; + + /** + * Error stack. Set when [Error] (or its subclass) has been thrown. + */ + stack?: string; + + /** + * The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown. + */ + value?: string; + + /** + * Error location in the source code. + */ + location?: Location; +} + /** * Represents a step in the [TestRun]. */ diff --git a/tests/playwright-test/hooks.spec.ts b/tests/playwright-test/hooks.spec.ts index c1cb331ffa..31dbfcec7f 100644 --- a/tests/playwright-test/hooks.spec.ts +++ b/tests/playwright-test/hooks.spec.ts @@ -636,7 +636,7 @@ test('should not hang and report results when worker process suddenly exits duri expect(result.exitCode).toBe(1); expect(result.passed).toBe(0); expect(result.failed).toBe(1); - expect(result.output).toContain('Worker process exited unexpectedly'); + expect(result.output).toContain('Internal error: worker process exited unexpectedly'); expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › failing due to afterall'); }); diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts index 264265db22..3495064f65 100644 --- a/tests/playwright-test/reporter.spec.ts +++ b/tests/playwright-test/reporter.spec.ts @@ -475,7 +475,7 @@ test('should report forbid-only error to reporter', async ({ runInlineTest }) => }, { 'reporter': '', 'forbid-only': true }); expect(result.exitCode).toBe(1); - expect(result.output).toContain(`%%got error: =====================================\n --forbid-only found a focused test.`); + expect(result.output).toContain(`%%got error: Error: focused item found in the --forbid-only mode`); }); test('should report no-tests error to reporter', async ({ runInlineTest }) => { diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts index db99ee5439..da338dc37e 100644 --- a/tests/playwright-test/runner.spec.ts +++ b/tests/playwright-test/runner.spec.ts @@ -29,10 +29,10 @@ test('it should not allow multiple tests with the same name per suite', async ({ ` }); expect(result.exitCode).toBe(1); - expect(result.output).toContain('duplicate test titles are not allowed'); - expect(result.output).toContain(`- title: suite › i-am-a-duplicate`); - expect(result.output).toContain(` - tests${path.sep}example.spec.js:7`); - expect(result.output).toContain(` - tests${path.sep}example.spec.js:10`); + expect(result.output).toContain(`Error: duplicate test title`); + expect(result.output).toContain(`i-am-a-duplicate`); + expect(result.output).toContain(`tests${path.sep}example.spec.js:7`); + expect(result.output).toContain(`tests${path.sep}example.spec.js:10`); }); test('it should not allow multiple tests with the same name in multiple files', async ({ runInlineTest }) => { @@ -49,13 +49,12 @@ test('it should not allow multiple tests with the same name in multiple files', `, }); expect(result.exitCode).toBe(1); - expect(result.output).toContain('duplicate test titles are not allowed'); - expect(result.output).toContain(`- title: i-am-a-duplicate`); - expect(result.output).toContain(` - tests${path.sep}example1.spec.js:6`); - expect(result.output).toContain(` - tests${path.sep}example1.spec.js:7`); - expect(result.output).toContain(`- title: i-am-a-duplicate`); - expect(result.output).toContain(` - tests${path.sep}example2.spec.js:6`); - expect(result.output).toContain(` - tests${path.sep}example2.spec.js:7`); + expect(result.output).toContain('Error: duplicate test title'); + expect(stripAnsi(result.output)).toContain(`test('i-am-a-duplicate'`); + expect(result.output).toContain(`tests${path.sep}example1.spec.js:6`); + expect(result.output).toContain(`tests${path.sep}example1.spec.js:7`); + expect(result.output).toContain(`tests${path.sep}example2.spec.js:6`); + expect(result.output).toContain(`tests${path.sep}example2.spec.js:7`); }); test('it should not allow a focused test when forbid-only is used', async ({ runInlineTest }) => { @@ -66,8 +65,9 @@ test('it should not allow a focused test when forbid-only is used', async ({ run ` }, { 'forbid-only': true }); expect(result.exitCode).toBe(1); - expect(result.output).toContain('--forbid-only found a focused test.'); - expect(result.output).toContain(`- tests${path.sep}focused-test.spec.js:6 > i-am-focused`); + expect(result.output).toContain('Error: focused item found in the --forbid-only mode'); + expect(stripAnsi(result.output)).toContain(`test.only('i-am-focused'`); + expect(result.output).toContain(`tests${path.sep}focused-test.spec.js:6`); }); test('should continue with other tests after worker process suddenly exits', async ({ runInlineTest }) => { @@ -85,7 +85,7 @@ test('should continue with other tests after worker process suddenly exits', asy expect(result.passed).toBe(4); expect(result.failed).toBe(1); expect(result.skipped).toBe(0); - expect(result.output).toContain('Worker process exited unexpectedly'); + expect(result.output).toContain('Internal error: worker process exited unexpectedly'); }); test('sigint should stop workers', async ({ runInlineTest }) => { @@ -321,7 +321,7 @@ test('should not hang if test suites in worker are inconsistent with runner', as expect(result.passed).toBe(1); expect(result.failed).toBe(1); expect(result.skipped).toBe(1); - expect(result.report.suites[0].specs[1].tests[0].results[0].error!.message).toBe('Unknown test(s) in worker:\nproject-name > a.spec.js > Test 1 - bar\nproject-name > a.spec.js > Test 2 - baz'); + expect(result.report.suites[0].specs[1].tests[0].results[0].error!.message).toBe('Internal error: unknown test(s) in worker:\nproject-name > a.spec.js > Test 1 - bar\nproject-name > a.spec.js > Test 2 - baz'); }); test('sigint should stop global setup', async ({ runInlineTest }) => { @@ -452,13 +452,13 @@ test('should not crash with duplicate titles and .only', async ({ runInlineTest ` }); expect(result.exitCode).toBe(1); - expect(stripAnsi(result.output)).toContain([ - ` duplicate test titles are not allowed.`, - ` - title: non unique title`, - ` - example.spec.ts:6`, - ` - example.spec.ts:7`, - ` - example.spec.ts:8`, - ].join('\n')); + expect(result.output).toContain(`Error: duplicate test title`); + expect(stripAnsi(result.output)).toContain(`test('non unique title'`); + expect(stripAnsi(result.output)).toContain(`test.skip('non unique title'`); + expect(stripAnsi(result.output)).toContain(`test.only('non unique title'`); + expect(result.output).toContain(`example.spec.ts:6`); + expect(result.output).toContain(`example.spec.ts:7`); + expect(result.output).toContain(`example.spec.ts:8`); }); test('should not crash with duplicate titles and line filter', async ({ runInlineTest }) => { @@ -471,13 +471,11 @@ test('should not crash with duplicate titles and line filter', async ({ runInlin ` }, {}, {}, { additionalArgs: ['example.spec.ts:8'] }); expect(result.exitCode).toBe(1); - expect(stripAnsi(result.output)).toContain([ - ` duplicate test titles are not allowed.`, - ` - title: non unique title`, - ` - example.spec.ts:6`, - ` - example.spec.ts:7`, - ` - example.spec.ts:8`, - ].join('\n')); + expect(result.output).toContain(`Error: duplicate test title`); + expect(stripAnsi(result.output)).toContain(`test('non unique title'`); + expect(result.output).toContain(`example.spec.ts:6`); + expect(result.output).toContain(`example.spec.ts:7`); + expect(result.output).toContain(`example.spec.ts:8`); }); test('should not load tests not matching filter', async ({ runInlineTest }) => { diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 75760d4334..484c63a7f2 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -14,8 +14,8 @@ * limitations under the License. */ -import type { FullConfig, FullProject, TestStatus, TestError, Metadata } from './test'; -export type { FullConfig, TestStatus, TestError } from './test'; +import type { FullConfig, FullProject, TestStatus, Metadata } from './test'; +export type { FullConfig, TestStatus } from './test'; export interface Suite { project(): FullProject | undefined;