chore: report error location for fatal errors (#19610)

docs/src/test-api/class-testinfo.md

@@ -149,14 +149,14 @@ The number of milliseconds the test took to finish. Always zero before the test
 
 ## property: TestInfo.error
 * since: v1.10
-- type: ?<[TestError]>
+- type: ?<[TestInfoError]>
 
 First error thrown during test execution, if any. This is equal to the first
 element in [`property: TestInfo.errors`].
 
 ## property: TestInfo.errors
 * since: v1.10
-- type: <[Array]<[TestError]>>
+- type: <[Array]<[TestInfoError]>>
 
 Errors thrown during test execution, if any.
 

docs/src/test-api/class-testinfoerror.md

@@ -0,0 +1,23 @@
+# class: TestInfoError
+* since: v1.10
+* langs: js
+
+Information about an error thrown during test execution.
+
+## property: TestInfoError.message
+* since: v1.10
+- type: ?<[string]>
+
+Error message. Set when [Error] (or its subclass) has been thrown.
+
+## property: TestInfoError.stack
+* since: v1.10
+- type: ?<[string]>
+
+Error stack. Set when [Error] (or its subclass) has been thrown.
+
+## property: TestInfoError.value
+* since: v1.10
+- type: ?<[string]>
+
+The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown.

docs/src/test-reporter-api/class-testerror.md

@@ -21,3 +21,9 @@ Error stack. Set when [Error] (or its subclass) has been thrown.
 - type: ?<[string]>
 
 The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown.
+
+## property: TestError.location
+* since: v1.30
+- type: ?<[Location]>
+
+Error location in the source code.

packages/playwright-test/src/dispatcher.ts

@@ -350,7 +350,9 @@ export class Dispatcher {
           const test = this._testById.get(testId)!.test;
           return test.titlePath().slice(1).join(' > ');
         });
-        massSkipTestsFromRemaining(new Set(params.fatalUnknownTestIds), [{ message: `Unknown test(s) in worker:\n${titles.join('\n')}` }]);
+        massSkipTestsFromRemaining(new Set(params.fatalUnknownTestIds), [{
+          message: `Internal error: unknown test(s) in worker:\n${titles.join('\n')}`
+        }]);
       }
       if (params.fatalErrors.length) {
         // In case of fatal errors, report first remaining test as failing with these errors,
@@ -423,7 +425,9 @@ export class Dispatcher {
     worker.on('done', onDone);
 
     const onExit = (data: WorkerExitData) => {
-      const unexpectedExitError = data.unexpectedly ? { value: `Worker process exited unexpectedly (code=${data.code}, signal=${data.signal})` } : undefined;
+      const unexpectedExitError: TestError | undefined = data.unexpectedly ? {
+        message: `Internal error: worker process exited unexpectedly (code=${data.code}, signal=${data.signal})`
+      } : undefined;
       onDone({ skipTestsDueToSetupFailure: [], fatalErrors: [], unexpectedExitError });
     };
     worker.on('exit', onExit);

packages/playwright-test/src/ipc.ts

@@ -14,9 +14,8 @@
  * limitations under the License.
  */
 
-import type { TestError } from '../types/testReporter';
 import type { ConfigCLIOverrides } from './runner';
-import type { TestStatus } from './types';
+import type { TestInfoError, TestStatus } from './types';
 
 export type SerializedLoaderData = {
   configFile: string | undefined;
@@ -61,7 +60,7 @@ export type TestEndPayload = {
   testId: string;
   duration: number;
   status: TestStatus;
-  errors: TestError[];
+  errors: TestInfoError[];
   expectedStatus: TestStatus;
   annotations: { type: string, description?: string }[];
   timeout: number;
@@ -84,7 +83,7 @@ export type StepEndPayload = {
   stepId: string;
   refinedTitle?: string;
   wallTime: number;  // milliseconds since unix epoch
-  error?: TestError;
+  error?: TestInfoError;
 };
 
 export type TestEntry = {
@@ -100,7 +99,7 @@ export type RunPayload = {
 };
 
 export type DonePayload = {
-  fatalErrors: TestError[];
+  fatalErrors: TestInfoError[];
   skipTestsDueToSetupFailure: string[];  // test ids
   fatalUnknownTestIds?: string[];
 };
@@ -112,5 +111,5 @@ export type TestOutputPayload = {
 };
 
 export type TeardownErrorsPayload = {
-  fatalErrors: TestError[];
+  fatalErrors: TestInfoError[];
 };

packages/playwright-test/src/reporters/base.ts

@@ -160,7 +160,7 @@ export class BaseReporter implements ReporterInternal  {
       tokens.push(colors.green(`  ${expected} passed`) + colors.dim(` (${milliseconds(this.duration)})`));
     if (this.result.status === 'timedout')
       tokens.push(colors.red(`  Timed out waiting ${this.config.globalTimeout / 1000}s for the entire test run`));
-    if (fatalErrors.length)
+    if (fatalErrors.length && expected + unexpected.length + interrupted.length + flaky.length > 0)
       tokens.push(colors.red(`  ${fatalErrors.length === 1 ? '1 error was not a part of any test' : fatalErrors.length + ' errors were not a part of any test'}, see above for details`));
 
     return tokens.join('\n');
@@ -377,37 +377,42 @@ function formatTestHeader(config: FullConfig, test: TestCase, indent: string, in
 }
 
 export function formatError(config: FullConfig, error: TestError, highlightCode: boolean, file?: string): ErrorDetails {
+  const message = error.message || error.value || '';
   const stack = error.stack;
+  if (!stack && !error.location)
+    return { message };
+
   const tokens = [];
-  let location: Location | undefined;
-  if (stack) {
-    // Now that we filter out internals from our stack traces, we can safely render
-    // the helper / original exception locations.
-    const parsed = prepareErrorStack(stack);
-    tokens.push(parsed.message);
-    location = parsed.location;
-    if (location) {
-      try {
-        const source = fs.readFileSync(location.file, 'utf8');
-        const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode });
-        // Convert /var/folders to /private/var/folders on Mac.
-        if (!file || fs.realpathSync(file) !== location.file) {
-          tokens.push('');
-          tokens.push(colors.gray(`   at `) + `${relativeFilePath(config, location.file)}:${location.line}`);
-        }
+
+  // Now that we filter out internals from our stack traces, we can safely render
+  // the helper / original exception locations.
+  const parsedStack = stack ? prepareErrorStack(stack) : undefined;
+  tokens.push(parsedStack?.message || message);
+
+  let location = error.location;
+  if (parsedStack && !location)
+    location = parsedStack.location;
+
+  if (location) {
+    try {
+      const source = fs.readFileSync(location.file, 'utf8');
+      const codeFrame = codeFrameColumns(source, { start: location }, { highlightCode });
+      // Convert /var/folders to /private/var/folders on Mac.
+      if (!file || fs.realpathSync(file) !== location.file) {
         tokens.push('');
-        tokens.push(codeFrame);
-      } catch (e) {
-        // Failed to read the source file - that's ok.
+        tokens.push(colors.gray(`   at `) + `${relativeFilePath(config, location.file)}:${location.line}`);
       }
+      tokens.push('');
+      tokens.push(codeFrame);
+    } catch (e) {
+      // Failed to read the source file - that's ok.
     }
-    tokens.push('');
-    tokens.push(colors.dim(parsed.stackLines.join('\n')));
-  } else if (error.message) {
-    tokens.push(error.message);
-  } else if (error.value) {
-    tokens.push(error.value);
   }
+  if (parsedStack) {
+    tokens.push('');
+    tokens.push(colors.dim(parsedStack.stackLines.join('\n')));
+  }
+
   return {
     location,
     message: tokens.join('\n'),

packages/playwright-test/src/runner.ts

@@ -17,7 +17,6 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import { MultiMap } from 'playwright-core/lib/utils/multimap';
 import { raceAgainstTimeout } from 'playwright-core/lib/utils/timeoutRunner';
 import { colors, minimatch, rimraf } from 'playwright-core/lib/utilsBundle';
 import { promisify } from 'util';
@@ -177,7 +176,8 @@ export class Runner {
     const result = await raceAgainstTimeout(() => this._run(options), config.globalTimeout);
     let fullResult: FullResult;
     if (result.timedOut) {
-      this._reporter.onError?.(createStacklessError(`Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
+      this._reporter.onError?.(createStacklessError(
+          `Timed out waiting ${config.globalTimeout / 1000}s for the entire test run`));
       fullResult = { status: 'timedout' };
     } else {
       fullResult = result.result;
@@ -325,9 +325,7 @@ export class Runner {
     }
 
     // Complain about duplicate titles.
-    const duplicateTitlesError = createDuplicateTitlesError(config, preprocessRoot);
-    if (duplicateTitlesError)
-      fatalErrors.push(duplicateTitlesError);
+    fatalErrors.push(...createDuplicateTitlesErrors(config, preprocessRoot));
 
     // Filter tests to respect line/column filter.
     filterByFocusedLine(preprocessRoot, options.testFileFilters, doNotFilterFiles);
@@ -336,7 +334,7 @@ export class Runner {
     if (config.forbidOnly) {
       const onlyTestsAndSuites = preprocessRoot._getOnlyItems();
       if (onlyTestsAndSuites.length > 0)
-        fatalErrors.push(createForbidOnlyError(config, onlyTestsAndSuites));
+        fatalErrors.push(...createForbidOnlyErrors(config, onlyTestsAndSuites));
     }
 
     // Filter only.
@@ -782,7 +780,6 @@ async function collectFiles(testDir: string, respectGitIgnore: boolean): Promise
   return files;
 }
 
-
 function buildItemLocation(rootDir: string, testOrSuite: Suite | TestCase) {
   if (!testOrSuite.location)
     return '';
@@ -927,53 +924,46 @@ class ListModeReporter implements Reporter {
   }
 }
 
-function createForbidOnlyError(config: FullConfigInternal, onlyTestsAndSuites: (TestCase | Suite)[]): TestError {
-  const errorMessage = [
-    '=====================================',
-    ' --forbid-only found a focused test.',
-  ];
+function createForbidOnlyErrors(config: FullConfigInternal, onlyTestsAndSuites: (TestCase | Suite)[]): TestError[] {
+  const errors: TestError[] = [];
   for (const testOrSuite of onlyTestsAndSuites) {
     // Skip root and file.
     const title = testOrSuite.titlePath().slice(2).join(' ');
-    errorMessage.push(` - ${buildItemLocation(config.rootDir, testOrSuite)} > ${title}`);
+    const error: TestError = {
+      message: `Error: focused item found in the --forbid-only mode: "${title}"`,
+      location: testOrSuite.location!,
+    };
+    errors.push(error);
   }
-  errorMessage.push('=====================================');
-  return createStacklessError(errorMessage.join('\n'));
+  return errors;
 }
 
-function createDuplicateTitlesError(config: FullConfigInternal, rootSuite: Suite): TestError | undefined {
-  const lines: string[] = [];
+function createDuplicateTitlesErrors(config: FullConfigInternal, rootSuite: Suite): TestError[] {
+  const errors: TestError[] = [];
   for (const fileSuite of rootSuite.suites) {
-    const testsByFullTitle = new MultiMap<string, TestCase>();
+    const testsByFullTitle = new Map<string, TestCase>();
     for (const test of fileSuite.allTests()) {
       const fullTitle = test.titlePath().slice(2).join('\x1e');
+      const existingTest = testsByFullTitle.get(fullTitle);
+      if (existingTest) {
+        const error: TestError = {
+          message: `Error: duplicate test title "${fullTitle}", first declared in ${buildItemLocation(config.rootDir, existingTest)}`,
+          location: test.location,
+        };
+        errors.push(error);
+      }
       testsByFullTitle.set(fullTitle, test);
     }
-    for (const fullTitle of testsByFullTitle.keys()) {
-      const tests = testsByFullTitle.get(fullTitle);
-      if (tests.length > 1) {
-        lines.push(` - title: ${fullTitle.replace(/\u001e/g, ' › ')}`);
-        for (const test of tests)
-          lines.push(`   - ${buildItemLocation(config.rootDir, test)}`);
-      }
-    }
   }
-  if (!lines.length)
-    return;
-  return createStacklessError([
-    '========================================',
-    ' duplicate test titles are not allowed.',
-    ...lines,
-    '========================================',
-  ].join('\n'));
+  return errors;
 }
 
 function createNoTestsError(): TestError {
   return createStacklessError(`=================\n no tests found.\n=================`);
 }
 
-function createStacklessError(message: string): TestError {
-  return { message, __isNotAFatalError: true } as any;
+function createStacklessError(message: string, location?: TestError['location']): TestError {
+  return { message, location };
 }
 
 function sanitizeConfigForJSON(object: any, visited: Set<any>): any {

packages/playwright-test/src/testInfo.ts

@@ -17,7 +17,7 @@
 import fs from 'fs';
 import path from 'path';
 import { monotonicTime } from 'playwright-core/lib/utils';
-import type { TestError, TestInfo, TestStatus } from '../types/test';
+import type { TestInfoError, TestInfo, TestStatus } from '../types/test';
 import type { WorkerInitParams } from './ipc';
 import type { Loader } from './loader';
 import type { TestCase } from './test';
@@ -58,14 +58,14 @@ export class TestInfoImpl implements TestInfo {
   snapshotSuffix: string = '';
   readonly outputDir: string;
   readonly snapshotDir: string;
-  errors: TestError[] = [];
+  errors: TestInfoError[] = [];
   currentStep: TestStepInternal | undefined;
 
-  get error(): TestError | undefined {
+  get error(): TestInfoError | undefined {
     return this.errors[0];
   }
 
-  set error(e: TestError | undefined) {
+  set error(e: TestInfoError | undefined) {
     if (e === undefined)
       throw new Error('Cannot assign testInfo.error undefined value!');
     this.errors[0] = e;
@@ -168,7 +168,7 @@ export class TestInfoImpl implements TestInfo {
     this.duration = this._timeoutManager.defaultSlotTimings().elapsed | 0;
   }
 
-  async _runFn(fn: Function, skips?: 'allowSkips'): Promise<TestError | undefined> {
+  async _runFn(fn: Function, skips?: 'allowSkips'): Promise<TestInfoError | undefined> {
     try {
       await fn();
     } catch (error) {
@@ -187,7 +187,7 @@ export class TestInfoImpl implements TestInfo {
     return this._addStepImpl(data);
   }
 
-  _failWithError(error: TestError, isHardError: boolean) {
+  _failWithError(error: TestInfoError, isHardError: boolean) {
     // Do not overwrite any previous hard errors.
     // Some (but not all) scenarios include:
     //   - expect() that fails after uncaught exception.

packages/playwright-test/src/timeoutManager.ts

@@ -16,8 +16,7 @@
 
 import { colors } from 'playwright-core/lib/utilsBundle';
 import { TimeoutRunner, TimeoutRunnerError } from 'playwright-core/lib/utils/timeoutRunner';
-import type { TestError } from '../types/test';
-import type { Location } from './types';
+import type { Location, TestInfoError } from './types';
 
 export type TimeSlot = {
   timeout: number;
@@ -72,7 +71,7 @@ export class TimeoutManager {
     this._timeoutRunner.updateTimeout(slot.timeout);
   }
 
-  async runWithTimeout(cb: () => Promise<any>): Promise<TestError | undefined> {
+  async runWithTimeout(cb: () => Promise<any>): Promise<TestInfoError | undefined> {
     try {
       await this._timeoutRunner.run(cb);
     } catch (error) {
@@ -105,7 +104,7 @@ export class TimeoutManager {
     this._timeoutRunner.updateTimeout(slot.timeout, slot.elapsed);
   }
 
-  private _createTimeoutError(): TestError {
+  private _createTimeoutError(): TestInfoError {
     let message = '';
     const timeout = this._currentSlot().timeout;
     switch (this._runnable.type) {
@@ -142,7 +141,7 @@ export class TimeoutManager {
     return {
       message,
       // Include location for hooks, modifiers and fixtures to distinguish between them.
-      stack: location ? message + `\n    at ${location.file}:${location.line}:${location.column}` : undefined,
+      stack: location ? message + `\n    at ${location.file}:${location.line}:${location.column}` : undefined
     };
   }
 }

packages/playwright-test/src/types.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import type { Fixtures, TestError, Project } from '../types/test';
+import type { Fixtures, TestInfoError, Project } from '../types/test';
 import type { Location, Reporter } from '../types/testReporter';
 import type { WorkerIsolation } from './ipc';
 import type { FullConfig as FullConfigPublic, FullProject as FullProjectPublic } from './types';
@@ -28,7 +28,7 @@ export type FixturesWithLocation = {
 export type Annotation = { type: string, description?: string };
 
 export interface TestStepInternal {
-  complete(result: { error?: Error | TestError }): void;
+  complete(result: { error?: Error | TestInfoError }): void;
   title: string;
   category: string;
   canHaveChildren: boolean;

packages/playwright-test/src/util.ts

@@ -20,7 +20,7 @@ import util from 'util';
 import path from 'path';
 import url from 'url';
 import { colors, debug, minimatch } from 'playwright-core/lib/utilsBundle';
-import type { TestError, Location } from './types';
+import type { TestInfoError, Location } from './types';
 import { calculateSha1, isRegExp, isString } from 'playwright-core/lib/utils';
 import { isInternalFileName } from 'playwright-core/lib/utils/stackTrace';
 import { currentTestInfo } from './globals';
@@ -86,7 +86,7 @@ export function captureStackTrace(customApiName?: string): ParsedStackTrace {
   };
 }
 
-export function serializeError(error: Error | any): TestError {
+export function serializeError(error: Error | any): TestInfoError {
   if (error instanceof Error) {
     filterStackTrace(error);
     return {

packages/playwright-test/src/workerRunner.ts

@@ -22,7 +22,7 @@ import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerI
 import { setCurrentTestInfo } from './globals';
 import { Loader } from './loader';
 import type { Suite, TestCase } from './test';
-import type { Annotation, FullProjectInternal, TestError, TestStepInternal } from './types';
+import type { Annotation, FullProjectInternal, TestInfoError, TestStepInternal } from './types';
 import { FixtureRunner } from './fixtures';
 import { ManualPromise } from 'playwright-core/lib/utils/manualPromise';
 import { TestInfoImpl } from './testInfo';
@@ -38,7 +38,7 @@ export class WorkerRunner extends EventEmitter {
   private _fixtureRunner: FixtureRunner;
 
   // Accumulated fatal errors that cannot be attributed to a test.
-  private _fatalErrors: TestError[] = [];
+  private _fatalErrors: TestInfoError[] = [];
   // Whether we should skip running remaining tests in this suite because
   // of a setup error, usually beforeAll hook.
   private _skipRemainingTestsInSuite: Suite | undefined;
@@ -91,7 +91,7 @@ export class WorkerRunner extends EventEmitter {
     }
   }
 
-  appendWorkerTeardownDiagnostics(error: TestError) {
+  appendWorkerTeardownDiagnostics(error: TestInfoError) {
     if (!this._lastRunningTests.length)
       return;
     const count = this._totalRunningTests === 1 ? '1 test' : `${this._totalRunningTests} tests`;
@@ -408,7 +408,7 @@ export class WorkerRunner extends EventEmitter {
       canHaveChildren: true,
       forceNoParent: true
     });
-    let firstAfterHooksError: TestError | undefined;
+    let firstAfterHooksError: TestInfoError | undefined;
 
     let afterHooksSlot: TimeSlot | undefined;
     if (testInfo._didTimeout) {
@@ -561,7 +561,7 @@ export class WorkerRunner extends EventEmitter {
     if (!this._activeSuites.has(suite))
       return;
     this._activeSuites.delete(suite);
-    let firstError: TestError | undefined;
+    let firstError: TestInfoError | undefined;
     for (const hook of suite._hooks) {
       if (hook.type !== 'afterAll')
         continue;

packages/playwright-test/types/test.d.ts

@@ -1615,12 +1615,12 @@ export interface TestInfo {
    * First error thrown during test execution, if any. This is equal to the first element in
    * [testInfo.errors](https://playwright.dev/docs/api/class-testinfo#test-info-errors).
    */
-  error?: TestError;
+  error?: TestInfoError;
 
   /**
    * Errors thrown during test execution, if any.
    */
-  errors: Array<TestError>;
+  errors: Array<TestInfoError>;
 
   /**
    * Expected status for the currently running test. This is usually `'passed'`, except for a few cases:
@@ -4458,7 +4458,7 @@ interface SnapshotAssertions {
 /**
  * Information about an error thrown during test execution.
  */
-export interface TestError {
+export interface TestInfoError {
   /**
    * Error message. Set when [Error] (or its subclass) has been thrown.
    */

packages/playwright-test/types/testReporter.d.ts

@@ -15,8 +15,8 @@
  * limitations under the License.
  */
 
-import type { FullConfig, FullProject, TestStatus, TestError, Metadata } from './test';
-export type { FullConfig, TestStatus, TestError } from './test';
+import type { FullConfig, FullProject, TestStatus, Metadata } from './test';
+export type { FullConfig, TestStatus } from './test';
 
 /**
  * `Suite` is a group of tests. All tests in Playwright Test form the following hierarchy:
@@ -563,6 +563,31 @@ export interface Location {
   column: number;
 }
 
+/**
+ * Information about an error thrown during test execution.
+ */
+export interface TestError {
+  /**
+   * Error message. Set when [Error] (or its subclass) has been thrown.
+   */
+  message?: string;
+
+  /**
+   * Error stack. Set when [Error] (or its subclass) has been thrown.
+   */
+  stack?: string;
+
+  /**
+   * The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown.
+   */
+  value?: string;
+
+  /**
+   * Error location in the source code.
+   */
+  location?: Location;
+}
+
 /**
  * Represents a step in the [TestRun].
  */

tests/playwright-test/hooks.spec.ts

@@ -636,7 +636,7 @@ test('should not hang and report results when worker process suddenly exits duri
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(1);
-  expect(result.output).toContain('Worker process exited unexpectedly');
+  expect(result.output).toContain('Internal error: worker process exited unexpectedly');
   expect(stripAnsi(result.output)).toContain('[1/1] a.spec.js:6:7 › failing due to afterall');
 });
 

tests/playwright-test/reporter.spec.ts

@@ -475,7 +475,7 @@ test('should report forbid-only error to reporter', async ({ runInlineTest }) =>
   }, { 'reporter': '', 'forbid-only': true });
 
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`%%got error: =====================================\n --forbid-only found a focused test.`);
+  expect(result.output).toContain(`%%got error: Error: focused item found in the --forbid-only mode`);
 });
 
 test('should report no-tests error to reporter', async ({ runInlineTest }) => {

tests/playwright-test/runner.spec.ts

@@ -29,10 +29,10 @@ test('it should not allow multiple tests with the same name per suite', async ({
     `
   });
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain('duplicate test titles are not allowed');
-  expect(result.output).toContain(`- title: suite › i-am-a-duplicate`);
-  expect(result.output).toContain(`  - tests${path.sep}example.spec.js:7`);
-  expect(result.output).toContain(`  - tests${path.sep}example.spec.js:10`);
+  expect(result.output).toContain(`Error: duplicate test title`);
+  expect(result.output).toContain(`i-am-a-duplicate`);
+  expect(result.output).toContain(`tests${path.sep}example.spec.js:7`);
+  expect(result.output).toContain(`tests${path.sep}example.spec.js:10`);
 });
 
 test('it should not allow multiple tests with the same name in multiple files', async ({ runInlineTest }) => {
@@ -49,13 +49,12 @@ test('it should not allow multiple tests with the same name in multiple files',
     `,
   });
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain('duplicate test titles are not allowed');
-  expect(result.output).toContain(`- title: i-am-a-duplicate`);
-  expect(result.output).toContain(`  - tests${path.sep}example1.spec.js:6`);
-  expect(result.output).toContain(`  - tests${path.sep}example1.spec.js:7`);
-  expect(result.output).toContain(`- title: i-am-a-duplicate`);
-  expect(result.output).toContain(`  - tests${path.sep}example2.spec.js:6`);
-  expect(result.output).toContain(`  - tests${path.sep}example2.spec.js:7`);
+  expect(result.output).toContain('Error: duplicate test title');
+  expect(stripAnsi(result.output)).toContain(`test('i-am-a-duplicate'`);
+  expect(result.output).toContain(`tests${path.sep}example1.spec.js:6`);
+  expect(result.output).toContain(`tests${path.sep}example1.spec.js:7`);
+  expect(result.output).toContain(`tests${path.sep}example2.spec.js:6`);
+  expect(result.output).toContain(`tests${path.sep}example2.spec.js:7`);
 });
 
 test('it should not allow a focused test when forbid-only is used', async ({ runInlineTest }) => {
@@ -66,8 +65,9 @@ test('it should not allow a focused test when forbid-only is used', async ({ run
     `
   }, { 'forbid-only': true });
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain('--forbid-only found a focused test.');
-  expect(result.output).toContain(`- tests${path.sep}focused-test.spec.js:6 > i-am-focused`);
+  expect(result.output).toContain('Error: focused item found in the --forbid-only mode');
+  expect(stripAnsi(result.output)).toContain(`test.only('i-am-focused'`);
+  expect(result.output).toContain(`tests${path.sep}focused-test.spec.js:6`);
 });
 
 test('should continue with other tests after worker process suddenly exits', async ({ runInlineTest }) => {
@@ -85,7 +85,7 @@ test('should continue with other tests after worker process suddenly exits', asy
   expect(result.passed).toBe(4);
   expect(result.failed).toBe(1);
   expect(result.skipped).toBe(0);
-  expect(result.output).toContain('Worker process exited unexpectedly');
+  expect(result.output).toContain('Internal error: worker process exited unexpectedly');
 });
 
 test('sigint should stop workers', async ({ runInlineTest }) => {
@@ -321,7 +321,7 @@ test('should not hang if test suites in worker are inconsistent with runner', as
   expect(result.passed).toBe(1);
   expect(result.failed).toBe(1);
   expect(result.skipped).toBe(1);
-  expect(result.report.suites[0].specs[1].tests[0].results[0].error!.message).toBe('Unknown test(s) in worker:\nproject-name > a.spec.js > Test 1 - bar\nproject-name > a.spec.js > Test 2 - baz');
+  expect(result.report.suites[0].specs[1].tests[0].results[0].error!.message).toBe('Internal error: unknown test(s) in worker:\nproject-name > a.spec.js > Test 1 - bar\nproject-name > a.spec.js > Test 2 - baz');
 });
 
 test('sigint should stop global setup', async ({ runInlineTest }) => {
@@ -452,13 +452,13 @@ test('should not crash with duplicate titles and .only', async ({ runInlineTest
     `
   });
   expect(result.exitCode).toBe(1);
-  expect(stripAnsi(result.output)).toContain([
-    ` duplicate test titles are not allowed.`,
-    ` - title: non unique title`,
-    `   - example.spec.ts:6`,
-    `   - example.spec.ts:7`,
-    `   - example.spec.ts:8`,
-  ].join('\n'));
+  expect(result.output).toContain(`Error: duplicate test title`);
+  expect(stripAnsi(result.output)).toContain(`test('non unique title'`);
+  expect(stripAnsi(result.output)).toContain(`test.skip('non unique title'`);
+  expect(stripAnsi(result.output)).toContain(`test.only('non unique title'`);
+  expect(result.output).toContain(`example.spec.ts:6`);
+  expect(result.output).toContain(`example.spec.ts:7`);
+  expect(result.output).toContain(`example.spec.ts:8`);
 });
 
 test('should not crash with duplicate titles and line filter', async ({ runInlineTest }) => {
@@ -471,13 +471,11 @@ test('should not crash with duplicate titles and line filter', async ({ runInlin
     `
   }, {}, {}, { additionalArgs: ['example.spec.ts:8'] });
   expect(result.exitCode).toBe(1);
-  expect(stripAnsi(result.output)).toContain([
-    ` duplicate test titles are not allowed.`,
-    ` - title: non unique title`,
-    `   - example.spec.ts:6`,
-    `   - example.spec.ts:7`,
-    `   - example.spec.ts:8`,
-  ].join('\n'));
+  expect(result.output).toContain(`Error: duplicate test title`);
+  expect(stripAnsi(result.output)).toContain(`test('non unique title'`);
+  expect(result.output).toContain(`example.spec.ts:6`);
+  expect(result.output).toContain(`example.spec.ts:7`);
+  expect(result.output).toContain(`example.spec.ts:8`);
 });
 
 test('should not load tests not matching filter', async ({ runInlineTest }) => {

utils/generate_types/overrides-testReporter.d.ts

@@ -14,8 +14,8 @@
  * limitations under the License.
  */
 
-import type { FullConfig, FullProject, TestStatus, TestError, Metadata } from './test';
-export type { FullConfig, TestStatus, TestError } from './test';
+import type { FullConfig, FullProject, TestStatus, Metadata } from './test';
+export type { FullConfig, TestStatus } from './test';
 
 export interface Suite {
   project(): FullProject | undefined;