diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 69a786bbe5..fcbc095f98 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -150,14 +150,24 @@ These are integration tests, making sure public API methods and events work as e
 - To run all tests:
 
 ```bash
+npx playwright install
 npm run test
 ```
 
+Be sure to run `npm run build` or let `npm run watch` run before you re-run the
+tests after making your changes to check them.
+
 - To run all tests in Chromium
 ```bash
 npm run ctest # also `ftest` for firefox and `wtest` for WebKit
 ```
 
+- To run the Playwright test runner tests
+```bash
+npm run ttest
+npm run ttest -- --grep "specific test"
+```
+
 - To run a specific test, substitute `it` with `it.only`, or use the `--grep 'My test'` CLI parameter:
 
 ```js
diff --git a/packages/playwright-test/src/runner/loadUtils.ts b/packages/playwright-test/src/runner/loadUtils.ts
index 55fd9a33b3..5b99829905 100644
--- a/packages/playwright-test/src/runner/loadUtils.ts
+++ b/packages/playwright-test/src/runner/loadUtils.ts
@@ -148,8 +148,10 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
   // Complain about only.
   if (config.config.forbidOnly) {
     const onlyTestsAndSuites = rootSuite._getOnlyItems();
-    if (onlyTestsAndSuites.length > 0)
-      errors.push(...createForbidOnlyErrors(onlyTestsAndSuites));
+    if (onlyTestsAndSuites.length > 0) {
+      const configFilePath = config.config.configFile ? path.relative(config.config.rootDir, config.config.configFile) : undefined;
+      errors.push(...createForbidOnlyErrors(onlyTestsAndSuites, config.configCLIOverrides.forbidOnly, configFilePath));
+    }
   }
 
   // Filter only for top-level projects.
@@ -220,13 +222,15 @@ async function createProjectSuite(fileSuites: Suite[], project: FullProjectInter
   return projectSuite;
 }
 
-function createForbidOnlyErrors(onlyTestsAndSuites: (TestCase | Suite)[]): TestError[] {
+function createForbidOnlyErrors(onlyTestsAndSuites: (TestCase | Suite)[], forbidOnlyCLIFlag: boolean | undefined, configFilePath: string | undefined): TestError[] {
   const errors: TestError[] = [];
   for (const testOrSuite of onlyTestsAndSuites) {
     // Skip root and file.
     const title = testOrSuite.titlePath().slice(2).join(' ');
+    const configFilePathName = configFilePath ? `'${configFilePath}'` : 'the Playwright configuration file';
+    const forbidOnlySource = forbidOnlyCLIFlag ? `'--forbid-only' CLI flag` : `'forbidOnly' option in ${configFilePathName}`;
     const error: TestError = {
-      message: `Error: focused item found in the --forbid-only mode: "${title}"`,
+      message: `Error: item focused with '.only' is not allowed due to the ${forbidOnlySource}: "${title}"`,
       location: testOrSuite.location!,
     };
     errors.push(error);
diff --git a/tests/playwright-test/reporter.spec.ts b/tests/playwright-test/reporter.spec.ts
index aecc6b9838..f15cd9224d 100644
--- a/tests/playwright-test/reporter.spec.ts
+++ b/tests/playwright-test/reporter.spec.ts
@@ -547,7 +547,7 @@ test('should report forbid-only error to reporter', async ({ runInlineTest }) =>
   }, { 'reporter': '', 'forbid-only': true });
 
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain(`%%got error: Error: focused item found in the --forbid-only mode`);
+  expect(result.output).toContain(`%%got error: Error: item focused with '.only' is not allowed due to the '--forbid-only' CLI flag: \"a.test.ts pass\"`);
 });
 
 test('should report no-tests error to reporter', async ({ runInlineTest }) => {
diff --git a/tests/playwright-test/runner.spec.ts b/tests/playwright-test/runner.spec.ts
index 8fcec7630b..8d48385a0c 100644
--- a/tests/playwright-test/runner.spec.ts
+++ b/tests/playwright-test/runner.spec.ts
@@ -59,13 +59,18 @@ test('it should not allow multiple tests with the same name in multiple files',
 
 test('it should not allow a focused test when forbid-only is used', async ({ runInlineTest }) => {
   const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        forbidOnly: true,
+      };
+    `,
     'tests/focused-test.spec.js': `
       import { test, expect } from '@playwright/test';
       test.only('i-am-focused', async () => {});
     `
-  }, { 'forbid-only': true });
+  });
   expect(result.exitCode).toBe(1);
-  expect(result.output).toContain('Error: focused item found in the --forbid-only mode');
+  expect(result.output).toContain(`Error: item focused with '.only' is not allowed due to the 'forbidOnly' option in 'playwright.config.ts': \"tests${path.sep}focused-test.spec.js i-am-focused\"`);
   expect(result.output).toContain(`test.only('i-am-focused'`);
   expect(result.output).toContain(`tests${path.sep}focused-test.spec.js:3`);
 });