docs(test runner): api reference for reporter api (#7811)

docs/src/test-api/class-fixtures.md

@@ -201,9 +201,9 @@ test('basic test', async ({ page }) => {
 - type: <[Screenshot]<"off"|"on"|"only-on-failure">>
 
 Whether to automatically capture a screenshot after each test. Defaults to `'off'`.
-1. `'off'`: Do not capture screenshots.
-1. `'on'`: Capture screenshot after each test.
-1. `'only-on-failure'`: Capture screenshot after each test failure.
+* `'off'`: Do not capture screenshots.
+* `'on'`: Capture screenshot after each test.
+* `'only-on-failure'`: Capture screenshot after each test failure.
 
 Learn more about [automatic screenshots](./test-configuration.md#automatic-screenshots).
 
@@ -215,10 +215,10 @@ Learn more about [automatic screenshots](./test-configuration.md#automatic-scree
 - type: <[Screenshot]<"off"|"on"|"retain-on-failure"|"on-first-retry">>
 
 Whether to record a trace for each test. Defaults to `'off'`.
-1. `'off'`: Do not record a trace.
-1. `'on'`: Record a trace for each test.
-1. `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs.
-1. `'on-first-retry'`: Record a trace only when retrying a test for the first time.
+* `'off'`: Do not record a trace.
+* `'on'`: Record a trace for each test.
+* `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs.
+* `'on-first-retry'`: Record a trace only when retrying a test for the first time.
 
 Learn more about [recording trace](./test-configuration.md#record-test-trace).
 
@@ -232,10 +232,10 @@ Learn more about [recording trace](./test-configuration.md#record-test-trace).
     - `height` <[int]>
 
 Whether to record video for each test. Defaults to `'off'`.
-1. `'off'`: Do not record video.
-1. `'on'`: Record video for each test.
-1. `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
-1. `'on-first-retry'`: Record video only when retrying a test for the first time.
+* `'off'`: Do not record video.
+* `'on'`: Record video for each test.
+* `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
+* `'on-first-retry'`: Record video only when retrying a test for the first time.
 
 Learn more about [recording video](./test-configuration.md#record-video).
 

docs/src/test-api/class-testconfig.md

@@ -22,7 +22,7 @@ module.exports = config;
 
 ```js js-flavor=ts
 // playwright.config.ts
-import { PlaywrightTestConfig, devices } from '@playwright/test';
+import { PlaywrightTestConfig } from '@playwright/test';
 
 const config: PlaywrightTestConfig = {
   timeout: 30000,
@@ -150,9 +150,9 @@ test('example test', async ({}, testInfo) => {
 - type: <[PreserveOutput]<"always"|"never"|"failures-only">>
 
 Whether to preserve test output in the [`property: TestConfig.outputDir`]. Defaults to `'always'`.
-1. `'always'` - preserve output for all tests;
-1. `'never'` - do not preserve output for any tests;
-1. `'failures-only'` - only preserve output for failed tests.
+* `'always'` - preserve output for all tests;
+* `'never'` - do not preserve output for any tests;
+* `'failures-only'` - only preserve output for failed tests.
 
 
 ## property: TestConfig.projects
@@ -177,9 +177,9 @@ The number of times to repeat each test, useful for debugging flaky tests.
   - `1` <[Object]> An object with reporter options if any
 
 The list of reporters to use. Each reporter can be:
-1. A builtin reporter name like `'list'` or `'json'`.
-2. A module name like `'my-awesome-reporter'`.
-3. A relative path to the reporter like `'./reporters/my-awesome-reporter.js'`.
+* A builtin reporter name like `'list'` or `'json'`.
+* A module name like `'my-awesome-reporter'`.
+* A relative path to the reporter like `'./reporters/my-awesome-reporter.js'`.
 
 You can pass options to the reporter in a tuple like `['json', { outputFile: './report.json' }]`.
 
@@ -260,9 +260,9 @@ This is a base timeout for all tests. In addition, each test can configure its o
 - type: <[UpdateSnapshots]<"all"|"none"|"missing">>
 
 Whether to update expected snapshots with the actual results produced by the test run. Defaults to `'missing'`.
-1. `'all'` - All tests that are executed will update snapshots.
-1. `'none'` - No snapshots are updated.
-1. `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default.
+* `'all'` - All tests that are executed will update snapshots.
+* `'none'` - No snapshots are updated.
+* `'missing'` - Missing snapshots are created, for example when authoring a new test and running it for the first time. This is the default.
 
 Learn more about [snapshots](./test-snapshots.md).
 

docs/src/test-api/class-testerror.md

@@ -0,0 +1,19 @@
+# class: TestError
+* langs: js
+
+Information about an error thrown during test execution.
+
+## property: TestError.message
+- type: <[void]|[string]>
+
+Error message. Set when [Error] (or its subclass) has been thrown.
+
+## property: TestError.stack
+- type: <[void]|[string]>
+
+Error stack. Set when [Error] (or its subclass) has been thrown.
+
+## property: TestError.value
+- type: <[void]|[string]>
+
+The value that was thrown. Set when anything except the [Error] (or its subclass) has been thrown.

docs/src/test-api/class-testinfo.md

@@ -97,8 +97,8 @@ An error thrown during test execution, if any.
 - type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>
 
 Expected status for the currently running test. This is usually `'passed'`, except for a few cases:
-1. `'skipped'` for skipped tests, e.g. with [`method: Test.skip`];
-1. `'failed'` for tests marked as failed with [`method: Test.fail`].
+* `'skipped'` for skipped tests, e.g. with [`method: Test.skip`];
+* `'failed'` for tests marked as failed with [`method: Test.fail`].
 
 Expected status is usually compared with the actual [`property: TestInfo.status`]:
 

docs/src/test-reporter-api/class-location.md

@@ -0,0 +1,19 @@
+# class: Location
+* langs: js
+
+Represents a location in the source code where [TestCase] or [Suite] is defined.
+
+## property: Location.file
+- type: <[string]>
+
+Path to the source file.
+
+## property: Location.line
+- type: <[int]>
+
+Line number in the source file.
+
+## property: Location.column
+- type: <[string]>
+
+Column number in the source file.

docs/src/test-reporter-api/class-reporter.md

@@ -0,0 +1,182 @@
+# class: Reporter
+* langs: js
+
+Test runner notifies the reporter about various events during test execution. All methods of the reporter are optional.
+
+You can create a custom reporter my implementing a class with some of the reporter methods. Make sure to export this class as default.
+
+```js js-flavor=js
+// my-awesome-reporter.js
+// @ts-check
+
+/** @implements {import('@playwright/test/reporter').Reporter} */
+class MyReporter {
+  onBegin(config, suite) {
+    console.log(`Starting the run with ${suite.allTests().length} tests`);
+  }
+
+  onTestBegin(test) {
+    console.log(`Starting test ${test.title}`);
+  }
+
+  onTestEnd(test, result) {
+    console.log(`Finished test ${test.title}: ${result.status}`);
+  }
+
+  onEnd(result) {
+    console.log(`Finished the run: ${result.status}`);
+  }
+}
+
+module.exports = MyReporter;
+```
+
+```js js-flavor=ts
+// playwright.config.ts
+import { Reporter } from '@playwright/test/reporter';
+
+class MyReporter implements Reporter {
+  onBegin(config, suite) {
+    console.log(`Starting the run with ${suite.allTests().length} tests`);
+  }
+
+  onTestBegin(test) {
+    console.log(`Starting test ${test.title}`);
+  }
+
+  onTestEnd(test, result) {
+    console.log(`Finished test ${test.title}: ${result.status}`);
+  }
+
+  onEnd(result) {
+    console.log(`Finished the run: ${result.status}`);
+  }
+}
+export default MyReporter;
+```
+
+Now use this reporter with [`property: TestConfig.reporter`].
+
+```js js-flavor=js
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  reporter: './my-awesome-reporter.js',
+};
+
+module.exports = config;
+```
+
+```js js-flavor=ts
+// playwright.config.ts
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  reporter: './my-awesome-reporter.ts',
+};
+export default config;
+```
+
+Learn more about [reporters](./test-reporters.md).
+
+## method: Reporter.onBegin
+
+Called once before running tests. All tests have been already discovered and put into a hierarchy of [Suite]s.
+
+### param: Reporter.onBegin.config
+- `config` <[TestConfig]>
+
+Resolved configuration.
+
+### param: Reporter.onBegin.suite
+- `suite` <[Suite]>
+
+The root suite that contains all projects, files and test cases.
+
+
+
+## async method: Reporter.onEnd
+
+Called after all tests has been run, or testing has been interrupted. Note that this method may return a [Promise] and Playwright Test will await it.
+
+### param: Reporter.onEnd.result
+- `result` <[Object]>
+  - `status` <[FullStatus]<"passed"|"failed"|"timedout"|"interrupted">>
+
+Result of the full test run.
+* `'passed'` - Everything went as expected.
+* `'failed'` - Any test has failed.
+* `'timedout'` - The [`property: TestConfig.globalTimeout`] has been reached.
+* `'interrupted'` - Interrupted by the user.
+
+
+
+
+## method: Reporter.onError
+
+Called on some global error, for example unhandled exception in the worker process.
+
+### param: Reporter.onError.error
+- `error` <[TestError]>
+
+The error.
+
+
+## method: Reporter.onStdErr
+
+Called when something has been written to the standard error in the worker process.
+
+### param: Reporter.onStdErr.chunk
+- `chunk` <[string]|[Buffer]>
+
+Output chunk.
+
+### param: Reporter.onStdErr.test
+- `test` <[void]|[TestCase]>
+
+Test that was running. Note that output may happen when to test is running, in which case this will be [void].
+
+
+
+## method: Reporter.onStdOut
+
+Called when something has been written to the standard output in the worker process.
+
+### param: Reporter.onStdOut.chunk
+- `chunk` <[string]|[Buffer]>
+
+Output chunk.
+
+### param: Reporter.onStdOut.test
+- `test` <[void]|[TestCase]>
+
+Test that was running. Note that output may happen when to test is running, in which case this will be [void].
+
+
+
+## method: Reporter.onTestBegin
+
+Called after a test has been started in the worker process.
+
+### param: Reporter.onTestBegin.test
+- `test` <[TestCase]>
+
+Test that has been started.
+
+
+
+## method: Reporter.onTestEnd
+
+Called after a test has been finished in the worker process.
+
+### param: Reporter.onTestEnd.test
+- `test` <[TestCase]>
+
+Test that has been finished.
+
+### param: Reporter.onTestEnd.result
+- `result` <[TestResult]>
+
+Result of the test run.

docs/src/test-reporter-api/class-suite.md

@@ -0,0 +1,55 @@
+# class: Suite
+* langs: js
+
+`Suite` is a group of tests. All tests in Playwright Test form the following hierarchy:
+
+* Root suite has a child suite for each [TestProject].
+  * Project suite #1. Has a child suite for each test file in the project.
+    * File suite #1
+      * [TestCase] #1
+      * [TestCase] #2
+      * Suite corresponding to a [`method: Test.describe`] group
+        * [TestCase] #1 in a group
+        * [TestCase] #2 in a group
+      * < more test cases ... >
+    * File suite #2
+    * < more file suites ... >
+  * Project suite #2
+  * < more project suites ... >
+
+Reporter is given a root suite in the [`method: Reporter.onBegin`] method.
+
+## method: Suite.allTests
+- returns: <[Array]<[TestCase]>>
+
+Returns the list of all test cases in this suite and its descendants, as opposite to [`property: Suite.tests`].
+
+## property: Suite.location
+- type: <[void]|[Location]>
+
+Location in the source where the suite is defined. Missing for root and project suites.
+
+## property: Suite.suites
+- type: <[Array]<[Suite]>>
+
+Child suites. See [Suite] for the hierarchy of suites.
+
+## property: Suite.tests
+- type: <[Array]<[TestCase]>>
+
+Test cases in the suite. Note that only test cases defined directly in this suite are in the list. Any test cases defined in nested [`method: Test.describe`] groups are listed
+in the child [`property: Suite.suites`].
+
+## property: Suite.title
+- type: <[string]>
+
+Suite title.
+* Empty for root suite.
+* Project name for project suite.
+* File path for file suite.
+* Title passed to [`method: Test.describe`] for a group suite.
+
+## method: Suite.titlePath
+- returns: <[Array]<[string]>>
+
+Returns a list of titles from the root down to this suite.

docs/src/test-reporter-api/class-testcase.md

@@ -0,0 +1,70 @@
+# class: TestCase
+* langs: js
+
+`TestCase` corresponds to every [`method: Test.(call)`] call in a test file. When a single [`method: Test.(call)`] is running in multiple projects or repeated multiple times, it will have multiple `TestCase` objects in corresponding projects' suites.
+
+## property: TestCase.annotations
+- type: <[Array]<[Object]>>
+  - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
+  - `description` <[void]|[string]> Optional description.
+
+The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file.
+
+Annotations are available during test execution through [`property: TestInfo.annotations`].
+
+Learn more about [test annotations](./test-annotations.md).
+
+## property: TestCase.expectedStatus
+- type: <[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>
+
+Expected test status.
+* Tests marked as [`method: Test.skip`] or [`method: Test.fixme`] are expected to be `'skipped'`.
+* Tests marked as [`method: Test.fail`] are expected to be `'failed'`.
+* Other tests are expected to be `'passed'`.
+
+See also [`property: TestResult.status`] for the actual status.
+
+## property: TestCase.location
+- type: <[void]|[Location]>
+
+Location in the source where the test is defined.
+
+## method: TestCase.ok
+- returns: <[boolean]>
+
+Whether the test is considered running fine. Non-ok tests fail the test run with non-zero exit code.
+
+## method: TestCase.outcome
+- returns: <[TestOutcome]<"skipped"|"expected"|"unexpected"|"flaky">>
+
+Testing outcome for this test. Note that outcome is not the same as [`property: TestResult.status`]:
+* Test that is expected to fail and actually fails is `'expected'`.
+* Test that passes on a second retry is `'flaky'`.
+
+## property: TestCase.results
+- type: <[Array]<[TestResult]>>
+
+Results for each run of this test.
+
+## property: TestCase.retries
+- type: <[int]>
+
+The maximum number of retries given to this test in the configuration.
+
+Learn more about [test retries](./test-retries.md).
+
+## property: TestCase.timeout
+- type: <[float]>
+
+The timeout given to the test. Affected by [`property: TestConfig.timeout`], [`property: TestProject.timeout`], [`method: Test.setTimeout`], [`method: Test.slow`] and [`method: TestInfo.setTimeout`].
+
+## property: TestCase.title
+- type: <[string]>
+
+Test title as passed to the [`method: Test.(call)`] call.
+
+## method: TestCase.titlePath
+- returns: <[Array]<[string]>>
+
+Returns a list of titles from the root down to this test.
+

docs/src/test-reporter-api/class-testresult.md

@@ -0,0 +1,57 @@
+# class: TestResult
+* langs: js
+
+A result of a single [TestCase] run.
+
+## property: TestResult.attachments
+- type: <[Array]<[Object]>>
+  - `name` <[string]> Attachment name.
+  - `contentType` <[string]> Content type of this attachment to properly present in the report, for example `'application/json'` or `'image/png'`.
+  - `path` <[void]|[string]> Optional path on the filesystem to the attached file.
+  - `body` <[void]|[Buffer]> Optional attachment body used instead of a file.
+
+The list of files or buffers attached during the test execution through [`property: TestInfo.attachments`].
+
+## property: TestResult.duration
+- type: <[float]>
+
+Running time in milliseconds.
+
+## property: TestResult.error
+- type: <[void]|[TestError]>
+
+An error thrown during the test execution, if any.
+
+## property: TestResult.retry
+- type: <[int]>
+
+When test is retries multiple times, each retry attempt is given a sequential number.
+
+Learn more about [test retries](./test-retries.md).
+
+## property: TestResult.startTime
+- type: <[Date]>
+
+Start time of this particular test run.
+
+## property: TestResult.status
+- type: <[void]|[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>
+
+The status of this test result. See also [`property: TestCase.expectedStatus`].
+
+## property: TestResult.stderr
+- type: <[Array]<[string]|[Buffer]>>
+
+Anything written to the standard error during the test run.
+
+## property: TestResult.stdout
+- type: <[Array]<[string]|[Buffer]>>
+
+Anything written to the standard output during the test run.
+
+## property: TestResult.workerIndex
+- type: <[int]>
+
+Index of the worker where the test was run.
+
+Learn more about [parallelism and sharding](./test-parallel.md) with Playwright Test.

utils/doclint/cli.js

@@ -41,7 +41,9 @@ async function run() {
   apiDocumentation.filterForLanguage('js');
   const testDocumentation = parseApi(path.join(PROJECT_DIR, 'docs', 'src', 'test-api'), path.join(PROJECT_DIR, 'docs', 'src', 'api', 'params.md'));
   testDocumentation.filterForLanguage('js');
-  const documentation = apiDocumentation.mergeWith(testDocumentation);
+  const testRerpoterDocumentation = parseApi(path.join(PROJECT_DIR, 'docs', 'src', 'test-reporter-api'));
+  testRerpoterDocumentation.filterForLanguage('js');
+  const documentation = apiDocumentation.mergeWith(testDocumentation).mergeWith(testRerpoterDocumentation);
   documentation.mergeWith(testDocumentation);
 
   // This validates member links.