docs: update test reporters (#21337)

This commit is contained in:
Debbie O'Brien 2023-03-03 16:52:28 +01:00 committed by GitHub
parent eb3f8bfba8
commit e29a6e7bb0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -12,19 +12,7 @@ npx playwright test --reporter=line
For more control, you can specify reporters programmatically in the [configuration file](./test-configuration.md). For more control, you can specify reporters programmatically in the [configuration file](./test-configuration.md).
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'line',
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -36,22 +24,7 @@ export default defineConfig({
You can use multiple reporters at the same time. For example you can use `'list'` for nice terminal output and `'json'` to get a comprehensive json file with the test results. You can use multiple reporters at the same time. For example you can use `'list'` for nice terminal output and `'json'` to get a comprehensive json file with the test results.
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [
['list'],
['json', { outputFile: 'test-results.json' }]
],
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -66,20 +39,7 @@ export default defineConfig({
You can use different reporters locally and on CI. For example, using concise `'dot'` reporter avoids too much output. This is the default on CI. You can use different reporters locally and on CI. For example, using concise `'dot'` reporter avoids too much output. This is the default on CI.
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
// Concise 'dot' for CI, default 'list' when running locally
reporter: process.env.CI ? 'dot' : 'list',
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -100,19 +60,7 @@ List reporter is default (except on CI where the `dot` reporter is default). It
npx playwright test --reporter=list npx playwright test --reporter=list
``` ```
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'list',
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -139,19 +87,7 @@ Running 124 tests using 6 workers
You can opt into the step rendering via passing the following config option: You can opt into the step rendering via passing the following config option:
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['list', { printSteps: true }]],
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -167,19 +103,7 @@ Line reporter is more concise than the list reporter. It uses a single line to r
npx playwright test --reporter=line npx playwright test --reporter=line
``` ```
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'line',
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -209,19 +133,7 @@ Dot reporter is very concise - it only produces a single character per successfu
npx playwright test --reporter=dot npx playwright test --reporter=dot
``` ```
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: 'dot',
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -250,23 +162,7 @@ By default, HTML report is opened automatically if some of the tests failed. You
You can also configure `host` and `port` that are used to serve the HTML report. You can also configure `host` and `port` that are used to serve the HTML report.
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['html', {
open: 'never',
host: '0.0.0.0',
port: 9223,
}]],
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -278,19 +174,8 @@ By default, report is written into the `playwright-report` folder in the current
that location using the `PLAYWRIGHT_HTML_REPORT` environment variable or a reporter configuration. that location using the `PLAYWRIGHT_HTML_REPORT` environment variable or a reporter configuration.
In configuration file, pass options directly: In configuration file, pass options directly:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test'); ```js
module.exports = defineConfig({
reporter: [['html', { outputFolder: 'my-report' }]],
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -334,19 +219,8 @@ npx playwright test --reporter=json
``` ```
In configuration file, pass options directly: In configuration file, pass options directly:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test'); ```js
module.exports = defineConfig({
reporter: [['json', { outputFile: 'results.json' }]],
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -375,19 +249,8 @@ npx playwright test --reporter=junit
``` ```
In configuration file, pass options directly: In configuration file, pass options directly:
```js tab=js-js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test'); ```js
module.exports = defineConfig({
reporter: [['junit', { outputFile: 'results.xml' }]],
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -399,34 +262,7 @@ The JUnit reporter provides support for embedding additional information on the
In configuration file, a set of options can be used to configure this behavior. A full example, in this case for Xray, follows ahead. In configuration file, a set of options can be used to configure this behavior. A full example, in this case for Xray, follows ahead.
```js tab=js-js ```js
// playwright.config.js
// @ts-check
// JUnit reporter config for Xray
const xrayOptions = {
// Whether to add <properties> with all annotations; default is false
embedAnnotationsAsProperties: true,
// By default, annotation is reported as <property name='' value=''>.
// These annotations are reported as <property name=''>value</property>.
textContentAnnotations: ['test_description'],
// This will create a "testrun_evidence" property that contains all attachments. Each attachment is added as an inner <item> element.
// Disables [[ATTACHMENT|path]] in the <system-out>.
embedAttachmentsAsProperty: 'testrun_evidence',
// Where to put the report.
outputFile: './xray-report.xml'
};
module.exports = defineConfig({
reporter: [['junit', xrayOptions]]
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
// JUnit reporter config for Xray // JUnit reporter config for Xray
@ -454,20 +290,8 @@ export default defineConfig({
In the previous configuration sample, all annotations will be added as `<property>` elements on the JUnit XML report. The annotation type is mapped to the `name` attribute of the `<property>`, and the annotation description will be added as a `value` attribute. In this case, the exception will be the annotation type `testrun_evidence` whose description will be added as inner content on the respective `<property>`. In the previous configuration sample, all annotations will be added as `<property>` elements on the JUnit XML report. The annotation type is mapped to the `name` attribute of the `<property>`, and the annotation description will be added as a `value` attribute. In this case, the exception will be the annotation type `testrun_evidence` whose description will be added as inner content on the respective `<property>`.
Annotations can be used to, for example, link a Playwright test with an existing Test in Xray or to link a test with an existing story/requirement in Jira (i.e., "cover" it). Annotations can be used to, for example, link a Playwright test with an existing Test in Xray or to link a test with an existing story/requirement in Jira (i.e., "cover" it).
```js tab=js-js ```js
// @ts-check // example.spec.ts/js
const { test } = require('@playwright/test');
test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => {
testInfo.annotations.push({ type: 'test_id', description: '1234' });
testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
});
```
```js tab=js-ts
import { test } from '@playwright/test'; import { test } from '@playwright/test';
test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => { test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => {
@ -486,21 +310,9 @@ Embedding attachments can be used to attach screenshots or any other relevant ev
The following configuration sample enables embedding attachments by using the `testrun_evidence` element on the JUnit XML report: The following configuration sample enables embedding attachments by using the `testrun_evidence` element on the JUnit XML report:
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]],
});
```
```js tab=js-ts
// playwright.config.js
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]], reporter: [['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }]],
}); });
@ -508,19 +320,8 @@ export default defineConfig({
The following test adds attachments: The following test adds attachments:
```js tab=js-js ```js
// @ts-check // example.spec.ts/js
const { test } = require('@playwright/test');
test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => {
const file = testInfo.outputPath('evidence1.txt');
require('fs').writeFileSync(file, 'hello', 'utf8');
await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
});
```
```js tab=js-ts
import { test } from '@playwright/test'; import { test } from '@playwright/test';
test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => { test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => {
@ -537,21 +338,7 @@ You can use the built in `github` reporter to get automatic failure annotations
Note that all other reporters work on GitHub Actions as well, but do not provide annotations. Note that all other reporters work on GitHub Actions as well, but do not provide annotations.
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
// 'github' for GitHub Actions CI to generate annotations, plus a concise 'dot'
// default 'list' when running locally
reporter: process.env.CI ? 'github' : 'list',
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
@ -565,33 +352,7 @@ export default defineConfig({
You can create a custom reporter by implementing a class with some of the reporter methods. Learn more about the [Reporter] API. You can create a custom reporter by implementing a class with some of the reporter methods. Learn more about the [Reporter] API.
```js tab=js-js ```js
// my-awesome-reporter.js
// @ts-check
/** @implements {import('@playwright/test/reporter').Reporter} */
class MyReporter {
onBegin(config, suite) {
console.log(`Starting the run with ${suite.allTests().length} tests`);
}
onTestBegin(test) {
console.log(`Starting test ${test.title}`);
}
onTestEnd(test, result) {
console.log(`Finished test ${test.title}: ${result.status}`);
}
onEnd(result) {
console.log(`Finished the run: ${result.status}`);
}
}
module.exports = MyReporter;
```
```js tab=js-ts
// my-awesome-reporter.ts // my-awesome-reporter.ts
import { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter'; import { FullConfig, FullResult, Reporter, Suite, TestCase, TestResult } from '@playwright/test/reporter';
@ -618,31 +379,16 @@ export default MyReporter;
Now use this reporter with [`property: TestConfig.reporter`]. Now use this reporter with [`property: TestConfig.reporter`].
```js tab=js-js ```js
// playwright.config.js
// @ts-check
const { defineConfig } = require('@playwright/test');
module.exports = defineConfig({
reporter: './my-awesome-reporter.js',
});
```
```js tab=js-ts
// playwright.config.ts
import { defineConfig } from '@playwright/test'; import { defineConfig } from '@playwright/test';
export default defineConfig({ export default defineConfig({
reporter: './my-awesome-reporter.ts', reporter: './my-awesome-reporter.ts',
}); });
``` ```
## Third party reporter showcase ## Third party reporter showcase
* [Allure](https://www.npmjs.com/package/allure-playwright) * [Allure](https://www.npmjs.com/package/allure-playwright)
* [Monocart](https://github.com/cenfun/monocart-reporter) * [Monocart](https://github.com/cenfun/monocart-reporter)
* [Tesults](https://www.tesults.com/docs/playwright) * [Tesults](https://www.tesults.com/docs/playwright)
* [ReportPortal](https://github.com/reportportal/agent-js-playwright) * [ReportPortal](https://github.com/reportportal/agent-js-playwright)