diff --git a/docs/src/test-reporters-js.md b/docs/src/test-reporters-js.md index f00e1031bb..133b0eb63d 100644 --- a/docs/src/test-reporters-js.md +++ b/docs/src/test-reporters-js.md @@ -389,6 +389,151 @@ const config: PlaywrightTestConfig = { export default config; ``` +The JUnit reporter provides support for embedding additional information on the `testcase` elements using inner `properties`. This is based on an [evolved JUnit XML format](https://docs.getxray.app/display/XRAYCLOUD/Taking+advantage+of+JUnit+XML+reports) from Xray Test Management, but can also be used by other tools if they support this way of embedding additonal information for test results; please check it first. + +In configuration file, a set of options can be used to configure this behavior. A full example, in this case for Xray, follows ahead. + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ + +// JUnit reporter config for Xray +const xrayOptions = { + // Whether to add with all annotations; default is false + embedAnnotationsAsProperties: true, + + // By default, annotation is reported as . + // These annotations are reported as value. + textContentAnnotations: ['test_description'], + + // This will create a "testrun_evidence" property that contains all attachments. Each attachment is added as an inner element. + // Disables [[ATTACHMENT|path]] in the . + embedAttachmentsAsProperty: 'testrun_evidence', + + // Where to put the report. + outputFile: './xray-report.xml' +}; + +const config = { + reporter: [ ['junit', xrayOptions] ] +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.ts +import { PlaywrightTestConfig } from '@playwright/test'; + +// JUnit reporter config for Xray +const xrayOptions = { + // Whether to add with all annotations; default is false + embedAnnotationsAsProperties: true, + + // By default, annotation is reported as . + // These annotations are reported as value. + textContentAnnotations: ['test_description'], + + // This will create a "testrun_evidence" property that contains all attachments. Each attachment is added as an inner element. + // Disables [[ATTACHMENT|path]] in the . + embedAttachmentsAsProperty: 'testrun_evidence', + + // Where to put the report. + outputFile: './xray-report.xml' +}; + +const config: PlaywrightTestConfig = { + reporter: [ ['junit', xrayOptions] ] +}; + +export default config; +``` + +In the previous configuration sample, all annotations will be added as `` elements on the JUnit XML report. The annotation type is mapped to the `name` attribute of the ``, and the annotation description will be added as a `value` attribute. In this case, the exception will be the annotation type `testrun_evidence` whose description will be added as inner content on the respective ``. +Annotations can be used to, for example, link a Playwright test with an existing Test in Xray or to link a test with an existing story/requirement in Jira (i.e., "cover" it). + +```js js-flavor=js +// @ts-check +const { test } = require('@playwright/test'); + +test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => { + testInfo.annotations.push({ type: 'test_id', description: '1234' }); + testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' }); + testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' }); + testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' }); + testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); +}); +``` + +```js js-flavor=ts +import { test } from '@playwright/test'; + +test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => { + testInfo.annotations.push({ type: 'test_id', description: '1234' }); + testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' }); + testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' }); + testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' }); + testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); +}); +``` + +Please note that the semantics of these properties will depend on the tool that will process this evoled report format; there are no standard property names/annotations. + +If the configuration option `embedAttachmentsAsProperty` is defined, then a `property` with its name is created. Attachments, including their contents, will be embeded on the JUnit XML report inside `` elements under this `property`. Attachments are obtained from the `TestInfo` object, using either a path or a body, and are added as base64 encoded content. +Embedding attachments can be used to attach screenshots or any other relevant evidence; nevertheless, use it wisely as it affects the report size. + +The following configuration sample enables embedding attachments by using the `testrun_evidence` element on the JUnit XML report: + +```js js-flavor=js +// playwright.config.js +// @ts-check + +/** @type {import('@playwright/test').PlaywrightTestConfig} */ +const config = { + reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ], +}; + +module.exports = config; +``` + +```js js-flavor=ts +// playwright.config.js + +import { PlaywrightTestConfig } from '@playwright/test'; +const config: PlaywrightTestConfig = { + reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ], +}; + +export default config; +``` + +The following test adds attachments: + +```js js-flavor=js +// @ts-check +const { test } = require('@playwright/test'); + +test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => { + const file = testInfo.outputPath('evidence1.txt'); + require('fs').writeFileSync(file, 'hello', 'utf8'); + await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); + await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); +}); +``` + +```js js-flavor=ts +import { test } from '@playwright/test'; + +test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => { + const file = testInfo.outputPath('evidence1.txt'); + require('fs').writeFileSync(file, 'hello', 'utf8'); + await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); + await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); +}); +``` + ### GitHub Actions annotations You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions. diff --git a/packages/playwright-test/src/reporters/junit.ts b/packages/playwright-test/src/reporters/junit.ts index f8b5d39ce5..ec13e87e9d 100644 --- a/packages/playwright-test/src/reporters/junit.ts +++ b/packages/playwright-test/src/reporters/junit.ts @@ -30,10 +30,17 @@ class JUnitReporter implements Reporter { private totalSkipped = 0; private outputFile: string | undefined; private stripANSIControlSequences = false; + private embedAnnotationsAsProperties = false; + private textContentAnnotations: string[] | undefined; + private embedAttachmentsAsProperty: string | undefined; - constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean } = {}) { + + constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean, embedAnnotationsAsProperties?: boolean, textContentAnnotations?: string[], embedAttachmentsAsProperty?: string } = {}) { this.outputFile = options.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]; this.stripANSIControlSequences = options.stripANSIControlSequences || false; + this.embedAnnotationsAsProperties = options.embedAnnotationsAsProperties || false; + this.textContentAnnotations = options.textContentAnnotations || []; + this.embedAttachmentsAsProperty = options.embedAttachmentsAsProperty; } printsToStdio() { @@ -133,6 +140,85 @@ class JUnitReporter implements Reporter { }; entries.push(entry); + // Xray Test Management supports testcase level properties, where additional metadata may be provided + // some annotations are encoded as value attributes, other as cdata content; this implementation supports + // Xray JUnit extensions but it also agnostic, so other tools can also take advantage of this format + const properties: XMLEntry = { + name: 'properties', + children: [] as XMLEntry[] + }; + + if (this.embedAnnotationsAsProperties && test.annotations) { + for (const annotation of test.annotations) { + if (this.textContentAnnotations?.includes(annotation.type)) { + const property: XMLEntry = { + name: 'property', + attributes: { + name: annotation.type + }, + text: annotation.description + }; + properties.children?.push(property); + } else { + const property: XMLEntry = { + name: 'property', + attributes: { + name: annotation.type, + value: (annotation?.description ? annotation.description : '') + } + }; + properties.children?.push(property); + } + } + } + + const systemErr: string[] = []; + // attachments are optionally embed as base64 encoded content on inner elements + if (this.embedAttachmentsAsProperty) { + const evidence: XMLEntry = { + name: 'property', + attributes: { + name: this.embedAttachmentsAsProperty + }, + children: [] as XMLEntry[] + }; + for (const result of test.results) { + for (const attachment of result.attachments) { + let contents; + if (attachment.body) { + contents = attachment.body.toString('base64'); + } else { + if (!attachment.path) + continue; + try { + const attachmentPath = path.relative(this.config.rootDir, attachment.path); + if (fs.existsSync(attachmentPath)) + contents = fs.readFileSync(attachmentPath, { encoding: 'base64' }); + else + systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`); + } catch (e) { + } + } + + if (contents) { + const item: XMLEntry = { + name: 'item', + attributes: { + name: attachment.name + }, + text: contents + }; + evidence.children?.push(item); + } + + } + } + properties.children?.push(evidence); + } + + if (properties.children?.length) + entry.children.push(properties); + if (test.outcome() === 'skipped') { entry.children.push({ name: 'skipped' }); return; @@ -150,20 +236,21 @@ class JUnitReporter implements Reporter { } const systemOut: string[] = []; - const systemErr: string[] = []; for (const result of test.results) { systemOut.push(...result.stdout.map(item => item.toString())); systemErr.push(...result.stderr.map(item => item.toString())); - for (const attachment of result.attachments) { - if (!attachment.path) - continue; - try { - const attachmentPath = path.relative(this.config.rootDir, attachment.path); - if (fs.existsSync(attachment.path)) - systemOut.push(`\n[[ATTACHMENT|${attachmentPath}]]\n`); - else - systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`); - } catch (e) { + if (!this.embedAttachmentsAsProperty) { + for (const attachment of result.attachments) { + if (!attachment.path) + continue; + try { + const attachmentPath = path.relative(this.config.rootDir, attachment.path); + if (fs.existsSync(attachment.path)) + systemOut.push(`\n[[ATTACHMENT|${attachmentPath}]]\n`); + else + systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`); + } catch (e) { + } } } } diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts index cdc25142d1..c8aa4e1cb3 100644 --- a/tests/playwright-test/reporter-junit.spec.ts +++ b/tests/playwright-test/reporter-junit.spec.ts @@ -277,3 +277,142 @@ function parseXML(xml: string): any { xml2js.parseString(xml, (err, r) => result = r); return result; } + +test('should not render annotations to custom testcase properties by default', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('one', async ({}, testInfo) => { + testInfo.annotations.push({ type: 'unknown_annotation', description: 'unknown' }); + });2 + ` + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).not.toBeTruthy(); + expect(result.exitCode).toBe(0); +}); + +test('should render text content based annotations to custom testcase properties', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + const xrayOptions = { + embedAnnotationsAsProperties: true, + textContentAnnotations: ['test_description'] + } + module.exports = { + reporter: [ ['junit', xrayOptions] ], + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('one', async ({}, testInfo) => { + testInfo.annotations.push({ type: 'test_description', description: 'sample description' }); + testInfo.annotations.push({ type: 'unknown_annotation', description: 'unknown' }); + }); + ` + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).toBeTruthy(); + expect(testcase['properties'][0]['property'].length).toBe(2); + expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description'); + expect(testcase['properties'][0]['property'][0]['_']).toBe('\nsample description\n'); + expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('unknown_annotation'); + expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('unknown'); + expect(result.exitCode).toBe(0); +}); + +test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + const xrayOptions = { + embedAnnotationsAsProperties: true + } + module.exports = { + reporter: [ ['junit', xrayOptions] ], + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('one', async ({}, testInfo) => { + testInfo.annotations.push({ type: 'test_id', description: '1234' }); + testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' }); + testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' }); + testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' }); + }); + ` + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).toBeTruthy(); + expect(testcase['properties'][0]['property'].length).toBe(4); + expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id'); + expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234'); + expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key'); + expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2'); + expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary'); + expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary'); + expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements'); + expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6'); + expect(result.exitCode).toBe(0); +}); + +test('should embed attachments to a custom testcase property, if explictly requested', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'playwright.config.ts': ` + const xrayOptions = { + embedAttachmentsAsProperty: 'testrun_evidence' + } + module.exports = { + reporter: [ ['junit', xrayOptions] ], + }; + `, + 'a.test.js': ` + const { test } = pwt; + test('one', async ({}, testInfo) => { + const file = testInfo.outputPath('evidence1.txt'); + require('fs').writeFileSync(file, 'hello', 'utf8'); + testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' }); + testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' }); + // await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); + // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); + console.log('log here'); + }); + ` + }, { reporter: '' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).toBeTruthy(); + expect(testcase['properties'][0]['property'].length).toBe(1); + expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('testrun_evidence'); + expect(testcase['properties'][0]['property'][0]['item'][0]['$']['name']).toBe('evidence1.txt'); + expect(testcase['properties'][0]['property'][0]['item'][0]['_']).toBe('\naGVsbG8=\n'); + expect(testcase['properties'][0]['property'][0]['item'][1]['$']['name']).toBe('evidence2.txt'); + expect(testcase['properties'][0]['property'][0]['item'][1]['_']).toBe('\nd29ybGQ=\n'); + expect(testcase['system-out'].length).toBe(1); + expect(testcase['system-out'][0].trim()).toBe([ + `log here` + ].join('\n')); + expect(result.exitCode).toBe(0); +}); + +test('should not embed attachments to a custom testcase property, if not explictly requested', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.js': ` + const { test } = pwt; + test('one', async ({}, testInfo) => { + const file = testInfo.outputPath('evidence1.txt'); + require('fs').writeFileSync(file, 'hello', 'utf8'); + testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' }); + testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' }); + // await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' }); + // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' }); + }); + ` + }, { reporter: 'junit' }); + const xml = parseXML(result.output); + const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; + expect(testcase['properties']).not.toBeTruthy(); + expect(result.exitCode).toBe(0); +}); \ No newline at end of file