feat(junit reporter): link testcases to Xray test issues and provide additional metadata for Xray Test Management (#11374)

Co-authored-by: Sergio Freire <sergio.freire@xpand-it.com>

docs/src/test-reporters-js.md

@@ -389,6 +389,151 @@ const config: PlaywrightTestConfig = {
 export default config;
 ```
 
+The JUnit reporter provides support for embedding additional information on the `testcase` elements using inner `properties`. This is based on an [evolved JUnit XML format](https://docs.getxray.app/display/XRAYCLOUD/Taking+advantage+of+JUnit+XML+reports) from Xray Test Management, but can also be used by other tools if they support this way of embedding additonal information for test results; please check it first.
+
+In configuration file, a set of options can be used to configure this behavior. A full example, in this case for Xray, follows ahead.
+
+```js js-flavor=js
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+
+// JUnit reporter config for Xray
+const xrayOptions = {
+  // Whether to add <properties> with all annotations; default is false
+  embedAnnotationsAsProperties: true,
+
+  // By default, annotation is reported as <property name='' value=''>.
+  // These annotations are reported as <property name=''>value</property>.
+  textContentAnnotations: ['test_description'],
+
+  // This will create a "testrun_evidence" property that contains all attachments. Each attachment is added as an inner <item> element.
+  // Disables [[ATTACHMENT|path]] in the <system-out>.
+  embedAttachmentsAsProperty: 'testrun_evidence',
+
+  // Where to put the report.
+  outputFile: './xray-report.xml'
+};
+
+const config = {
+  reporter: [ ['junit', xrayOptions] ]
+};
+
+module.exports = config;
+```
+
+```js js-flavor=ts
+// playwright.config.ts
+import { PlaywrightTestConfig } from '@playwright/test';
+
+// JUnit reporter config for Xray
+const xrayOptions = {
+  // Whether to add <properties> with all annotations; default is false
+  embedAnnotationsAsProperties: true,
+
+  // By default, annotation is reported as <property name='' value=''>.
+  // These annotations are reported as <property name=''>value</property>.
+  textContentAnnotations: ['test_description'],
+
+  // This will create a "testrun_evidence" property that contains all attachments. Each attachment is added as an inner <item> element.
+  // Disables [[ATTACHMENT|path]] in the <system-out>.
+  embedAttachmentsAsProperty: 'testrun_evidence',
+
+  // Where to put the report.
+  outputFile: './xray-report.xml'
+};
+
+const config: PlaywrightTestConfig = {
+  reporter: [ ['junit', xrayOptions] ]
+};
+
+export default config;
+```
+
+In the previous configuration sample, all annotations will be added as `<property>` elements on the JUnit XML report. The annotation type is mapped to the `name` attribute of the `<property>`, and the annotation description will be added as a `value` attribute. In this case, the exception will be the annotation type `testrun_evidence` whose description will be added as inner content on the respective `<property>`.
+Annotations can be used to, for example, link a Playwright test with an existing Test in Xray or to link a test with an existing story/requirement in Jira (i.e., "cover" it).
+
+```js js-flavor=js
+// @ts-check
+const { test } = require('@playwright/test');
+
+test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => {
+  testInfo.annotations.push({ type: 'test_id', description: '1234' });
+  testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
+  testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
+  testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
+  testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
+});
+```
+
+```js js-flavor=ts
+import { test } from '@playwright/test';
+
+test('using specific annotations for passing test metadata to Xray', async ({}, testInfo) => {
+  testInfo.annotations.push({ type: 'test_id', description: '1234' });
+  testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
+  testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
+  testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
+  testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
+});
+```
+
+Please note that the semantics of these properties will depend on the tool that will process this evoled report format; there are no standard property names/annotations.
+
+If the configuration option `embedAttachmentsAsProperty` is defined, then a `property` with its name is created. Attachments, including their contents, will be embeded on the JUnit XML report inside `<item>` elements under this `property`. Attachments are obtained from the `TestInfo` object, using either a path or a body, and are added as base64 encoded content.
+Embedding attachments can be used to attach screenshots or any other relevant evidence; nevertheless, use it wisely as it affects the report size.
+
+The following configuration sample enables embedding attachments by using the `testrun_evidence` element on the JUnit XML report:
+
+```js js-flavor=js
+// playwright.config.js
+// @ts-check
+
+/** @type {import('@playwright/test').PlaywrightTestConfig} */
+const config = {
+  reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ],
+};
+
+module.exports = config;
+```
+
+```js js-flavor=ts
+// playwright.config.js
+
+import { PlaywrightTestConfig } from '@playwright/test';
+const config: PlaywrightTestConfig = {
+  reporter: [ ['junit', { embedAttachmentsAsProperty: 'testrun_evidence', outputFile: 'results.xml' }] ],
+};
+
+export default config;
+```
+
+The following test adds attachments:
+
+```js js-flavor=js
+// @ts-check
+const { test } = require('@playwright/test');
+
+test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => {
+  const file = testInfo.outputPath('evidence1.txt');
+  require('fs').writeFileSync(file, 'hello', 'utf8');
+  await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
+  await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
+});
+```
+
+```js js-flavor=ts
+import { test } from '@playwright/test';
+
+test('embed attachments, including its content, on the JUnit report', async ({}, testInfo) => {
+  const file = testInfo.outputPath('evidence1.txt');
+  require('fs').writeFileSync(file, 'hello', 'utf8');
+  await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
+  await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
+});
+```
+
 ### GitHub Actions annotations
 
 You can use the built in `github` reporter to get automatic failure annotations when running in GitHub actions.

packages/playwright-test/src/reporters/junit.ts

@@ -30,10 +30,17 @@ class JUnitReporter implements Reporter {
   private totalSkipped = 0;
   private outputFile: string | undefined;
   private stripANSIControlSequences = false;
+  private embedAnnotationsAsProperties = false;
+  private textContentAnnotations: string[] | undefined;
+  private embedAttachmentsAsProperty: string | undefined;
 
-  constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean } = {}) {
+
+  constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean, embedAnnotationsAsProperties?: boolean, textContentAnnotations?: string[], embedAttachmentsAsProperty?: string } = {}) {
     this.outputFile = options.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`];
     this.stripANSIControlSequences = options.stripANSIControlSequences || false;
+    this.embedAnnotationsAsProperties = options.embedAnnotationsAsProperties || false;
+    this.textContentAnnotations = options.textContentAnnotations || [];
+    this.embedAttachmentsAsProperty = options.embedAttachmentsAsProperty;
   }
 
   printsToStdio() {
@@ -133,6 +140,85 @@ class JUnitReporter implements Reporter {
     };
     entries.push(entry);
 
+    // Xray Test Management supports testcase level properties, where additional metadata may be provided
+    // some annotations are encoded as value attributes, other as cdata content; this implementation supports
+    // Xray JUnit extensions but it also agnostic, so other tools can also take advantage of this format
+    const properties: XMLEntry = {
+      name: 'properties',
+      children: [] as XMLEntry[]
+    };
+
+    if (this.embedAnnotationsAsProperties && test.annotations) {
+      for (const annotation of test.annotations) {
+        if (this.textContentAnnotations?.includes(annotation.type)) {
+          const property: XMLEntry = {
+            name: 'property',
+            attributes: {
+              name: annotation.type
+            },
+            text: annotation.description
+          };
+          properties.children?.push(property);
+        } else {
+          const property: XMLEntry = {
+            name: 'property',
+            attributes: {
+              name: annotation.type,
+              value: (annotation?.description ? annotation.description : '')
+            }
+          };
+          properties.children?.push(property);
+        }
+      }
+    }
+
+    const systemErr: string[] = [];
+    // attachments are optionally embed as base64 encoded content on inner <item> elements
+    if (this.embedAttachmentsAsProperty) {
+      const evidence: XMLEntry = {
+        name: 'property',
+        attributes: {
+          name: this.embedAttachmentsAsProperty
+        },
+        children: [] as XMLEntry[]
+      };
+      for (const result of test.results) {
+        for (const attachment of result.attachments) {
+          let contents;
+          if (attachment.body) {
+            contents = attachment.body.toString('base64');
+          } else {
+            if (!attachment.path)
+              continue;
+            try {
+              const attachmentPath = path.relative(this.config.rootDir, attachment.path);
+              if (fs.existsSync(attachmentPath))
+                contents = fs.readFileSync(attachmentPath, { encoding: 'base64' });
+              else
+                systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`);
+            } catch (e) {
+            }
+          }
+
+          if (contents) {
+            const item: XMLEntry = {
+              name: 'item',
+              attributes: {
+                name: attachment.name
+              },
+              text: contents
+            };
+            evidence.children?.push(item);
+          }
+
+        }
+      }
+      properties.children?.push(evidence);
+    }
+
+    if (properties.children?.length)
+      entry.children.push(properties);
+
     if (test.outcome() === 'skipped') {
       entry.children.push({ name: 'skipped' });
       return;
@@ -150,20 +236,21 @@ class JUnitReporter implements Reporter {
     }
 
     const systemOut: string[] = [];
-    const systemErr: string[] = [];
     for (const result of test.results) {
       systemOut.push(...result.stdout.map(item => item.toString()));
       systemErr.push(...result.stderr.map(item => item.toString()));
-      for (const attachment of result.attachments) {
+      if (!this.embedAttachmentsAsProperty) {
-        if (!attachment.path)
+        for (const attachment of result.attachments) {
-          continue;
+          if (!attachment.path)
-        try {
+            continue;
-          const attachmentPath = path.relative(this.config.rootDir, attachment.path);
+          try {
-          if (fs.existsSync(attachment.path))
+            const attachmentPath = path.relative(this.config.rootDir, attachment.path);
-            systemOut.push(`\n[[ATTACHMENT|${attachmentPath}]]\n`);
+            if (fs.existsSync(attachment.path))
-          else
+              systemOut.push(`\n[[ATTACHMENT|${attachmentPath}]]\n`);
-            systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`);
+            else
-        } catch (e) {
+              systemErr.push(`\nWarning: attachment ${attachmentPath} is missing`);
+          } catch (e) {
+          }
         }
       }
     }

tests/playwright-test/reporter-junit.spec.ts

@@ -277,3 +277,142 @@ function parseXML(xml: string): any {
   xml2js.parseString(xml, (err, r) => result = r);
   return result;
 }
+
+test('should not render annotations to custom testcase properties by default', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        testInfo.annotations.push({ type: 'unknown_annotation', description: 'unknown' });
+      });2
+    `
+  }, { reporter: 'junit' });
+  const xml = parseXML(result.output);
+  const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+  expect(testcase['properties']).not.toBeTruthy();
+  expect(result.exitCode).toBe(0);
+});
+
+test('should render text content based annotations to custom testcase properties', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      const xrayOptions = {
+        embedAnnotationsAsProperties: true,
+        textContentAnnotations: ['test_description']
+      }
+      module.exports = {
+        reporter: [ ['junit', xrayOptions] ],
+      };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        testInfo.annotations.push({ type: 'test_description', description: 'sample description' });
+        testInfo.annotations.push({ type: 'unknown_annotation', description: 'unknown' });
+      });
+    `
+  }, { reporter: '' });
+  const xml = parseXML(result.output);
+  const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+  expect(testcase['properties']).toBeTruthy();
+  expect(testcase['properties'][0]['property'].length).toBe(2);
+  expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_description');
+  expect(testcase['properties'][0]['property'][0]['_']).toBe('\nsample description\n');
+  expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('unknown_annotation');
+  expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('unknown');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should render all annotations to testcase value based properties, if requested', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      const xrayOptions = {
+        embedAnnotationsAsProperties: true
+      }
+      module.exports = {
+        reporter: [ ['junit', xrayOptions] ],
+      };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        testInfo.annotations.push({ type: 'test_id', description: '1234' });
+        testInfo.annotations.push({ type: 'test_key', description: 'CALC-2' });
+        testInfo.annotations.push({ type: 'test_summary', description: 'sample summary' });
+        testInfo.annotations.push({ type: 'requirements', description: 'CALC-5,CALC-6' });
+      });
+    `
+  }, { reporter: '' });
+  const xml = parseXML(result.output);
+  const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+  expect(testcase['properties']).toBeTruthy();
+  expect(testcase['properties'][0]['property'].length).toBe(4);
+  expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('test_id');
+  expect(testcase['properties'][0]['property'][0]['$']['value']).toBe('1234');
+  expect(testcase['properties'][0]['property'][1]['$']['name']).toBe('test_key');
+  expect(testcase['properties'][0]['property'][1]['$']['value']).toBe('CALC-2');
+  expect(testcase['properties'][0]['property'][2]['$']['name']).toBe('test_summary');
+  expect(testcase['properties'][0]['property'][2]['$']['value']).toBe('sample summary');
+  expect(testcase['properties'][0]['property'][3]['$']['name']).toBe('requirements');
+  expect(testcase['properties'][0]['property'][3]['$']['value']).toBe('CALC-5,CALC-6');
+  expect(result.exitCode).toBe(0);
+});
+
+test('should embed attachments to a custom testcase property, if explictly requested', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      const xrayOptions = {
+        embedAttachmentsAsProperty: 'testrun_evidence'
+      }
+      module.exports = {
+        reporter: [ ['junit', xrayOptions] ],
+      };
+    `,
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        const file = testInfo.outputPath('evidence1.txt');
+        require('fs').writeFileSync(file, 'hello', 'utf8');
+        testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' });
+        testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' });
+        // await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
+        // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
+        console.log('log here');
+      });
+    `
+  }, { reporter: '' });
+  const xml = parseXML(result.output);
+  const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+  expect(testcase['properties']).toBeTruthy();
+  expect(testcase['properties'][0]['property'].length).toBe(1);
+  expect(testcase['properties'][0]['property'][0]['$']['name']).toBe('testrun_evidence');
+  expect(testcase['properties'][0]['property'][0]['item'][0]['$']['name']).toBe('evidence1.txt');
+  expect(testcase['properties'][0]['property'][0]['item'][0]['_']).toBe('\naGVsbG8=\n');
+  expect(testcase['properties'][0]['property'][0]['item'][1]['$']['name']).toBe('evidence2.txt');
+  expect(testcase['properties'][0]['property'][0]['item'][1]['_']).toBe('\nd29ybGQ=\n');
+  expect(testcase['system-out'].length).toBe(1);
+  expect(testcase['system-out'][0].trim()).toBe([
+    `log here`
+  ].join('\n'));
+  expect(result.exitCode).toBe(0);
+});
+
+test('should not embed attachments to a custom testcase property, if not explictly requested', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.test.js': `
+      const { test } = pwt;
+      test('one', async ({}, testInfo) => {
+        const file = testInfo.outputPath('evidence1.txt');
+        require('fs').writeFileSync(file, 'hello', 'utf8');
+        testInfo.attachments.push({ name: 'evidence1.txt', path: file, contentType: 'text/plain' });
+        testInfo.attachments.push({ name: 'evidence2.txt', body: Buffer.from('world'), contentType: 'text/plain' });
+        // await testInfo.attach('evidence1.txt', { path: file, contentType: 'text/plain' });
+        // await testInfo.attach('evidence2.txt', { body: Buffer.from('world'), contentType: 'text/plain' });
+      });
+    `
+  }, { reporter: 'junit' });
+  const xml = parseXML(result.output);
+  const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
+  expect(testcase['properties']).not.toBeTruthy();
+  expect(result.exitCode).toBe(0);
+});