fix(reporters): output relative to config (#17518)

Fixes #17412.
Supercedes #17413.

- if configured via playwright.config.ts, relative paths should be
relative to the config.
- if configured via env var, should be relative to `cwd`

packages/playwright-test/src/reporters/json.ts

@@ -19,6 +19,7 @@ import path from 'path';
 import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, Reporter, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep } from '../../types/testReporter';
 import { prepareErrorStack } from './base';
 import { MultiMap } from 'playwright-core/lib/utils/multimap';
+import { assert } from 'playwright-core/lib/utils';
 
 export function toPosixPath(aPath: string): string {
   return aPath.split(path.sep).join(path.posix.sep);
@@ -31,7 +32,7 @@ class JSONReporter implements Reporter {
   private _outputFile: string | undefined;
 
   constructor(options: { outputFile?: string } = {}) {
-    this._outputFile = options.outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`];
+    this._outputFile = options.outputFile || reportOutputNameFromEnv();
   }
 
   printsToStdio() {
@@ -48,7 +49,7 @@ class JSONReporter implements Reporter {
   }
 
   async onEnd(result: FullResult) {
-    outputReport(this._serializeReport(), this._outputFile);
+    outputReport(this._serializeReport(), this.config, this._outputFile);
   }
 
   private _serializeReport(): JSONReport {
@@ -210,9 +211,11 @@ class JSONReporter implements Reporter {
   }
 }
 
-function outputReport(report: JSONReport, outputFile: string | undefined) {
+function outputReport(report: JSONReport, config: FullConfig, outputFile: string | undefined) {
   const reportString = JSON.stringify(report, undefined, 2);
   if (outputFile) {
+    assert(config.configFile || path.isAbsolute(outputFile), 'Expected fully resolved path if not using config file.');
+    outputFile = config.configFile ? path.resolve(path.dirname(config.configFile), outputFile) : outputFile;
     fs.mkdirSync(path.dirname(outputFile), { recursive: true });
     fs.writeFileSync(outputFile, reportString);
   } else {
@@ -230,6 +233,12 @@ function removePrivateFields(config: FullConfig): FullConfig {
   return Object.fromEntries(Object.entries(config).filter(([name, value]) => !name.startsWith('_'))) as FullConfig;
 }
 
+function reportOutputNameFromEnv(): string | undefined {
+  if (process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`])
+    return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]);
+  return undefined;
+}
+
 export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] {
   if (!Array.isArray(patterns))
     patterns = [patterns];

packages/playwright-test/src/reporters/junit.ts

@@ -19,6 +19,7 @@ import path from 'path';
 import type { FullConfig, FullResult, Reporter, Suite, TestCase } from '../../types/testReporter';
 import { monotonicTime } from 'playwright-core/lib/utils';
 import { formatFailure, formatTestTitle, stripAnsiEscapes } from './base';
+import { assert } from 'playwright-core/lib/utils';
 
 class JUnitReporter implements Reporter {
   private config!: FullConfig;
@@ -36,7 +37,7 @@ class JUnitReporter implements Reporter {
 
 
   constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean, embedAnnotationsAsProperties?: boolean, textContentAnnotations?: string[], embedAttachmentsAsProperty?: string } = {}) {
-    this.outputFile = options.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`];
+    this.outputFile = options.outputFile || reportOutputNameFromEnv();
     this.stripANSIControlSequences = options.stripANSIControlSequences || false;
     this.embedAnnotationsAsProperties = options.embedAnnotationsAsProperties || false;
     this.textContentAnnotations = options.textContentAnnotations || [];
@@ -81,8 +82,10 @@ class JUnitReporter implements Reporter {
     serializeXML(root, tokens, this.stripANSIControlSequences);
     const reportString = tokens.join('\n');
     if (this.outputFile) {
-      fs.mkdirSync(path.dirname(this.outputFile), { recursive: true });
-      fs.writeFileSync(this.outputFile, reportString);
+      assert(this.config.configFile || path.isAbsolute(this.outputFile), 'Expected fully resolved path if not using config file.');
+      const outputFile = this.config.configFile ? path.resolve(path.dirname(this.config.configFile), this.outputFile) : this.outputFile;
+      fs.mkdirSync(path.dirname(outputFile), { recursive: true });
+      fs.writeFileSync(outputFile, reportString);
     } else {
       console.log(reportString);
     }
@@ -299,4 +302,10 @@ function escape(text: string, stripANSIControlSequences: boolean, isCharacterDat
   return text;
 }
 
+function reportOutputNameFromEnv(): string | undefined {
+  if (process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`])
+    return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]);
+  return undefined;
+}
+
 export default JUnitReporter;

tests/playwright-test/reporter-html.spec.ts

@@ -73,30 +73,6 @@ test('should generate report', async ({ runInlineTest, showReport, page }) => {
   await expect(page.locator('.metadata-view')).not.toBeVisible();
 });
 
-test('should generate report wrt package.json', async ({ runInlineTest }, testInfo) => {
-  const result = await runInlineTest({
-    'foo/package.json': `{ "name": "foo" }`,
-    'foo/bar/playwright.config.js': `
-      module.exports = { projects: [ {} ] };
-    `,
-    'foo/bar/baz/tests/a.spec.js': `
-      const { test } = pwt;
-      const fs = require('fs');
-      test('pass', ({}, testInfo) => {
-      });
-    `
-  }, { 'reporter': 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, {
-    cwd: 'foo/bar/baz/tests',
-    usesCustomOutputDir: true
-  });
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(1);
-  expect(fs.existsSync(testInfo.outputPath('playwright-report'))).toBe(false);
-  expect(fs.existsSync(testInfo.outputPath('foo', 'playwright-report'))).toBe(true);
-  expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'playwright-report'))).toBe(false);
-  expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'tests', 'playwright-report'))).toBe(false);
-});
-
 
 test('should not throw when attachment is missing', async ({ runInlineTest, page, showReport }, testInfo) => {
   const result = await runInlineTest({
@@ -911,3 +887,70 @@ test('should report clashing folders', async ({ runInlineTest }) => {
   expect(output).toContain('Configuration Error');
   expect(output).toContain('html-report');
 });
+
+test.describe('report location', () => {
+  test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'nested/project/playwright.config.ts': `
+        module.exports = { reporter: [['html', { outputFolder: '../my-report/' }]] };
+      `,
+      'nested/project/a.test.js': `
+        const { test } = pwt;
+        test('one', async ({}) => {
+          expect(1).toBe(1);
+        });
+      `,
+    }, { reporter: '', config: './nested/project/playwright.config.ts' });
+    expect(result.exitCode).toBe(0);
+    expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'index.html')))).toBeTruthy();
+  });
+
+  test('without config should create relative to package.json', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'foo/package.json': `{ "name": "foo" }`,
+      // unused config along "search path"
+      'foo/bar/playwright.config.js': `
+        module.exports = { projects: [ {} ] };
+      `,
+      'foo/bar/baz/tests/a.spec.js': `
+        const { test } = pwt;
+        const fs = require('fs');
+        test('pass', ({}, testInfo) => {
+        });
+      `
+    }, { 'reporter': 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, {
+      cwd: 'foo/bar/baz/tests',
+      usesCustomOutputDir: true
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(fs.existsSync(testInfo.outputPath('playwright-report'))).toBe(false);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'playwright-report'))).toBe(true);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'playwright-report'))).toBe(false);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'tests', 'playwright-report'))).toBe(false);
+  });
+
+  test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'foo/package.json': `{ "name": "foo" }`,
+      // unused config along "search path"
+      'foo/bar/playwright.config.js': `
+        module.exports = { projects: [ {} ] };
+      `,
+      'foo/bar/baz/tests/a.spec.js': `
+        const { test } = pwt;
+        const fs = require('fs');
+        test('pass', ({}, testInfo) => {
+        });
+      `
+    }, { 'reporter': 'html' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_HTML_REPORT': '../my-report' }, {
+      cwd: 'foo/bar/baz/tests',
+      usesCustomOutputDir: true
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report'))).toBe(true);
+  });
+});
+
+

tests/playwright-test/reporter-json.spec.ts

@@ -248,3 +248,43 @@ test('should have starting time in results', async ({ runInlineTest }, testInfo)
   const startTime = result.report.suites[0].specs[0].tests[0].results[0].startTime;
   expect(new Date(startTime).getTime()).toBeGreaterThan(new Date('1/1/2000').getTime());
 });
+
+test.describe('report location', () => {
+  test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'nested/project/playwright.config.ts': `
+        module.exports = { reporter: [['json', { outputFile: '../my-report/a.json' }]] };
+      `,
+      'nested/project/a.test.js': `
+        const { test } = pwt;
+        test('one', async ({}) => {
+          expect(1).toBe(1);
+        });
+      `,
+    }, { reporter: '', config: './nested/project/playwright.config.ts' });
+    expect(result.exitCode).toBe(0);
+    expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.json')))).toBeTruthy();
+  });
+
+  test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'foo/package.json': `{ "name": "foo" }`,
+      // unused config along "search path"
+      'foo/bar/playwright.config.js': `
+        module.exports = { projects: [ {} ] };
+      `,
+      'foo/bar/baz/tests/a.spec.js': `
+        const { test } = pwt;
+        const fs = require('fs');
+        test('pass', ({}, testInfo) => {
+        });
+      `
+    }, { 'reporter': 'json' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_JSON_OUTPUT_NAME': '../my-report.json' }, {
+      cwd: 'foo/bar/baz/tests',
+      usesCustomOutputDir: true
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true);
+  });
+});

tests/playwright-test/reporter-junit.spec.ts

@@ -17,6 +17,7 @@
 import xml2js from 'xml2js';
 import path from 'path';
 import { test, expect } from './playwright-test-fixtures';
+import fs from 'fs';
 
 test('should render expected', async ({ runInlineTest }) => {
   const result = await runInlineTest({
@@ -440,4 +441,45 @@ test('should not embed attachments to a custom testcase property, if not explict
   const testcase = xml['testsuites']['testsuite'][0]['testcase'][0];
   expect(testcase['properties']).not.toBeTruthy();
   expect(result.exitCode).toBe(0);
-});
+});
+
+
+test.describe('report location', () => {
+  test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'nested/project/playwright.config.ts': `
+        module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] };
+      `,
+      'nested/project/a.test.js': `
+        const { test } = pwt;
+        test('one', async ({}) => {
+          expect(1).toBe(1);
+        });
+      `,
+    }, { reporter: '', config: './nested/project/playwright.config.ts' });
+    expect(result.exitCode).toBe(0);
+    expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy();
+  });
+
+  test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => {
+    const result = await runInlineTest({
+      'foo/package.json': `{ "name": "foo" }`,
+      // unused config along "search path"
+      'foo/bar/playwright.config.js': `
+        module.exports = { projects: [ {} ] };
+      `,
+      'foo/bar/baz/tests/a.spec.js': `
+        const { test } = pwt;
+        const fs = require('fs');
+        test('pass', ({}, testInfo) => {
+        });
+      `
+    }, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, {
+      cwd: 'foo/bar/baz/tests',
+      usesCustomOutputDir: true
+    });
+    expect(result.exitCode).toBe(0);
+    expect(result.passed).toBe(1);
+    expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true);
+  });
+});