diff --git a/packages/playwright-test/src/reporters/json.ts b/packages/playwright-test/src/reporters/json.ts index 7cdf49d6ed..a1c8ee1cca 100644 --- a/packages/playwright-test/src/reporters/json.ts +++ b/packages/playwright-test/src/reporters/json.ts @@ -19,6 +19,7 @@ import path from 'path'; import type { FullConfig, TestCase, Suite, TestResult, TestError, TestStep, FullResult, Location, Reporter, JSONReport, JSONReportSuite, JSONReportSpec, JSONReportTest, JSONReportTestResult, JSONReportTestStep } from '../../types/testReporter'; import { prepareErrorStack } from './base'; import { MultiMap } from 'playwright-core/lib/utils/multimap'; +import { assert } from 'playwright-core/lib/utils'; export function toPosixPath(aPath: string): string { return aPath.split(path.sep).join(path.posix.sep); @@ -31,7 +32,7 @@ class JSONReporter implements Reporter { private _outputFile: string | undefined; constructor(options: { outputFile?: string } = {}) { - this._outputFile = options.outputFile || process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]; + this._outputFile = options.outputFile || reportOutputNameFromEnv(); } printsToStdio() { @@ -48,7 +49,7 @@ class JSONReporter implements Reporter { } async onEnd(result: FullResult) { - outputReport(this._serializeReport(), this._outputFile); + outputReport(this._serializeReport(), this.config, this._outputFile); } private _serializeReport(): JSONReport { @@ -210,9 +211,11 @@ class JSONReporter implements Reporter { } } -function outputReport(report: JSONReport, outputFile: string | undefined) { +function outputReport(report: JSONReport, config: FullConfig, outputFile: string | undefined) { const reportString = JSON.stringify(report, undefined, 2); if (outputFile) { + assert(config.configFile || path.isAbsolute(outputFile), 'Expected fully resolved path if not using config file.'); + outputFile = config.configFile ? path.resolve(path.dirname(config.configFile), outputFile) : outputFile; fs.mkdirSync(path.dirname(outputFile), { recursive: true }); fs.writeFileSync(outputFile, reportString); } else { @@ -230,6 +233,12 @@ function removePrivateFields(config: FullConfig): FullConfig { return Object.fromEntries(Object.entries(config).filter(([name, value]) => !name.startsWith('_'))) as FullConfig; } +function reportOutputNameFromEnv(): string | undefined { + if (process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]) + return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JSON_OUTPUT_NAME`]); + return undefined; +} + export function serializePatterns(patterns: string | RegExp | (string | RegExp)[]): string[] { if (!Array.isArray(patterns)) patterns = [patterns]; diff --git a/packages/playwright-test/src/reporters/junit.ts b/packages/playwright-test/src/reporters/junit.ts index 90b66b7f19..b8e74ab16d 100644 --- a/packages/playwright-test/src/reporters/junit.ts +++ b/packages/playwright-test/src/reporters/junit.ts @@ -19,6 +19,7 @@ import path from 'path'; import type { FullConfig, FullResult, Reporter, Suite, TestCase } from '../../types/testReporter'; import { monotonicTime } from 'playwright-core/lib/utils'; import { formatFailure, formatTestTitle, stripAnsiEscapes } from './base'; +import { assert } from 'playwright-core/lib/utils'; class JUnitReporter implements Reporter { private config!: FullConfig; @@ -36,7 +37,7 @@ class JUnitReporter implements Reporter { constructor(options: { outputFile?: string, stripANSIControlSequences?: boolean, embedAnnotationsAsProperties?: boolean, textContentAnnotations?: string[], embedAttachmentsAsProperty?: string } = {}) { - this.outputFile = options.outputFile || process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]; + this.outputFile = options.outputFile || reportOutputNameFromEnv(); this.stripANSIControlSequences = options.stripANSIControlSequences || false; this.embedAnnotationsAsProperties = options.embedAnnotationsAsProperties || false; this.textContentAnnotations = options.textContentAnnotations || []; @@ -81,8 +82,10 @@ class JUnitReporter implements Reporter { serializeXML(root, tokens, this.stripANSIControlSequences); const reportString = tokens.join('\n'); if (this.outputFile) { - fs.mkdirSync(path.dirname(this.outputFile), { recursive: true }); - fs.writeFileSync(this.outputFile, reportString); + assert(this.config.configFile || path.isAbsolute(this.outputFile), 'Expected fully resolved path if not using config file.'); + const outputFile = this.config.configFile ? path.resolve(path.dirname(this.config.configFile), this.outputFile) : this.outputFile; + fs.mkdirSync(path.dirname(outputFile), { recursive: true }); + fs.writeFileSync(outputFile, reportString); } else { console.log(reportString); } @@ -299,4 +302,10 @@ function escape(text: string, stripANSIControlSequences: boolean, isCharacterDat return text; } +function reportOutputNameFromEnv(): string | undefined { + if (process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]) + return path.resolve(process.cwd(), process.env[`PLAYWRIGHT_JUNIT_OUTPUT_NAME`]); + return undefined; +} + export default JUnitReporter; diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index faa994fa8d..306ffe87dd 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -73,30 +73,6 @@ test('should generate report', async ({ runInlineTest, showReport, page }) => { await expect(page.locator('.metadata-view')).not.toBeVisible(); }); -test('should generate report wrt package.json', async ({ runInlineTest }, testInfo) => { - const result = await runInlineTest({ - 'foo/package.json': `{ "name": "foo" }`, - 'foo/bar/playwright.config.js': ` - module.exports = { projects: [ {} ] }; - `, - 'foo/bar/baz/tests/a.spec.js': ` - const { test } = pwt; - const fs = require('fs'); - test('pass', ({}, testInfo) => { - }); - ` - }, { 'reporter': 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { - cwd: 'foo/bar/baz/tests', - usesCustomOutputDir: true - }); - expect(result.exitCode).toBe(0); - expect(result.passed).toBe(1); - expect(fs.existsSync(testInfo.outputPath('playwright-report'))).toBe(false); - expect(fs.existsSync(testInfo.outputPath('foo', 'playwright-report'))).toBe(true); - expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'playwright-report'))).toBe(false); - expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'tests', 'playwright-report'))).toBe(false); -}); - test('should not throw when attachment is missing', async ({ runInlineTest, page, showReport }, testInfo) => { const result = await runInlineTest({ @@ -911,3 +887,70 @@ test('should report clashing folders', async ({ runInlineTest }) => { expect(output).toContain('Configuration Error'); expect(output).toContain('html-report'); }); + +test.describe('report location', () => { + test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'nested/project/playwright.config.ts': ` + module.exports = { reporter: [['html', { outputFolder: '../my-report/' }]] }; + `, + 'nested/project/a.test.js': ` + const { test } = pwt; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: '', config: './nested/project/playwright.config.ts' }); + expect(result.exitCode).toBe(0); + expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'index.html')))).toBeTruthy(); + }); + + test('without config should create relative to package.json', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'foo/package.json': `{ "name": "foo" }`, + // unused config along "search path" + 'foo/bar/playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'foo/bar/baz/tests/a.spec.js': ` + const { test } = pwt; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }, { + cwd: 'foo/bar/baz/tests', + usesCustomOutputDir: true + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('playwright-report'))).toBe(false); + expect(fs.existsSync(testInfo.outputPath('foo', 'playwright-report'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'playwright-report'))).toBe(false); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'tests', 'playwright-report'))).toBe(false); + }); + + test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'foo/package.json': `{ "name": "foo" }`, + // unused config along "search path" + 'foo/bar/playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'foo/bar/baz/tests/a.spec.js': ` + const { test } = pwt; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'html' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_HTML_REPORT': '../my-report' }, { + cwd: 'foo/bar/baz/tests', + usesCustomOutputDir: true + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report'))).toBe(true); + }); +}); + + diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts index 799bdbafc5..6659d3c218 100644 --- a/tests/playwright-test/reporter-json.spec.ts +++ b/tests/playwright-test/reporter-json.spec.ts @@ -248,3 +248,43 @@ test('should have starting time in results', async ({ runInlineTest }, testInfo) const startTime = result.report.suites[0].specs[0].tests[0].results[0].startTime; expect(new Date(startTime).getTime()).toBeGreaterThan(new Date('1/1/2000').getTime()); }); + +test.describe('report location', () => { + test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'nested/project/playwright.config.ts': ` + module.exports = { reporter: [['json', { outputFile: '../my-report/a.json' }]] }; + `, + 'nested/project/a.test.js': ` + const { test } = pwt; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: '', config: './nested/project/playwright.config.ts' }); + expect(result.exitCode).toBe(0); + expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.json')))).toBeTruthy(); + }); + + test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'foo/package.json': `{ "name": "foo" }`, + // unused config along "search path" + 'foo/bar/playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'foo/bar/baz/tests/a.spec.js': ` + const { test } = pwt; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'json' }, { 'PW_TEST_HTML_REPORT_OPEN': 'never', 'PLAYWRIGHT_JSON_OUTPUT_NAME': '../my-report.json' }, { + cwd: 'foo/bar/baz/tests', + usesCustomOutputDir: true + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.json'))).toBe(true); + }); +}); diff --git a/tests/playwright-test/reporter-junit.spec.ts b/tests/playwright-test/reporter-junit.spec.ts index 077f0b31ef..754661fe67 100644 --- a/tests/playwright-test/reporter-junit.spec.ts +++ b/tests/playwright-test/reporter-junit.spec.ts @@ -17,6 +17,7 @@ import xml2js from 'xml2js'; import path from 'path'; import { test, expect } from './playwright-test-fixtures'; +import fs from 'fs'; test('should render expected', async ({ runInlineTest }) => { const result = await runInlineTest({ @@ -440,4 +441,45 @@ test('should not embed attachments to a custom testcase property, if not explict const testcase = xml['testsuites']['testsuite'][0]['testcase'][0]; expect(testcase['properties']).not.toBeTruthy(); expect(result.exitCode).toBe(0); -}); \ No newline at end of file +}); + + +test.describe('report location', () => { + test('with config should create report relative to config', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'nested/project/playwright.config.ts': ` + module.exports = { reporter: [['junit', { outputFile: '../my-report/a.xml' }]] }; + `, + 'nested/project/a.test.js': ` + const { test } = pwt; + test('one', async ({}) => { + expect(1).toBe(1); + }); + `, + }, { reporter: '', config: './nested/project/playwright.config.ts' }); + expect(result.exitCode).toBe(0); + expect(fs.existsSync(testInfo.outputPath(path.join('nested', 'my-report', 'a.xml')))).toBeTruthy(); + }); + + test('with env var should create relative to cwd', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + 'foo/package.json': `{ "name": "foo" }`, + // unused config along "search path" + 'foo/bar/playwright.config.js': ` + module.exports = { projects: [ {} ] }; + `, + 'foo/bar/baz/tests/a.spec.js': ` + const { test } = pwt; + const fs = require('fs'); + test('pass', ({}, testInfo) => { + }); + ` + }, { 'reporter': 'junit' }, { 'PLAYWRIGHT_JUNIT_OUTPUT_NAME': '../my-report.xml' }, { + cwd: 'foo/bar/baz/tests', + usesCustomOutputDir: true + }); + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + expect(fs.existsSync(testInfo.outputPath('foo', 'bar', 'baz', 'my-report.xml'))).toBe(true); + }); +});