diff --git a/packages/playwright-test/src/reporters/blob.ts b/packages/playwright-test/src/reporters/blob.ts index 1d55984f3d..8c90548d56 100644 --- a/packages/playwright-test/src/reporters/blob.ts +++ b/packages/playwright-test/src/reporters/blob.ts @@ -17,13 +17,15 @@ import type { EventEmitter } from 'events'; import fs from 'fs'; import path from 'path'; -import { ManualPromise, ZipFile } from 'playwright-core/lib/utils'; +import os from 'os'; +import { ManualPromise, ZipFile, calculateSha1, removeFolders } from 'playwright-core/lib/utils'; +import { mime } from 'playwright-core/lib/utilsBundle'; import { yazl } from 'playwright-core/lib/zipBundle'; import { Readable } from 'stream'; -import type { FullConfig, FullResult, Reporter } from '../../types/testReporter'; +import type { FullConfig, FullResult, Reporter, TestResult } from '../../types/testReporter'; import type { BuiltInReporter, FullConfigInternal } from '../common/config'; import type { Suite } from '../common/test'; -import { TeleReporterReceiver, type JsonEvent, type JsonProject, type JsonSuite } from '../isomorphic/teleReceiver'; +import { TeleReporterReceiver, type JsonEvent, type JsonProject, type JsonSuite, type JsonTestResultEnd } from '../isomorphic/teleReceiver'; import DotReporter from '../reporters/dot'; import EmptyReporter from '../reporters/empty'; import GitHubReporter from '../reporters/github'; @@ -44,7 +46,9 @@ type BlobReporterOptions = { export class BlobReporter extends TeleReporterEmitter { private _messages: any[] = []; private _options: BlobReporterOptions; - private _outputFile!: string; + + private readonly _zipFile = new yazl.ZipFile(); + private readonly _zipFinishPromise = new ManualPromise(); constructor(options: BlobReporterOptions) { super(message => this._messages.push(message)); @@ -53,14 +57,40 @@ export class BlobReporter extends TeleReporterEmitter { override onBegin(config: FullConfig<{}, {}>, suite: Suite): void { super.onBegin(config, suite); - this._computeOutputFileName(config); + this._initializeZipFile(config); } override async onEnd(result: FullResult): Promise { await super.onEnd(result); - fs.mkdirSync(path.dirname(this._outputFile), { recursive: true }); const lines = this._messages.map(m => JSON.stringify(m) + '\n'); - await zipReport(this._outputFile, lines); + const content = Readable.from(lines); + this._zipFile.addReadStream(content, 'report.jsonl'); + this._zipFile.end(); + await this._zipFinishPromise; + } + + override _serializeAttachments(attachments: TestResult['attachments']): TestResult['attachments'] { + return attachments.map(attachment => { + if (!attachment.path || !fs.statSync(attachment.path).isFile()) + return attachment; + const sha1 = calculateSha1(attachment.path); + const extension = mime.getExtension(attachment.contentType) || 'dat'; + const newPath = `resources/${sha1}.${extension}`; + this._zipFile.addFile(attachment.path, newPath); + return { + ...attachment, + path: newPath, + }; + }); + } + + private _initializeZipFile(config: FullConfig) { + (this._zipFile as any as EventEmitter).on('error', error => this._zipFinishPromise.reject(error)); + const zipFileName = this._computeOutputFileName(config); + fs.mkdirSync(path.dirname(zipFileName), { recursive: true }); + this._zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => { + this._zipFinishPromise.resolve(undefined); + }); } private _computeOutputFileName(config: FullConfig) { @@ -70,7 +100,7 @@ export class BlobReporter extends TeleReporterEmitter { const paddedNumber = `${config.shard.current}`.padStart(`${config.shard.total}`.length, '0'); shardSuffix = `-${paddedNumber}-of-${config.shard.total}`; } - this._outputFile = path.join(outputDir, `report${shardSuffix}.zip`); + return path.join(outputDir, `report${shardSuffix}.zip`); } private _resolveOutputDir(): string { @@ -83,58 +113,95 @@ export class BlobReporter extends TeleReporterEmitter { export async function createMergedReport(config: FullConfigInternal, dir: string, reporterName?: string) { const shardFiles = await sortedShardFiles(dir); - const events = await mergeEvents(dir, shardFiles); + const resourceDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-report-')); + await fs.promises.mkdir(resourceDir, { recursive: true }); + try { + const shardReports = await extractReports(dir, shardFiles, resourceDir); + const events = mergeEvents(shardReports); + patchAttachmentPaths(events, resourceDir); - const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { - dot: DotReporter, - line: LineReporter, - list: ListReporter, - github: GitHubReporter, - json: JSONReporter, - junit: JUnitReporter, - null: EmptyReporter, - html: HtmlReporter, - blob: BlobReporter, - }; - reporterName ??= 'list'; + const defaultReporters: {[key in BuiltInReporter]: new(arg: any) => Reporter} = { + dot: DotReporter, + line: LineReporter, + list: ListReporter, + github: GitHubReporter, + json: JSONReporter, + junit: JUnitReporter, + null: EmptyReporter, + html: HtmlReporter, + blob: BlobReporter, + }; + reporterName ??= 'list'; - const arg = config.config.reporter.find(([reporter, arg]) => reporter === reporterName)?.[1]; - const options = { - ...arg, - configDir: process.cwd(), - outputFolder: dir - }; + const arg = config.config.reporter.find(([reporter, arg]) => reporter === reporterName)?.[1]; + const options = { + ...arg, + configDir: process.cwd(), + }; - let reporter: Reporter | undefined; - if (reporterName in defaultReporters) { - reporter = new defaultReporters[reporterName as keyof typeof defaultReporters](options); - } else { - const reporterConstructor = await loadReporter(config, reporterName); - reporter = new reporterConstructor(options); + let reporter: Reporter | undefined; + if (reporterName in defaultReporters) { + reporter = new defaultReporters[reporterName as keyof typeof defaultReporters](options); + } else { + const reporterConstructor = await loadReporter(config, reporterName); + reporter = new reporterConstructor(options); + } + + const receiver = new TeleReporterReceiver(path.sep, reporter); + for (const event of events) + await receiver.dispatch(event); + } finally { + await removeFolders([resourceDir]); } - - const receiver = new TeleReporterReceiver(path.sep, reporter); - for (const event of events) - await receiver.dispatch(event); console.log(`Done.`); } -async function mergeEvents(dir: string, shardFiles: string[]) { - const events: JsonEvent[] = []; - const beginEvents: JsonEvent[] = []; - const endEvents: JsonEvent[] = []; +async function extractReports(dir: string, shardFiles: string[], resourceDir: string): Promise { + const reports = []; for (const file of shardFiles) { const zipFile = new ZipFile(path.join(dir, file)); const entryNames = await zipFile.entries(); - const reportEntryName = entryNames.find(e => e.endsWith('.jsonl')); - if (!reportEntryName) - throw new Error(`Zip file ${file} does not contain a .jsonl file`); - const reportJson = await zipFile.read(reportEntryName); - const parsedEvents = reportJson.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[]; + for (const entryName of entryNames) { + const content = await zipFile.read(entryName); + if (entryName.endsWith('report.jsonl')) { + reports.push(content.toString()); + } else { + const fileName = path.join(resourceDir, entryName); + await fs.promises.mkdir(path.dirname(fileName), { recursive: true }); + await fs.promises.writeFile(fileName, content); + } + } + } + return reports; +} + +function patchAttachmentPaths(events: JsonEvent[], resourceDir: string) { + for (const event of events) { + if (event.method !== 'onTestEnd') + continue; + for (const attachment of (event.params.result as JsonTestResultEnd).attachments) { + if (!attachment.path) + continue; + + attachment.path = path.join(resourceDir, attachment.path); + } + } +} + +function parseEvents(reportJsonl: string): JsonEvent[] { + return reportJsonl.toString().split('\n').filter(line => line.length).map(line => JSON.parse(line)) as JsonEvent[]; +} + +function mergeEvents(shardReports: string[]) { + const events: JsonEvent[] = []; + const beginEvents: JsonEvent[] = []; + const endEvents: JsonEvent[] = []; + for (const reportJsonl of shardReports) { + const parsedEvents = parseEvents(reportJsonl); for (const event of parsedEvents) { // TODO: show remaining events? if (event.method === 'onError') - throw new Error('Error in shard: ' + file); + throw new Error('Error in shard'); if (event.method === 'onBegin') beginEvents.push(event); else if (event.method === 'onEnd') @@ -142,7 +209,6 @@ async function mergeEvents(dir: string, shardFiles: string[]) { else events.push(event); } - } return [mergeBeginEvents(beginEvents), ...events, mergeEndEvents(endEvents)]; } @@ -212,17 +278,3 @@ async function sortedShardFiles(dir: string) { const files = await fs.promises.readdir(dir); return files.filter(file => file.endsWith('.zip')).sort(); } - -async function zipReport(zipFileName: string, lines: string[]) { - const zipFile = new yazl.ZipFile(); - const result = new ManualPromise(); - (zipFile as any as EventEmitter).on('error', error => result.reject(error)); - // TODO: feed events on the fly. - const content = Readable.from(lines); - zipFile.addReadStream(content, 'report.jsonl'); - zipFile.end(); - zipFile.outputStream.pipe(fs.createWriteStream(zipFileName)).on('close', () => { - result.resolve(undefined); - }); - await result; -} diff --git a/packages/playwright-test/src/reporters/teleEmitter.ts b/packages/playwright-test/src/reporters/teleEmitter.ts index d467abcfe2..33945e5dd2 100644 --- a/packages/playwright-test/src/reporters/teleEmitter.ts +++ b/packages/playwright-test/src/reporters/teleEmitter.ts @@ -193,10 +193,14 @@ export class TeleReporterEmitter implements Reporter { duration: result.duration, status: result.status, errors: result.errors, - attachments: result.attachments, + attachments: this._serializeAttachments(result.attachments), }; } + _serializeAttachments(attachments: TestResult['attachments']): TestResult['attachments'] { + return attachments; + } + private _serializeStepStart(step: TestStep): JsonTestStepStart { return { id: (step as any)[idSymbol], diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index e26a5768c1..2c1e94e062 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -28,17 +28,17 @@ const test = baseTest.extend<{ showReport: (reportFolder?: string) => Promise, mergeReports: (reportFolder: string, env?: NodeJS.ProcessEnv, options?: RunOptions) => Promise }>({ - showReport: async ({ page }, use, testInfo) => { + showReport: async ({ page }, use) => { let server: HttpServer | undefined; await use(async (reportFolder?: string) => { - reportFolder ??= testInfo.outputPath('playwright-report'); + reportFolder ??= test.info().outputPath('playwright-report'); server = startHtmlReportServer(reportFolder) as HttpServer; const location = await server.start(); await page.goto(location); }); await server?.stop(); }, - mergeReports: async ({ childProcess, page }, use, testInfo) => { + mergeReports: async ({ childProcess }, use) => { await use(async (reportFolder: string, env: NodeJS.ProcessEnv = {}, options: RunOptions = {}) => { const command = ['node', cliEntrypoint, 'merge-reports', reportFolder]; if (options.additionalArgs) @@ -47,7 +47,7 @@ const test = baseTest.extend<{ const testProcess = childProcess({ command, env: cleanEnv(env), - // cwd, + cwd: test.info().outputDir, }); const { exitCode } = await testProcess.exited; return { exitCode, output: testProcess.output.toString() }; @@ -110,7 +110,7 @@ test('should merge into html', async ({ runInlineTest, mergeReports, showReport, const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); - await showReport(reportDir); + await showReport(); await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('10'); await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('3'); @@ -169,7 +169,7 @@ test('be able to merge incomplete shards', async ({ runInlineTest, mergeReports, const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); expect(exitCode).toBe(0); - await showReport(reportDir); + await showReport(); await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('6'); await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('2'); @@ -257,3 +257,57 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports } `9 : - 10 b.test.js:9:12 › skipped 2` ]); }); + +test('preserve attachments', async ({ runInlineTest, mergeReports, showReport, page }) => { + test.slow(); + const reportDir = test.info().outputPath('blob-report'); + const files = { + 'playwright.config.ts': ` + module.exports = { + retries: 1, + reporter: [['blob', { outputDir: '${reportDir.replace(/\\/g, '/')}' }]] + }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + import fs from 'fs'; + + test('first', async ({}) => { + const attachmentPath = test.info().outputPath('foo.txt'); + fs.writeFileSync(attachmentPath, 'hello!'); + await test.info().attach('file-attachment', {path: attachmentPath}); + + console.log('console info'); + console.error('console error'); + }); + test('failing 1', async ({}) => { + await test.info().attach('text-attachment', { body: 'hi!' }); + expect(1).toBe(2); + }); + test.skip('skipped 1', async ({}) => {}); + `, + 'b.test.js': ` + import { test, expect } from '@playwright/test'; + test('math 2', async ({}) => { }); + test('failing 2', async ({}) => { + expect(1).toBe(2); + }); + test.skip('skipped 2', async ({}) => {}); + ` + }; + await runInlineTest(files, { shard: `1/2` }); + + const reportFiles = await fs.promises.readdir(reportDir); + reportFiles.sort(); + expect(reportFiles).toEqual(['report-1-of-2.zip']); + const { exitCode } = await mergeReports(reportDir, {}, { additionalArgs: ['--reporter', 'html'] }); + expect(exitCode).toBe(0); + + await showReport(); + await page.getByText('first').click(); + await expect(page.getByText('file-attachment')).toBeVisible(); + await page.goBack(); + + await page.getByText('failing 1').click(); + await expect(page.getByText('\'text-attachment\', { body: \'hi!\'')).toBeVisible(); +}); \ No newline at end of file