From 665143d62993d5d68b4e58cefc75db0aef6bb7c3 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Thu, 9 Sep 2021 17:35:31 -0700 Subject: [PATCH] chore(report): don't generate file per test (#8822) --- src/test/cli.ts | 4 +- src/test/html/htmlBuilder.ts | 61 ++++++++++++------- src/test/html/types.ts | 9 ++- src/test/reporters/raw.ts | 70 +++++++++++----------- src/web/htmlReport2/htmlReport.css | 3 +- src/web/htmlReport2/htmlReport.tsx | 63 ++++++++++++------- tests/config/browserTest.ts | 9 ++- tests/playwright-test/raw-reporter.spec.ts | 25 +++++--- 8 files changed, 152 insertions(+), 92 deletions(-) diff --git a/src/test/cli.ts b/src/test/cli.ts index e8efb8ec57..4d43f1df24 100644 --- a/src/test/cli.ts +++ b/src/test/cli.ts @@ -89,6 +89,8 @@ export function addGenerateHtmlCommand(program: commander.CommanderStatic) { command.option('-c, --config ', `Configuration file, or a test directory with optional "${tsConfig}"/"${jsConfig}"`); command.option('--output ', `Folder for output artifacts (default: "playwright-report")`, 'playwright-report'); command.action(async opts => { + const output = opts.output; + delete opts.output; const loader = await createLoader(opts); const outputFolders = new Set(loader.projects().map(p => p.config.outputDir)); const reportFiles = new Set(); @@ -98,7 +100,7 @@ export function addGenerateHtmlCommand(program: commander.CommanderStatic) { for (const file of files) reportFiles.add(path.join(reportFolder, file)); } - new HtmlBuilder([...reportFiles], opts.output); + new HtmlBuilder([...reportFiles], output); }).on('--help', () => { console.log(''); console.log('Examples:'); diff --git a/src/test/html/htmlBuilder.ts b/src/test/html/htmlBuilder.ts index 920c0a4393..73ba8c35ae 100644 --- a/src/test/html/htmlBuilder.ts +++ b/src/test/html/htmlBuilder.ts @@ -16,8 +16,9 @@ import fs from 'fs'; import path from 'path'; -import { ProjectTreeItem, SuiteTreeItem, TestTreeItem, TestCase, TestResult, TestStep } from './types'; +import { ProjectTreeItem, SuiteTreeItem, TestTreeItem, TestCase, TestResult, TestStep, TestFile } from './types'; import { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../reporters/raw'; +import { calculateSha1 } from '../../utils/utils'; export class HtmlBuilder { private _reportFolder: string; @@ -30,31 +31,44 @@ export class HtmlBuilder { const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport2'); for (const file of fs.readdirSync(appFolder)) fs.copyFileSync(path.join(appFolder, file), path.join(this._reportFolder, file)); - const projects: ProjectTreeItem[] = rawReports.map(rawReport => { - const json = JSON.parse(fs.readFileSync(rawReport, 'utf-8')) as JsonReport; - const suits = json.suites.map(s => this._createSuiteTreeItem(s)); - return { - name: json.project.name, - suits, - failedTests: suits.reduce((a, s) => a + s.failedTests, 0) - }; - }); - fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2)); - for (const [testId, test] of this._tests) { - const testCase: TestCase = { - testId: test.testId, - title: test.title, - location: test.location, - results: test.results.map(r => this._createTestResult(r)) - }; - fs.writeFileSync(path.join(dataFolder, testId + '.json'), JSON.stringify(testCase, undefined, 2)); + const projects: ProjectTreeItem[] = []; + for (const projectFile of rawReports) { + const projectJson = JSON.parse(fs.readFileSync(projectFile, 'utf-8')) as JsonReport; + const suites: SuiteTreeItem[] = []; + for (const file of projectJson.suites) { + const fileId = calculateSha1(projectFile + ':' + file.location!.file); + const tests: JsonTestCase[] = []; + suites.push(this._createSuiteTreeItem(file, fileId, tests)); + const testFile: TestFile = { + fileId, + path: file.location!.file, + tests: tests.map(t => this._createTestCase(t)) + }; + fs.writeFileSync(path.join(dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2)); + } + projects.push({ + name: projectJson.project.name, + suites, + failedTests: suites.reduce((a, s) => a + s.failedTests, 0) + }); } + fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2)); } - private _createSuiteTreeItem(suite: JsonSuite): SuiteTreeItem { - const suites = suite.suites.map(s => this._createSuiteTreeItem(s)); - const tests = suite.tests.map(t => this._createTestTreeItem(t)); + private _createTestCase(test: JsonTestCase): TestCase { + return { + testId: test.testId, + title: test.title, + location: test.location, + results: test.results.map(r => this._createTestResult(r)) + }; + } + + private _createSuiteTreeItem(suite: JsonSuite, fileId: string, testCollector: JsonTestCase[]): SuiteTreeItem { + const suites = suite.suites.map(s => this._createSuiteTreeItem(s, fileId, testCollector)); + const tests = suite.tests.map(t => this._createTestTreeItem(t, fileId)); + testCollector.push(...suite.tests); return { title: suite.title, location: suite.location, @@ -65,11 +79,12 @@ export class HtmlBuilder { }; } - private _createTestTreeItem(test: JsonTestCase): TestTreeItem { + private _createTestTreeItem(test: JsonTestCase, fileId: string): TestTreeItem { const duration = test.results.reduce((a, r) => a + r.duration, 0); this._tests.set(test.testId, test); return { testId: test.testId, + fileId: fileId, location: test.location, title: test.title, duration, diff --git a/src/test/html/types.ts b/src/test/html/types.ts index e130490190..410e8403ea 100644 --- a/src/test/html/types.ts +++ b/src/test/html/types.ts @@ -22,7 +22,7 @@ export type Location = { export type ProjectTreeItem = { name: string; - suits: SuiteTreeItem[]; + suites: SuiteTreeItem[]; failedTests: number; }; @@ -37,12 +37,19 @@ export type SuiteTreeItem = { export type TestTreeItem = { testId: string, + fileId: string, title: string; location: Location; duration: number; outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; }; +export type TestFile = { + fileId: string; + path: string; + tests: TestCase[]; +}; + export type TestCase = { testId: string, title: string; diff --git a/src/test/reporters/raw.ts b/src/test/reporters/raw.ts index 139b0cb100..a5455693cf 100644 --- a/src/test/reporters/raw.ts +++ b/src/test/reporters/raw.ts @@ -20,7 +20,7 @@ import { FullProject } from '../../../types/test'; import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter'; import { assert, calculateSha1 } from '../../utils/utils'; import { sanitizeForFilePath } from '../util'; -import { serializePatterns, toPosixPath } from './json'; +import { serializePatterns } from './json'; export type JsonStats = { expected: number, unexpected: number, flaky: number, skipped: number }; export type JsonLocation = Location; @@ -75,7 +75,8 @@ export type TestAttachment = { export type JsonAttachment = { name: string; - path: string; + body?: string; + path?: string; contentType: string; }; @@ -133,30 +134,30 @@ class RawReporter { project: { metadata: project.metadata, name: project.name, - outputDir: toPosixPath(project.outputDir), + outputDir: project.outputDir, repeatEach: project.repeatEach, retries: project.retries, - testDir: toPosixPath(project.testDir), + testDir: project.testDir, testIgnore: serializePatterns(project.testIgnore), testMatch: serializePatterns(project.testMatch), timeout: project.timeout, }, - suites: suite.suites.map(s => this._serializeSuite(s, reportFolder)) + suites: suite.suites.map(s => this._serializeSuite(s)) }; fs.writeFileSync(reportFile, JSON.stringify(report, undefined, 2)); } } - private _serializeSuite(suite: Suite, reportFolder: string): JsonSuite { + private _serializeSuite(suite: Suite): JsonSuite { return { title: suite.title, location: suite.location, - suites: suite.suites.map(s => this._serializeSuite(s, reportFolder)), - tests: suite.tests.map(t => this._serializeTest(t, reportFolder)), + suites: suite.suites.map(s => this._serializeSuite(s)), + tests: suite.tests.map(t => this._serializeTest(t)), }; } - private _serializeTest(test: TestCase, reportFolder: string): JsonTestCase { + private _serializeTest(test: TestCase): JsonTestCase { const testId = calculateSha1(test.titlePath().join('|')); return { testId, @@ -168,11 +169,11 @@ class RawReporter { retries: test.retries, ok: test.ok(), outcome: test.outcome(), - results: test.results.map(r => this._serializeResult(testId, test, r, reportFolder)), + results: test.results.map(r => this._serializeResult(testId, test, r)), }; } - private _serializeResult(testId: string, test: TestCase, result: TestResult, reportFolder: string): JsonTestResult { + private _serializeResult(testId: string, test: TestCase, result: TestResult): JsonTestResult { return { retry: result.retry, workerIndex: result.workerIndex, @@ -180,7 +181,7 @@ class RawReporter { duration: result.duration, status: result.status, error: result.error, - attachments: this._createAttachments(reportFolder, testId, result), + attachments: this._createAttachments(result), steps: this._serializeSteps(test, result.steps) }; } @@ -199,44 +200,43 @@ class RawReporter { }); } - private _createAttachments(reportFolder: string, testId: string, result: TestResult): JsonAttachment[] { + private _createAttachments(result: TestResult): JsonAttachment[] { const attachments: JsonAttachment[] = []; - for (const attachment of result.attachments.filter(a => !a.path)) { - const sha1 = calculateSha1(attachment.body!); - const file = path.join(reportFolder, sha1); - try { - fs.writeFileSync(path.join(reportFolder, sha1), attachment.body); + for (const attachment of result.attachments) { + if (attachment.body) { attachments.push({ name: attachment.name, contentType: attachment.contentType, - path: toPosixPath(file) + body: attachment.body.toString('base64') + }); + } else if (attachment.path) { + attachments.push({ + name: attachment.name, + contentType: attachment.contentType, + path: attachment.path }); - } catch (e) { } } - for (const attachment of result.attachments.filter(a => a.path)) - attachments.push(attachment as JsonAttachment); - if (result.stdout.length) - attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stdout')); - if (result.stderr.length) - attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stderr')); + for (const chunk of result.stdout) + attachments.push(this._stdioAttachment(chunk, 'stdout')); + for (const chunk of result.stderr) + attachments.push(this._stdioAttachment(chunk, 'stderr')); return attachments; } - private _stdioAttachment(reportFolder: string, testId: string, result: TestResult, type: 'stdout' | 'stderr'): JsonAttachment { - const file = `${testId}.${result.retry}.${type}`; - const fileName = path.join(reportFolder, file); - for (const chunk of type === 'stdout' ? result.stdout : result.stderr) { - if (typeof chunk === 'string') - fs.appendFileSync(fileName, chunk + '\n'); - else - fs.appendFileSync(fileName, chunk); + private _stdioAttachment(chunk: Buffer | string, type: 'stdout' | 'stderr'): JsonAttachment { + if (typeof chunk === 'string') { + return { + name: type, + contentType: 'text/plain', + body: chunk + }; } return { name: type, contentType: 'application/octet-stream', - path: toPosixPath(fileName) + body: chunk.toString('base64') }; } } diff --git a/src/web/htmlReport2/htmlReport.css b/src/web/htmlReport2/htmlReport.css index 35e1ce4caa..46c1c4a653 100644 --- a/src/web/htmlReport2/htmlReport.css +++ b/src/web/htmlReport2/htmlReport.css @@ -23,7 +23,7 @@ } .tree-item-title { - padding: 8px 0; + padding: 8px 8px 8px 0; cursor: pointer; } @@ -80,7 +80,6 @@ flex: auto; display: flex; flex-direction: column; - padding-right: 8px; } .test-overview-title { diff --git a/src/web/htmlReport2/htmlReport.tsx b/src/web/htmlReport2/htmlReport.tsx index 7e5ec79dae..8568d24f98 100644 --- a/src/web/htmlReport2/htmlReport.tsx +++ b/src/web/htmlReport2/htmlReport.tsx @@ -20,14 +20,19 @@ import { SplitView } from '../components/splitView'; import { TreeItem } from '../components/treeItem'; import { TabbedPane } from '../traceViewer/ui/tabbedPane'; import { msToString } from '../uiUtils'; -import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location } from '../../test/html/types'; +import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile } from '../../test/html/types'; type Filter = 'Failing' | 'All'; +type TestId = { + fileId: string; + testId: string; +}; + export const Report: React.FC = () => { const [report, setReport] = React.useState([]); const [fetchError, setFetchError] = React.useState(); - const [testId, setTestId] = React.useState(); + const [testId, setTestId] = React.useState(); React.useEffect(() => { (async () => { @@ -63,22 +68,22 @@ export const Report: React.FC = () => { const ProjectTreeItemView: React.FC<{ project: ProjectTreeItem; - testId?: string, - setTestId: (id: string) => void; + testId?: TestId, + setTestId: (id: TestId) => void; failingOnly?: boolean; }> = ({ project, testId, setTestId, failingOnly }) => { return {statusIconForFailedTests(project.failedTests)}
{project.name || 'Project'}
} loadChildren={() => { - return project.suits.map((s, i) => ) || []; + return project.suites.map((s, i) => ) || []; }} depth={0} expandByDefault={true}>; }; const SuiteTreeItemView: React.FC<{ suite: SuiteTreeItem, - testId?: string, - setTestId: (id: string) => void; + testId?: TestId, + setTestId: (id: TestId) => void; depth: number, showFileName: boolean, }> = ({ suite, testId, setTestId, showFileName, depth }) => { @@ -98,8 +103,8 @@ const SuiteTreeItemView: React.FC<{ const TestTreeItemView: React.FC<{ test: TestTreeItem, showFileName: boolean, - testId?: string, - setTestId: (id: string) => void; + testId?: TestId, + setTestId: (id: TestId) => void; depth: number, }> = ({ test, testId, setTestId, showFileName, depth }) => { const fileName = test.location.file; @@ -109,27 +114,36 @@ const TestTreeItemView: React.FC<{ {showFileName &&
{name}:{test.location.line}
} {!showFileName &&
{msToString(test.duration)}
} - } selected={test.testId === testId} depth={depth} onClick={() => setTestId(test.testId)}>; + } selected={test.testId === testId?.testId} depth={depth} onClick={() => setTestId({ testId: test.testId, fileId: test.fileId })}>; }; const TestCaseView: React.FC<{ - testId: string | undefined, + testId: TestId | undefined, }> = ({ testId }) => { - const [test, setTest] = React.useState(); + const [file, setFile] = React.useState(); React.useEffect(() => { (async () => { - if (!testId) + if (!testId || file?.fileId === testId.fileId) return; try { - const result = await fetch(`data/${testId}.json`, { cache: 'no-cache' }); - const json = (await result.json()) as TestCase; - setTest(json); + const result = await fetch(`data/${testId.fileId}.json`, { cache: 'no-cache' }); + setFile((await result.json()) as TestFile); } catch (e) { } })(); }); + let test: TestCase | undefined; + if (file && testId) { + for (const t of file.tests) { + if (t.testId === testId.testId) { + test = t; + break; + } + } + } + const [selectedResultIndex, setSelectedResultIndex] = React.useState(0); return
@@ -138,10 +152,10 @@ const TestCaseView: React.FC<{ { test &&
{test?.title}
} { test &&
{renderLocation(test.location, true)}
} { test && ({ + test.results.map((result, index) => ({ id: String(index), title:
{statusIcon(result.status)} {retryLabel(index)}
, - render: () => + render: () => })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />}
; @@ -165,11 +179,20 @@ const StepTreeItem: React.FC<{ {step.title}
{msToString(step.duration)}
- } loadChildren={step.steps.length ? () => { - return step.steps.map((s, i) => ); + } loadChildren={step.steps.length + (step.log || []).length ? () => { + const stepChildren = step.steps.map((s, i) => ); + const logChildren = (step.log || []).map((l, i) => ); + return [...stepChildren, ...logChildren]; } : undefined} depth={depth}>; }; +const LogTreeItem: React.FC<{ + log: string; + depth: number, +}> = ({ log, depth }) => { + return { log }} depth={depth}>; +}; + function statusIconForFailedTests(failedTests: number) { return failedTests ? statusIcon('failed') : statusIcon('passed'); } diff --git a/tests/config/browserTest.ts b/tests/config/browserTest.ts index c0a7513da5..8d9e195588 100644 --- a/tests/config/browserTest.ts +++ b/tests/config/browserTest.ts @@ -140,8 +140,13 @@ export const playwrightFixtures: Fixtures { - const step = (testInfo as any)._addStep('pw:api', stackTrace.apiName); - return (log, error) => step.complete(error); + const testInfoImpl = testInfo as any; + const existingStep = testInfoImpl._currentSteps().find(step => step.category === 'pw:api' || step.category === 'expect'); + const newStep = existingStep ? undefined : testInfoImpl._addStep('pw:api', stackTrace.apiName, { stack: stackTrace.frames, log: [] }); + return (log: string[], error?: Error) => { + (existingStep || newStep)?.data.log?.push(...log); + newStep?.complete(error); + }; }, }; contexts.push(context); diff --git a/tests/playwright-test/raw-reporter.spec.ts b/tests/playwright-test/raw-reporter.spec.ts index 8119d6cd99..84b6b71a2d 100644 --- a/tests/playwright-test/raw-reporter.spec.ts +++ b/tests/playwright-test/raw-reporter.spec.ts @@ -56,18 +56,28 @@ test('should save stdio', async ({ runInlineTest }, testInfo) => { const { test } = pwt; test('passes', async ({ page }, testInfo) => { console.log('STDOUT'); + process.stdout.write(Buffer.from([1, 2, 3])); console.error('STDERR'); + process.stderr.write(Buffer.from([4, 5, 6])); }); `, }, { usesCustomOutputDir: true }); const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8')); const result = json.suites[0].tests[0].results[0]; - expect(result.attachments[0].name).toBe('stdout'); - expect(result.attachments[1].name).toBe('stderr'); - const path1 = result.attachments[0].path; - expect(fs.readFileSync(path1, 'utf-8')).toContain('STDOUT'); - const path2 = result.attachments[1].path; - expect(fs.readFileSync(path2, 'utf-8')).toContain('STDERR'); + expect(result.attachments).toEqual([ + { name: 'stdout', contentType: 'text/plain', body: 'STDOUT\n' }, + { + name: 'stdout', + contentType: 'application/octet-stream', + body: 'AQID' + }, + { name: 'stderr', contentType: 'text/plain', body: 'STDERR\n' }, + { + name: 'stderr', + contentType: 'application/octet-stream', + body: 'BAUG' + } + ]); }); test('should save attachments', async ({ runInlineTest }, testInfo) => { @@ -91,9 +101,8 @@ test('should save attachments', async ({ runInlineTest }, testInfo) => { const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8')); const result = json.suites[0].tests[0].results[0]; expect(result.attachments[0].name).toBe('binary'); + expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from([1,2,3])); expect(result.attachments[1].name).toBe('text'); - const path1 = result.attachments[0].path; - expect(fs.readFileSync(path1)).toEqual(Buffer.from([1,2,3])); const path2 = result.attachments[1].path; expect(path2).toBe('dummy-path'); });