chore(report): don't generate file per test (#8822)

2025-06-26 21:40:17 +00:00 · 2021-09-09 17:35:31 -07:00 · 2021-09-09 17:35:31 -07:00 · 665143d629
commit 665143d629
parent e85fba1c7d
8 changed files with 152 additions and 92 deletions
--- a/src/test/cli.ts
+++ b/src/test/cli.ts
@ -89,6 +89,8 @@ export function addGenerateHtmlCommand(program: commander.CommanderStatic) {
  command.option('-c, --config <file>', `Configuration file, or a test directory with optional "${tsConfig}"/"${jsConfig}"`);
  command.option('--output <dir>', `Folder for output artifacts (default: "playwright-report")`, 'playwright-report');
  command.action(async opts => {
+    const output = opts.output;
+    delete opts.output;
    const loader = await createLoader(opts);
    const outputFolders = new Set(loader.projects().map(p => p.config.outputDir));
    const reportFiles = new Set<string>();
@ -98,7 +100,7 @@ export function addGenerateHtmlCommand(program: commander.CommanderStatic) {
      for (const file of files)
        reportFiles.add(path.join(reportFolder, file));
    }
-    new HtmlBuilder([...reportFiles], opts.output);
+    new HtmlBuilder([...reportFiles], output);
  }).on('--help', () => {
    console.log('');
    console.log('Examples:');
--- a/src/test/html/htmlBuilder.ts
+++ b/src/test/html/htmlBuilder.ts
@ -16,8 +16,9 @@

 import fs from 'fs';
 import path from 'path';
-import { ProjectTreeItem, SuiteTreeItem, TestTreeItem, TestCase, TestResult, TestStep } from './types';
+import { ProjectTreeItem, SuiteTreeItem, TestTreeItem, TestCase, TestResult, TestStep, TestFile } from './types';
 import { JsonReport, JsonSuite, JsonTestCase, JsonTestResult, JsonTestStep } from '../reporters/raw';
+import { calculateSha1 } from '../../utils/utils';

 export class HtmlBuilder {
  private _reportFolder: string;
@ -30,31 +31,44 @@ export class HtmlBuilder {
    const appFolder = path.join(__dirname, '..', '..', 'web', 'htmlReport2');
    for (const file of fs.readdirSync(appFolder))
      fs.copyFileSync(path.join(appFolder, file), path.join(this._reportFolder, file));
-    const projects: ProjectTreeItem[] = rawReports.map(rawReport => {
-      const json = JSON.parse(fs.readFileSync(rawReport, 'utf-8')) as JsonReport;
-      const suits = json.suites.map(s => this._createSuiteTreeItem(s));
-      return {
-        name: json.project.name,
-        suits,
-        failedTests: suits.reduce((a, s) => a + s.failedTests, 0)
-      };
-    });
-    fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2));

-    for (const [testId, test] of this._tests) {
-      const testCase: TestCase = {
-        testId: test.testId,
-        title: test.title,
-        location: test.location,
-        results: test.results.map(r => this._createTestResult(r))
-      };
-      fs.writeFileSync(path.join(dataFolder, testId + '.json'), JSON.stringify(testCase, undefined, 2));
+    const projects: ProjectTreeItem[] = [];
+    for (const projectFile of rawReports) {
+      const projectJson = JSON.parse(fs.readFileSync(projectFile, 'utf-8')) as JsonReport;
+      const suites: SuiteTreeItem[] = [];
+      for (const file of projectJson.suites) {
+        const fileId = calculateSha1(projectFile + ':' + file.location!.file);
+        const tests: JsonTestCase[] = [];
+        suites.push(this._createSuiteTreeItem(file, fileId, tests));
+        const testFile: TestFile = {
+          fileId,
+          path: file.location!.file,
+          tests: tests.map(t => this._createTestCase(t))
+        };
+        fs.writeFileSync(path.join(dataFolder, fileId + '.json'), JSON.stringify(testFile, undefined, 2));
+      }
+      projects.push({
+        name: projectJson.project.name,
+        suites,
+        failedTests: suites.reduce((a, s) => a + s.failedTests, 0)
+      });
    }
+    fs.writeFileSync(path.join(dataFolder, 'projects.json'), JSON.stringify(projects, undefined, 2));
  }

-  private _createSuiteTreeItem(suite: JsonSuite): SuiteTreeItem {
-    const suites = suite.suites.map(s => this._createSuiteTreeItem(s));
-    const tests = suite.tests.map(t => this._createTestTreeItem(t));
+  private _createTestCase(test: JsonTestCase): TestCase {
+    return {
+      testId: test.testId,
+      title: test.title,
+      location: test.location,
+      results: test.results.map(r => this._createTestResult(r))
+    };
+  }
+
+  private _createSuiteTreeItem(suite: JsonSuite, fileId: string, testCollector: JsonTestCase[]): SuiteTreeItem {
+    const suites = suite.suites.map(s => this._createSuiteTreeItem(s, fileId, testCollector));
+    const tests = suite.tests.map(t => this._createTestTreeItem(t, fileId));
+    testCollector.push(...suite.tests);
    return {
      title: suite.title,
      location: suite.location,
@ -65,11 +79,12 @@ export class HtmlBuilder {
    };
  }

-  private _createTestTreeItem(test: JsonTestCase): TestTreeItem {
+  private _createTestTreeItem(test: JsonTestCase, fileId: string): TestTreeItem {
    const duration = test.results.reduce((a, r) => a + r.duration, 0);
    this._tests.set(test.testId, test);
    return {
      testId: test.testId,
+      fileId: fileId,
      location: test.location,
      title: test.title,
      duration,
--- a/src/test/html/types.ts
+++ b/src/test/html/types.ts
@ -22,7 +22,7 @@ export type Location = {

 export type ProjectTreeItem = {
  name: string;
-  suits: SuiteTreeItem[];
+  suites: SuiteTreeItem[];
  failedTests: number;
 };

@ -37,12 +37,19 @@ export type SuiteTreeItem = {

 export type TestTreeItem = {
  testId: string,
+  fileId: string,
  title: string;
  location: Location;
  duration: number;
  outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky';
 };

+export type TestFile = {
+  fileId: string;
+  path: string;
+  tests: TestCase[];
+};
+
 export type TestCase = {
  testId: string,
  title: string;
--- a/src/test/reporters/raw.ts
+++ b/src/test/reporters/raw.ts
@ -20,7 +20,7 @@ import { FullProject } from '../../../types/test';
 import { FullConfig, Location, Suite, TestCase, TestError, TestResult, TestStatus, TestStep } from '../../../types/testReporter';
 import { assert, calculateSha1 } from '../../utils/utils';
 import { sanitizeForFilePath } from '../util';
-import { serializePatterns, toPosixPath } from './json';
+import { serializePatterns } from './json';

 export type JsonStats = { expected: number, unexpected: number, flaky: number, skipped: number };
 export type JsonLocation = Location;
@ -75,7 +75,8 @@ export type TestAttachment = {

 export type JsonAttachment = {
  name: string;
-  path: string;
+  body?: string;
+  path?: string;
  contentType: string;
 };

@ -133,30 +134,30 @@ class RawReporter {
        project: {
          metadata: project.metadata,
          name: project.name,
-          outputDir: toPosixPath(project.outputDir),
+          outputDir: project.outputDir,
          repeatEach: project.repeatEach,
          retries: project.retries,
-          testDir: toPosixPath(project.testDir),
+          testDir: project.testDir,
          testIgnore: serializePatterns(project.testIgnore),
          testMatch: serializePatterns(project.testMatch),
          timeout: project.timeout,
        },
-        suites: suite.suites.map(s => this._serializeSuite(s, reportFolder))
+        suites: suite.suites.map(s => this._serializeSuite(s))
      };
      fs.writeFileSync(reportFile, JSON.stringify(report, undefined, 2));
    }
  }

-  private _serializeSuite(suite: Suite, reportFolder: string): JsonSuite {
+  private _serializeSuite(suite: Suite): JsonSuite {
    return {
      title: suite.title,
      location: suite.location,
-      suites: suite.suites.map(s => this._serializeSuite(s, reportFolder)),
-      tests: suite.tests.map(t => this._serializeTest(t, reportFolder)),
+      suites: suite.suites.map(s => this._serializeSuite(s)),
+      tests: suite.tests.map(t => this._serializeTest(t)),
    };
  }

-  private _serializeTest(test: TestCase, reportFolder: string): JsonTestCase {
+  private _serializeTest(test: TestCase): JsonTestCase {
    const testId = calculateSha1(test.titlePath().join('|'));
    return {
      testId,
@ -168,11 +169,11 @@ class RawReporter {
      retries: test.retries,
      ok: test.ok(),
      outcome: test.outcome(),
-      results: test.results.map(r => this._serializeResult(testId, test, r, reportFolder)),
+      results: test.results.map(r => this._serializeResult(testId, test, r)),
    };
  }

-  private _serializeResult(testId: string, test: TestCase, result: TestResult, reportFolder: string): JsonTestResult {
+  private _serializeResult(testId: string, test: TestCase, result: TestResult): JsonTestResult {
    return {
      retry: result.retry,
      workerIndex: result.workerIndex,
@ -180,7 +181,7 @@ class RawReporter {
      duration: result.duration,
      status: result.status,
      error: result.error,
-      attachments: this._createAttachments(reportFolder, testId, result),
+      attachments: this._createAttachments(result),
      steps: this._serializeSteps(test, result.steps)
    };
  }
@ -199,44 +200,43 @@ class RawReporter {
    });
  }

-  private _createAttachments(reportFolder: string, testId: string, result: TestResult): JsonAttachment[] {
+  private _createAttachments(result: TestResult): JsonAttachment[] {
    const attachments: JsonAttachment[] = [];
-    for (const attachment of result.attachments.filter(a => !a.path)) {
-      const sha1 = calculateSha1(attachment.body!);
-      const file = path.join(reportFolder, sha1);
-      try {
-        fs.writeFileSync(path.join(reportFolder, sha1), attachment.body);
+    for (const attachment of result.attachments) {
+      if (attachment.body) {
        attachments.push({
          name: attachment.name,
          contentType: attachment.contentType,
-          path: toPosixPath(file)
+          body: attachment.body.toString('base64')
+        });
+      } else if (attachment.path) {
+        attachments.push({
+          name: attachment.name,
+          contentType: attachment.contentType,
+          path: attachment.path
        });
-      } catch (e) {
      }
    }
-    for (const attachment of result.attachments.filter(a => a.path))
-      attachments.push(attachment as JsonAttachment);

-    if (result.stdout.length)
-      attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stdout'));
-    if (result.stderr.length)
-      attachments.push(this._stdioAttachment(reportFolder, testId, result, 'stderr'));
+    for (const chunk of result.stdout)
+      attachments.push(this._stdioAttachment(chunk, 'stdout'));
+    for (const chunk of result.stderr)
+      attachments.push(this._stdioAttachment(chunk, 'stderr'));
    return attachments;
  }

-  private _stdioAttachment(reportFolder: string, testId: string, result: TestResult, type: 'stdout' | 'stderr'): JsonAttachment {
-    const file = `${testId}.${result.retry}.${type}`;
-    const fileName = path.join(reportFolder, file);
-    for (const chunk of type === 'stdout' ? result.stdout : result.stderr) {
-      if (typeof chunk === 'string')
-        fs.appendFileSync(fileName, chunk + '\n');
-      else
-        fs.appendFileSync(fileName, chunk);
+  private _stdioAttachment(chunk: Buffer | string, type: 'stdout' | 'stderr'): JsonAttachment {
+    if (typeof chunk === 'string') {
+      return {
+        name: type,
+        contentType: 'text/plain',
+        body: chunk
+      };
    }
    return {
      name: type,
      contentType: 'application/octet-stream',
-      path: toPosixPath(fileName)
+      body: chunk.toString('base64')
    };
  }
 }
--- a/src/web/htmlReport2/htmlReport.css
+++ b/src/web/htmlReport2/htmlReport.css
@ -23,7 +23,7 @@
 }

 .tree-item-title {
-  padding: 8px 0;
+  padding: 8px 8px 8px 0;
  cursor: pointer;
 }

@ -80,7 +80,6 @@
  flex: auto;
  display: flex;
  flex-direction: column;
-  padding-right: 8px;
 }

 .test-overview-title {
--- a/src/web/htmlReport2/htmlReport.tsx
+++ b/src/web/htmlReport2/htmlReport.tsx
@ -20,14 +20,19 @@ import { SplitView } from '../components/splitView';
 import { TreeItem } from '../components/treeItem';
 import { TabbedPane } from '../traceViewer/ui/tabbedPane';
 import { msToString } from '../uiUtils';
-import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location } from '../../test/html/types';
+import type { ProjectTreeItem, SuiteTreeItem, TestCase, TestResult, TestStep, TestTreeItem, Location, TestFile } from '../../test/html/types';

 type Filter = 'Failing' | 'All';

+type TestId = {
+  fileId: string;
+  testId: string;
+};
+
 export const Report: React.FC = () => {
  const [report, setReport] = React.useState<ProjectTreeItem[]>([]);
  const [fetchError, setFetchError] = React.useState<string | undefined>();
-  const [testId, setTestId] = React.useState<string | undefined>();
+  const [testId, setTestId] = React.useState<TestId | undefined>();

  React.useEffect(() => {
    (async () => {
@ -63,22 +68,22 @@ export const Report: React.FC = () => {

 const ProjectTreeItemView: React.FC<{
  project: ProjectTreeItem;
-  testId?: string,
-  setTestId: (id: string) => void;
+  testId?: TestId,
+  setTestId: (id: TestId) => void;
  failingOnly?: boolean;
 }> = ({ project, testId, setTestId, failingOnly }) => {
  return <TreeItem title={<div className='hbox'>
    {statusIconForFailedTests(project.failedTests)}<div className='tree-text'>{project.name || 'Project'}</div>
  </div>
  } loadChildren={() => {
-    return project.suits.map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={1} showFileName={true}></SuiteTreeItemView>) || [];
+    return project.suites.map((s, i) => <SuiteTreeItemView key={i} suite={s} setTestId={setTestId} testId={testId} depth={1} showFileName={true}></SuiteTreeItemView>) || [];
  }} depth={0} expandByDefault={true}></TreeItem>;
 };

 const SuiteTreeItemView: React.FC<{
  suite: SuiteTreeItem,
-  testId?: string,
-  setTestId: (id: string) => void;
+  testId?: TestId,
+  setTestId: (id: TestId) => void;
  depth: number,
  showFileName: boolean,
 }> = ({ suite, testId, setTestId, showFileName, depth }) => {
@ -98,8 +103,8 @@ const SuiteTreeItemView: React.FC<{
 const TestTreeItemView: React.FC<{
  test: TestTreeItem,
  showFileName: boolean,
-  testId?: string,
-  setTestId: (id: string) => void;
+  testId?: TestId,
+  setTestId: (id: TestId) => void;
  depth: number,
 }> = ({ test, testId, setTestId, showFileName, depth }) => {
  const fileName = test.location.file;
@ -109,27 +114,36 @@ const TestTreeItemView: React.FC<{
    {showFileName && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{name}:{test.location.line}</div>}
    {!showFileName && <div style={{ flex: 'none', padding: '0 4px', color: '#666' }}>{msToString(test.duration)}</div>}
  </div>
-  } selected={test.testId === testId} depth={depth} onClick={() => setTestId(test.testId)}></TreeItem>;
+  } selected={test.testId === testId?.testId} depth={depth} onClick={() => setTestId({ testId: test.testId, fileId: test.fileId })}></TreeItem>;
 };

 const TestCaseView: React.FC<{
-  testId: string | undefined,
+  testId: TestId | undefined,
 }> = ({ testId }) => {
-  const [test, setTest] = React.useState<TestCase | undefined>();
+  const [file, setFile] = React.useState<TestFile | undefined>();

  React.useEffect(() => {
    (async () => {
-      if (!testId)
+      if (!testId || file?.fileId === testId.fileId)
        return;
      try {
-        const result = await fetch(`data/${testId}.json`, { cache: 'no-cache' });
-        const json = (await result.json()) as TestCase;
-        setTest(json);
+        const result = await fetch(`data/${testId.fileId}.json`, { cache: 'no-cache' });
+        setFile((await result.json()) as TestFile);
      } catch (e) {
      }
    })();
  });

+  let test: TestCase | undefined;
+  if (file && testId) {
+    for (const t of file.tests) {
+      if (t.testId === testId.testId) {
+        test = t;
+        break;
+      }
+    }
+  }
+
  const [selectedResultIndex, setSelectedResultIndex] = React.useState(0);
  return <SplitView sidebarSize={500} orientation='horizontal' sidebarIsFirst={true}>
    <div className='test-details-column vbox'>
@ -138,10 +152,10 @@ const TestCaseView: React.FC<{
      { test && <div className='test-case-title'>{test?.title}</div> }
      { test && <div className='test-case-location'>{renderLocation(test.location, true)}</div> }
      { test && <TabbedPane tabs={
-        test?.results.map((result, index) => ({
+        test.results.map((result, index) => ({
          id: String(index),
          title: <div style={{ display: 'flex', alignItems: 'center' }}>{statusIcon(result.status)} {retryLabel(index)}</div>,
-          render: () => <TestResultView test={test} result={result}></TestResultView>
+          render: () => <TestResultView test={test!} result={result}></TestResultView>
        })) || []} selectedTab={String(selectedResultIndex)} setSelectedTab={id => setSelectedResultIndex(+id)} />}
    </div>
  </SplitView>;
@ -165,11 +179,20 @@ const StepTreeItem: React.FC<{
    <span style={{ whiteSpace: 'pre' }}>{step.title}</span>
    <div style={{ flex: 'auto' }}></div>
    <div>{msToString(step.duration)}</div>
-  </div>} loadChildren={step.steps.length ? () => {
-    return step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1}></StepTreeItem>);
+  </div>} loadChildren={step.steps.length + (step.log || []).length ? () => {
+    const stepChildren = step.steps.map((s, i) => <StepTreeItem key={i} step={s} depth={depth + 1}></StepTreeItem>);
+    const logChildren = (step.log || []).map((l, i) => <LogTreeItem key={step.steps.length + i} log={l} depth={depth + 1}></LogTreeItem>);
+    return [...stepChildren, ...logChildren];
  } : undefined} depth={depth}></TreeItem>;
 };

+const LogTreeItem: React.FC<{
+  log: string;
+  depth: number,
+}> = ({ log, depth }) => {
+  return <TreeItem title={<div style={{ display: 'flex', alignItems: 'center', flex: 'auto' }}>{ log }</div>} depth={depth}></TreeItem>;
+};
+
 function statusIconForFailedTests(failedTests: number) {
  return failedTests ? statusIcon('failed') : statusIcon('passed');
 }
--- a/tests/config/browserTest.ts
+++ b/tests/config/browserTest.ts
@ -140,8 +140,13 @@ export const playwrightFixtures: Fixtures<PlaywrightTestOptions & PlaywrightTest
        await context.tracing.start({ screenshots: true, snapshots: true });
      (context as any)._csi = {
        onApiCall: (stackTrace: any) => {
-          const step = (testInfo as any)._addStep('pw:api', stackTrace.apiName);
-          return (log, error) => step.complete(error);
+          const testInfoImpl = testInfo as any;
+          const existingStep = testInfoImpl._currentSteps().find(step => step.category === 'pw:api' || step.category === 'expect');
+          const newStep = existingStep ? undefined : testInfoImpl._addStep('pw:api', stackTrace.apiName, { stack: stackTrace.frames, log: [] });
+          return (log: string[], error?: Error) => {
+            (existingStep || newStep)?.data.log?.push(...log);
+            newStep?.complete(error);
+          };
        },
      };
      contexts.push(context);
--- a/tests/playwright-test/raw-reporter.spec.ts
+++ b/tests/playwright-test/raw-reporter.spec.ts
@ -56,18 +56,28 @@ test('should save stdio', async ({ runInlineTest }, testInfo) => {
      const { test } = pwt;
      test('passes', async ({ page }, testInfo) => {
        console.log('STDOUT');
+        process.stdout.write(Buffer.from([1, 2, 3]));
        console.error('STDERR');
+        process.stderr.write(Buffer.from([4, 5, 6]));
      });
    `,
  }, { usesCustomOutputDir: true });
  const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8'));
  const result = json.suites[0].tests[0].results[0];
-  expect(result.attachments[0].name).toBe('stdout');
-  expect(result.attachments[1].name).toBe('stderr');
-  const path1 = result.attachments[0].path;
-  expect(fs.readFileSync(path1, 'utf-8')).toContain('STDOUT');
-  const path2 = result.attachments[1].path;
-  expect(fs.readFileSync(path2, 'utf-8')).toContain('STDERR');
+  expect(result.attachments).toEqual([
+    { name: 'stdout', contentType: 'text/plain', body: 'STDOUT\n' },
+    {
+      name: 'stdout',
+      contentType: 'application/octet-stream',
+      body: 'AQID'
+    },
+    { name: 'stderr', contentType: 'text/plain', body: 'STDERR\n' },
+    {
+      name: 'stderr',
+      contentType: 'application/octet-stream',
+      body: 'BAUG'
+    }
+  ]);
 });

 test('should save attachments', async ({ runInlineTest }, testInfo) => {
@ -91,9 +101,8 @@ test('should save attachments', async ({ runInlineTest }, testInfo) => {
  const json = JSON.parse(fs.readFileSync(testInfo.outputPath('test-results', 'report', 'project.report'), 'utf-8'));
  const result = json.suites[0].tests[0].results[0];
  expect(result.attachments[0].name).toBe('binary');
+  expect(Buffer.from(result.attachments[0].body, 'base64')).toEqual(Buffer.from([1,2,3]));
  expect(result.attachments[1].name).toBe('text');
-  const path1 = result.attachments[0].path;
-  expect(fs.readFileSync(path1)).toEqual(Buffer.from([1,2,3]));
  const path2 = result.attachments[1].path;
  expect(path2).toBe('dummy-path');
 });