chore(ui): show live trace (#21677)

packages/playwright-core/src/server/trace/viewer/traceViewer.ts

@@ -31,12 +31,18 @@ type Options = { app?: string, headless?: boolean, host?: string, port?: number
 export async function showTraceViewer(traceUrls: string[], browserName: string, options?: Options): Promise<Page> {
   const { headless = false, host, port, app } = options || {};
   for (const traceUrl of traceUrls) {
-    if (!traceUrl.startsWith('http://') && !traceUrl.startsWith('https://') && !fs.existsSync(traceUrl)) {
+    let traceFile = traceUrl;
+    // If .json is requested, we'll synthesize it.
+    if (traceUrl.endsWith('.json'))
+      traceFile = traceUrl.substring(0, traceUrl.length - '.json'.length);
+
+    if (!traceUrl.startsWith('http://') && !traceUrl.startsWith('https://') && !fs.existsSync(traceFile) && !fs.existsSync(traceFile + '.trace')) {
       // eslint-disable-next-line no-console
       console.error(`Trace file ${traceUrl} does not exist!`);
       process.exit(1);
     }
   }
+
   const server = new HttpServer();
   server.routePrefix('/trace', (request, response) => {
     const url = new URL('http://localhost' + request.url!);
@@ -45,7 +51,18 @@ export async function showTraceViewer(traceUrls: string[], browserName: string,
       return true;
     if (relativePath.startsWith('/file')) {
       try {
-        return server.serveFile(request, response, url.searchParams.get('path')!);
+        const filePath = url.searchParams.get('path')!;
+        if (fs.existsSync(filePath))
+          return server.serveFile(request, response, url.searchParams.get('path')!);
+
+        // If .json is requested, we'll synthesize it for zip-less operation.
+        if (filePath.endsWith('.json')) {
+          const traceName = filePath.substring(0, filePath.length - '.json'.length);
+          response.statusCode = 200;
+          response.setHeader('Content-Type', 'application/json');
+          response.end(JSON.stringify(traceDescriptor(traceName)));
+          return true;
+        }
       } catch (e) {
         return false;
       }
@@ -102,3 +119,24 @@ export async function showTraceViewer(traceUrls: string[], browserName: string,
   await page.mainFrame().goto(serverSideCallMetadata(), urlPrefix + `/trace/${app || 'index.html'}${searchQuery}`);
   return page;
 }
+
+function traceDescriptor(traceName: string) {
+  const result: { entries: { name: string, path: string }[] } = {
+    entries: []
+  };
+
+  const traceDir = path.dirname(traceName);
+  const traceFile = path.basename(traceName);
+  for (const name of fs.readdirSync(traceDir)) {
+    // 23423423.trace => 23423423-trace.trace
+    if (name.startsWith(traceFile))
+      result.entries.push({ name: name.replace(traceFile, traceFile + '-trace'), path: path.join(traceDir, name) });
+  }
+
+  const resourcesDir = path.join(traceDir, 'resources');
+  if (fs.existsSync(resourcesDir)) {
+    for (const name of fs.readdirSync(resourcesDir))
+      result.entries.push({ name: 'resources/' + name, path: path.join(resourcesDir, name) });
+  }
+  return result;
+}

packages/trace-viewer/src/traceModel.ts

@@ -29,66 +29,48 @@ const zipjs = zipImport as typeof zip;
 export class TraceModel {
   contextEntries: ContextEntry[] = [];
   pageEntries = new Map<string, PageEntry>();
-  private _snapshotStorage: PersistentSnapshotStorage | undefined;
-  private _entries = new Map<string, zip.Entry>();
+  private _snapshotStorage: BaseSnapshotStorage | undefined;
   private _version: number | undefined;
-  private _zipReader: zip.ZipReader | undefined;
+  private _backend!: TraceModelBackend;
 
   constructor() {
   }
 
-  private _formatUrl(trace: string) {
-    let url = trace.startsWith('http') || trace.startsWith('blob') ? trace : `file?path=${trace}`;
-    // Dropbox does not support cors.
-    if (url.startsWith('https://www.dropbox.com/'))
-      url = 'https://dl.dropboxusercontent.com/' + url.substring('https://www.dropbox.com/'.length);
-    return url;
-  }
-
   async load(traceURL: string, progress: (done: number, total: number) => void) {
-    this._zipReader = new zipjs.ZipReader( // @ts-ignore
-        new zipjs.HttpReader(this._formatUrl(traceURL), { mode: 'cors', preventHeadRequest: true }),
-        { useWebWorkers: false }) as zip.ZipReader;
+    this._backend = traceURL.endsWith('json') ? new FetchTraceModelBackend(traceURL) : new ZipTraceModelBackend(traceURL, progress);
 
     const ordinals: string[] = [];
     let hasSource = false;
-    for (const entry of await this._zipReader.getEntries({ onprogress: progress })) {
-      const match = entry.filename.match(/([\d]+-)?trace\.trace/);
+    for (const entryName of await this._backend.entryNames()) {
+      const match = entryName.match(/(.+-)?trace\.trace/);
       if (match)
         ordinals.push(match[1] || '');
-      if (entry.filename.includes('src@'))
+      if (entryName.includes('src@'))
         hasSource = true;
-      this._entries.set(entry.filename, entry);
     }
     if (!ordinals.length)
       throw new Error('Cannot find .trace file');
 
-    this._snapshotStorage = new PersistentSnapshotStorage(this._entries);
+    this._snapshotStorage = new PersistentSnapshotStorage(this._backend);
 
     for (const ordinal of ordinals) {
       const contextEntry = createEmptyContext();
       contextEntry.traceUrl = traceURL;
       contextEntry.hasSource = hasSource;
 
-      const traceWriter = new zipjs.TextWriter() as zip.TextWriter;
-      const traceEntry = this._entries.get(ordinal + 'trace.trace')!;
-      await traceEntry!.getData!(traceWriter);
-      for (const line of (await traceWriter.getData()).split('\n'))
+      const trace = await this._backend.readText(ordinal + 'trace.trace') || '';
+      for (const line of trace.split('\n'))
         this.appendEvent(contextEntry, line);
 
-      const networkWriter = new zipjs.TextWriter();
-      const networkEntry = this._entries.get(ordinal + 'trace.network')!;
-      await networkEntry?.getData?.(networkWriter);
-      for (const line of (await networkWriter.getData()).split('\n'))
+      const network = await this._backend.readText(ordinal + 'trace.network') || '';
+      for (const line of network.split('\n'))
         this.appendEvent(contextEntry, line);
 
-      const stacksWriter = new zipjs.TextWriter();
-      const stacksEntry = this._entries.get(ordinal + 'trace.stacks');
-      if (stacksEntry) {
-        await stacksEntry!.getData!(stacksWriter);
-        const stacks = parseClientSideCallMetadata(JSON.parse(await stacksWriter.getData()));
+      const stacks = await this._backend.readText(ordinal + 'trace.stacks');
+      if (stacks) {
+        const callMetadata = parseClientSideCallMetadata(JSON.parse(stacks));
         for (const action of contextEntry.actions)
-          action.stack = action.stack || stacks.get(action.callId);
+          action.stack = action.stack || callMetadata.get(action.callId);
       }
 
       contextEntry.actions.sort((a1, a2) => a1.startTime - a2.startTime);
@@ -97,25 +79,14 @@ export class TraceModel {
   }
 
   async hasEntry(filename: string): Promise<boolean> {
-    if (!this._zipReader)
-      return false;
-    for (const entry of await this._zipReader.getEntries()) {
-      if (entry.filename === filename)
-        return true;
-    }
-    return false;
+    return this._backend.hasEntry(filename);
   }
 
   async resourceForSha1(sha1: string): Promise<Blob | undefined> {
-    const entry = this._entries.get('resources/' + sha1);
-    if (!entry)
-      return;
-    const blobWriter = new zipjs.BlobWriter() as zip.BlobWriter;
-    await entry!.getData!(blobWriter);
-    return await blobWriter.getData();
+    return this._backend.readBlob('resources/' + sha1);
   }
 
-  storage(): PersistentSnapshotStorage {
+  storage(): BaseSnapshotStorage {
     return this._snapshotStorage!;
   }
 
@@ -289,18 +260,120 @@ export class TraceModel {
   }
 }
 
-export class PersistentSnapshotStorage extends BaseSnapshotStorage {
-  private _entries: Map<string, zip.Entry>;
+export interface TraceModelBackend {
+  entryNames(): Promise<string[]>;
+  hasEntry(entryName: string): Promise<boolean>;
+  readText(entryName: string): Promise<string | undefined>;
+  readBlob(entryName: string): Promise<Blob | undefined>;
+}
 
-  constructor(entries: Map<string, zip.Entry>) {
-    super();
-    this._entries = entries;
+class ZipTraceModelBackend implements TraceModelBackend {
+  private _zipReader: zip.ZipReader;
+  private _entriesPromise: Promise<Map<string, zip.Entry>>;
+
+  constructor(traceURL: string, progress: (done: number, total: number) => void) {
+    this._zipReader = new zipjs.ZipReader(
+        new zipjs.HttpReader(formatUrl(traceURL), { mode: 'cors', preventHeadRequest: true } as any),
+        { useWebWorkers: false }) as zip.ZipReader;
+    this._entriesPromise = this._zipReader.getEntries({ onprogress: progress }).then(entries => {
+      const map = new Map<string, zip.Entry>();
+      for (const entry of entries)
+        map.set(entry.filename, entry);
+      return map;
+    });
   }
 
-  async resourceContent(sha1: string): Promise<Blob | undefined> {
-    const entry = this._entries.get('resources/' + sha1)!;
-    const writer = new zipjs.BlobWriter();
+  async entryNames(): Promise<string[]> {
+    const entries = await this._entriesPromise;
+    return [...entries.keys()];
+  }
+
+  async hasEntry(entryName: string): Promise<boolean> {
+    const entries = await this._entriesPromise;
+    return entries.has(entryName);
+  }
+
+  async readText(entryName: string): Promise<string | undefined> {
+    const entries = await this._entriesPromise;
+    const entry = entries.get(entryName);
+    if (!entry)
+      return;
+    const writer = new zipjs.TextWriter();
+    await entry.getData?.(writer);
+    return writer.getData();
+  }
+
+  async readBlob(entryName: string): Promise<Blob | undefined> {
+    const entries = await this._entriesPromise;
+    const entry = entries.get(entryName);
+    if (!entry)
+      return;
+    const writer = new zipjs.BlobWriter() as zip.BlobWriter;
     await entry.getData!(writer);
     return writer.getData();
   }
 }
+
+class FetchTraceModelBackend implements TraceModelBackend {
+  private _entriesPromise: Promise<Map<string, string>>;
+
+  constructor(traceURL: string) {
+
+    this._entriesPromise = fetch('/trace/file?path=' + encodeURI(traceURL)).then(async response => {
+      const json = JSON.parse(await response.text());
+      const entries = new Map<string, string>();
+      for (const entry of json.entries)
+        entries.set(entry.name, entry.path);
+      return entries;
+    });
+  }
+
+  async entryNames(): Promise<string[]> {
+    const entries = await this._entriesPromise;
+    return [...entries.keys()];
+  }
+
+  async hasEntry(entryName: string): Promise<boolean> {
+    const entries = await this._entriesPromise;
+    return entries.has(entryName);
+  }
+
+  async readText(entryName: string): Promise<string | undefined> {
+    const response = await this._readEntry(entryName);
+    return response?.text();
+  }
+
+  async readBlob(entryName: string): Promise<Blob | undefined> {
+    const response = await this._readEntry(entryName);
+    return response?.blob();
+  }
+
+  private async _readEntry(entryName: string): Promise<Response | undefined> {
+    const entries = await this._entriesPromise;
+    const fileName = entries.get(entryName);
+    if (!fileName)
+      return;
+    return fetch('/trace/file?path=' + encodeURI(fileName));
+  }
+}
+
+export class PersistentSnapshotStorage extends BaseSnapshotStorage {
+  private _backend: TraceModelBackend;
+
+  constructor(backend: TraceModelBackend) {
+    super();
+    this._backend = backend;
+  }
+
+  async resourceContent(sha1: string): Promise<Blob | undefined> {
+    return this._backend.readBlob('resources/' + sha1);
+  }
+}
+
+function formatUrl(trace: string) {
+  let url = trace.startsWith('http') || trace.startsWith('blob') ? trace : `file?path=${trace}`;
+  // Dropbox does not support cors.
+  if (url.startsWith('https://www.dropbox.com/'))
+    url = 'https://dl.dropboxusercontent.com/' + url.substring('https://www.dropbox.com/'.length);
+  return url;
+}

packages/trace-viewer/src/ui/watchMode.tsx

@@ -22,21 +22,19 @@ import { TreeView } from '@web/components/treeView';
 import type { TreeState } from '@web/components/treeView';
 import { TeleReporterReceiver, TeleSuite } from '@testIsomorphic/teleReceiver';
 import type { TeleTestCase } from '@testIsomorphic/teleReceiver';
-import type { FullConfig, Suite, TestCase, TestResult, TestStep, Location } from '../../../playwright-test/types/testReporter';
+import type { FullConfig, Suite, TestCase, TestResult, Location } from '../../../playwright-test/types/testReporter';
 import { SplitView } from '@web/components/splitView';
 import { MultiTraceModel } from './modelUtil';
 import './watchMode.css';
 import { ToolbarButton } from '@web/components/toolbarButton';
 import { Toolbar } from '@web/components/toolbar';
 import type { ContextEntry } from '../entries';
-import type * as trace from '@trace/trace';
 import type { XtermDataSource } from '@web/components/xtermWrapper';
 import { XtermWrapper } from '@web/components/xtermWrapper';
 import { Expandable } from '@web/components/expandable';
 import { toggleTheme } from '@web/theme';
 
 let updateRootSuite: (rootSuite: Suite, progress: Progress) => void = () => {};
-let updateStepsProgress: () => void = () => {};
 let runWatchedTests = (fileName: string) => {};
 let xtermSize = { cols: 80, rows: 24 };
 
@@ -100,6 +98,15 @@ export const WatchModeView: React.FC<{}> = ({
     setProgress(newProgress);
   };
 
+  const outputDir = React.useMemo(() => {
+    let outputDir = '';
+    for (const p of rootSuite.value?.suites || []) {
+      outputDir = p.project()?.outputDir || '';
+      break;
+    }
+    return outputDir;
+  }, [rootSuite]);
+
   const runTests = (testIds: string[]) => {
     // Clear test results.
     {
@@ -122,7 +129,7 @@ export const WatchModeView: React.FC<{}> = ({
 
   const isRunningTest = !!runningState;
   const result = selectedTest?.results[0];
-  const isFinished = result && result.duration >= 0;
+
   return <div className='vbox watch-mode'>
     <SplitView sidebarSize={250} orientation='horizontal' sidebarIsFirst={true}>
       <div className='vbox'>
@@ -135,8 +142,7 @@ export const WatchModeView: React.FC<{}> = ({
           <XtermWrapper source={xtermDataSource}></XtermWrapper>;
         </div>
         <div className={'vbox' + (isShowingOutput ? ' hidden' : '')}>
-          {isFinished && <FinishedTraceView testResult={result} />}
-          {!isFinished && <InProgressTraceView testResult={result} />}
+          <TraceView outputDir={outputDir} testCase={selectedTest} result={result} />
         </div>
       </div>
       <div className='vbox watch-mode-sidebar'>
@@ -380,31 +386,36 @@ const TestList: React.FC<{
     noItemsMessage='No tests' />;
 };
 
-const InProgressTraceView: React.FC<{
-  testResult: TestResult | undefined,
-}> = ({ testResult }) => {
+const TraceView: React.FC<{
+  outputDir: string,
+  testCase: TestCase | undefined,
+  result: TestResult | undefined,
+}> = ({ outputDir, testCase, result }) => {
   const [model, setModel] = React.useState<MultiTraceModel | undefined>();
-  const [stepsProgress, setStepsProgress] = React.useState(0);
-  updateStepsProgress = () => setStepsProgress(stepsProgress + 1);
+  const [currentStep, setCurrentStep] = React.useState(0);
+  const pollTimer = React.useRef<NodeJS.Timeout | null>(null);
 
   React.useEffect(() => {
-    setModel(testResult ? stepsToModel(testResult) : undefined);
-  }, [stepsProgress, testResult]);
+    if (pollTimer.current)
+      clearTimeout(pollTimer.current);
 
-  return <Workbench model={model} hideTimelineBars={true} hideStackFrames={true} showSourcesFirst={true} />;
-};
-
-const FinishedTraceView: React.FC<{
-  testResult: TestResult,
-}> = ({ testResult }) => {
-  const [model, setModel] = React.useState<MultiTraceModel | undefined>();
-
-  React.useEffect(() => {
     // Test finished.
-    const attachment = testResult.attachments.find(a => a.name === 'trace');
-    if (attachment && attachment.path)
-      loadSingleTraceFile(attachment.path).then(setModel);
-  }, [testResult]);
+    const isFinished = result && result.duration >= 0;
+    if (isFinished) {
+      const attachment = result.attachments.find(a => a.name === 'trace');
+      if (attachment && attachment.path)
+        loadSingleTraceFile(attachment.path).then(setModel);
+      return;
+    }
+
+    const traceLocation = `${outputDir}/.playwright-artifacts-${result?.workerIndex}/traces/${testCase?.id}.json`;
+    // Start polling running test.
+    pollTimer.current = setTimeout(() => {
+      loadSingleTraceFile(traceLocation).then(setModel).then(() => {
+        setCurrentStep(currentStep + 1);
+      });
+    }, 250);
+  }, [result, outputDir, testCase, currentStep, setCurrentStep]);
 
   return <Workbench key='workbench' model={model} hideTimelineBars={true} hideStackFrames={true} showSourcesFirst={true} />;
 };
@@ -471,16 +482,6 @@ const refreshRootSuite = (eraseResults: boolean): Promise<void> => {
       else
         ++progress.passed;
       throttleUpdateRootSuite(rootSuite, progress);
-      // This will update selected trace viewer.
-      updateStepsProgress();
-    },
-
-    onStepBegin: () => {
-      updateStepsProgress();
-    },
-
-    onStepEnd: () => {
-      updateStepsProgress();
     },
   });
   return sendMessage('list', {});
@@ -741,66 +742,3 @@ async function loadSingleTraceFile(url: string): Promise<MultiTraceModel> {
   const contextEntries = await response.json() as ContextEntry[];
   return new MultiTraceModel(contextEntries);
 }
-
-function stepsToModel(result: TestResult): MultiTraceModel {
-  let startTime = Number.MAX_VALUE;
-  let endTime = Number.MIN_VALUE;
-  const actions: trace.ActionTraceEvent[] = [];
-
-  const flatSteps: TestStep[] = [];
-  const visit = (step: TestStep) => {
-    flatSteps.push(step);
-    step.steps.forEach(visit);
-  };
-  result.steps.forEach(visit);
-
-  for (const step of flatSteps) {
-    let callId: string;
-    if (step.category === 'pw:api')
-      callId = `call@${actions.length}`;
-    else if (step.category === 'expect')
-      callId = `expect@${actions.length}`;
-    else
-      continue;
-    const action: trace.ActionTraceEvent = {
-      type: 'action',
-      callId,
-      startTime: step.startTime.getTime(),
-      endTime: step.startTime.getTime() + step.duration,
-      apiName: step.title,
-      class: '',
-      method: '',
-      params: {},
-      wallTime: step.startTime.getTime(),
-      log: [],
-      snapshots: [],
-      error: step.error ? { name: 'Error', message: step.error.message || step.error.value || '' } : undefined,
-    };
-    if (startTime > action.startTime)
-      startTime = action.startTime;
-    if (endTime < action.endTime)
-      endTime = action.endTime;
-    actions.push(action);
-  }
-
-  const contextEntry: ContextEntry = {
-    traceUrl: '',
-    startTime,
-    endTime,
-    browserName: '',
-    options: {
-      viewport: undefined,
-      deviceScaleFactor: undefined,
-      isMobile: undefined,
-      userAgent: undefined
-    },
-    pages: [],
-    resources: [],
-    actions,
-    events: [],
-    initializers: {},
-    hasSource: false
-  };
-
-  return new MultiTraceModel([contextEntry]);
-}