chore: throttle thumbnail workers, remove video processing (#5097)

src/cli/traceViewer/screenshotGenerator.ts

@@ -27,45 +27,43 @@ const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs));
 
 export class ScreenshotGenerator {
   private _traceStorageDir: string;
-  private _browserPromise: Promise<playwright.Browser> | undefined;
+  private _browserPromise: Promise<playwright.Browser>;
   private _traceModel: TraceModel;
   private _rendering = new Map<ActionEntry, Promise<Buffer | undefined>>();
+  private _lock = new Lock(3);
 
   constructor(traceStorageDir: string, traceModel: TraceModel) {
     this._traceStorageDir = traceStorageDir;
     this._traceModel = traceModel;
+    this._browserPromise = playwright.chromium.launch();
   }
 
-  async generateScreenshot(actionId: string): Promise<Buffer | undefined> {
+  generateScreenshot(actionId: string): Promise<Buffer | undefined> {
     const { context, action } = actionById(this._traceModel, actionId);
     if (!action.action.snapshot)
-      return;
+      return Promise.resolve(undefined);
-    const imageFileName = path.join(this._traceStorageDir, action.action.snapshot.sha1 + '-thumbnail.png');
+    if (!this._rendering.has(action)) {
-
+      this._rendering.set(action, this._render(context, action).then(body => {
-    let body: Buffer | undefined;
+        this._rendering.delete(action);
-    try {
+        return body;
-      body = await fsReadFileAsync(imageFileName);
+      }));
-    } catch (e) {
-      if (!this._rendering.has(action)) {
-        this._rendering.set(action, this._render(context, action, imageFileName).then(body => {
-          this._rendering.delete(action);
-          return body;
-        }));
-      }
-      body = await this._rendering.get(action)!;
     }
-    return body;
+    return this._rendering.get(action)!;
   }
 
-  private _browser() {
+  private async _render(contextEntry: ContextEntry, actionEntry: ActionEntry): Promise<Buffer | undefined> {
-    if (!this._browserPromise)
+    const imageFileName = path.join(this._traceStorageDir, actionEntry.action.snapshot!.sha1 + '-screenshot.png');
-      this._browserPromise = playwright.chromium.launch();
+    try {
-    return this._browserPromise;
+      return await fsReadFileAsync(imageFileName);
-  }
+    } catch (e) {
+      // fall through
+    }
 
-  private async _render(contextEntry: ContextEntry, actionEntry: ActionEntry, imageFileName: string): Promise<Buffer | undefined> {
     const { action } = actionEntry;
-    const browser = await this._browser();
+    const browser = await this._browserPromise;
+
+    await this._lock.obtain();
+
     const page = await browser.newPage({
       viewport: contextEntry.created.viewportSize,
       deviceScaleFactor: contextEntry.created.deviceScaleFactor
@@ -88,49 +86,44 @@ export class ScreenshotGenerator {
       console.log('Generating screenshot for ' + action.action, snapshotObject.frames[0].url); // eslint-disable-line no-console
       await page.goto(url);
 
-      let clip: any = undefined;
       const element = await page.$(action.selector || '*[__playwright_target__]');
       if (element) {
         await element.evaluate(e => {
           e.style.backgroundColor = '#ff69b460';
         });
-
-        clip = await element.boundingBox() || undefined;
-        if (clip) {
-          const thumbnailSize = {
-            width: 400,
-            height: 200
-          };
-          const insets = {
-            width: 60,
-            height: 30
-          };
-          clip.width = Math.min(thumbnailSize.width, clip.width);
-          clip.height = Math.min(thumbnailSize.height, clip.height);
-          if (clip.width < thumbnailSize.width) {
-            clip.x -= (thumbnailSize.width - clip.width) / 2;
-            clip.x = Math.max(0, clip.x);
-            clip.width = thumbnailSize.width;
-          } else {
-            clip.x = Math.max(0, clip.x - insets.width);
-          }
-          if (clip.height < thumbnailSize.height) {
-            clip.y -= (thumbnailSize.height - clip.height) / 2;
-            clip.y = Math.max(0, clip.y);
-            clip.height = thumbnailSize.height;
-          } else {
-            clip.y = Math.max(0, clip.y - insets.height);
-          }
-        }
       }
-
+      const imageData = await page.screenshot();
-      const imageData = await page.screenshot({ clip });
       await fsWriteFileAsync(imageFileName, imageData);
       return imageData;
     } catch (e) {
       console.log(e); // eslint-disable-line no-console
     } finally {
       await page.close();
+      this._lock.release();
     }
   }
 }
+
+class Lock {
+  private _maxWorkers: number;
+  private _callbacks: (() => void)[] = [];
+  private _workers = 0;
+
+  constructor(maxWorkers: number) {
+    this._maxWorkers = maxWorkers;
+  }
+
+  async obtain() {
+    while (this._workers === this._maxWorkers)
+      await new Promise(f => this._callbacks.push(f));
+    ++this._workers;
+  }
+
+  release() {
+    --this._workers;
+    const callbacks = this._callbacks;
+    this._callbacks = [];
+    for (const callback of callbacks)
+      callback();
+  }
+}

src/cli/traceViewer/traceViewer.ts

@@ -22,7 +22,6 @@ import { ScreenshotGenerator } from './screenshotGenerator';
 import { SnapshotRouter } from './snapshotRouter';
 import { readTraceFile, TraceModel } from './traceModel';
 import type { ActionTraceEvent, PageSnapshot, TraceEvent } from '../../trace/traceTypes';
-import { VideoTileGenerator } from './videoTileGenerator';
 
 const fsReadFileAsync = util.promisify(fs.readFile.bind(fs));
 
@@ -31,7 +30,6 @@ type TraceViewerDocument = {
   model: TraceModel;
   snapshotRouter: SnapshotRouter;
   screenshotGenerator: ScreenshotGenerator;
-  videoTileGenerator: VideoTileGenerator;
 };
 
 const emptyModel: TraceModel = {
@@ -75,7 +73,6 @@ class TraceViewer {
       resourcesDir,
       snapshotRouter: new SnapshotRouter(resourcesDir),
       screenshotGenerator: new ScreenshotGenerator(resourcesDir, model),
-      videoTileGenerator: new VideoTileGenerator(model)
     };
 
     for (const name of fs.readdirSync(traceDir)) {
@@ -119,7 +116,7 @@ class TraceViewer {
             console.error(e);
           return;
         }
-        const element = await snapshotFrame.$(action.selector || '*[__playwright_target__]');
+        const element = await snapshotFrame.$(action.selector || '*[__playwright_target__]').catch(e => undefined);
         if (element) {
           await element.evaluate(e => {
             e.style.backgroundColor = '#ff69b460';
@@ -130,9 +127,6 @@ class TraceViewer {
       }
     });
     await uiPage.exposeBinding('getTraceModel', () => this._document ? this._document.model : emptyModel);
-    await uiPage.exposeBinding('getVideoMetaInfo', async (_, videoId: string) => {
-      return this._document ? this._document.videoTileGenerator.render(videoId) : null;
-    });
     await uiPage.route('**/*', (route, request) => {
       if (request.frame().parentFrame() && this._document) {
         this._document.snapshotRouter.route(route);
@@ -151,13 +145,7 @@ class TraceViewer {
           });
           return;
         }
-        let filePath: string;
+        const filePath = path.join(__dirname, 'web', url.pathname.substring(1));
-        if (this._document && request.url().includes('video-tile')) {
-          const fullPath = url.pathname.substring('/video-tile/'.length);
-          filePath = this._document.videoTileGenerator.tilePath(fullPath);
-        } else {
-          filePath = path.join(__dirname, 'web', url.pathname.substring(1));
-        }
         const body = fs.readFileSync(filePath);
         route.fulfill({
           contentType: extensionToMime[path.extname(url.pathname).substring(1)] || 'text/plain',

src/cli/traceViewer/videoTileGenerator.ts

@@ -1,87 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { spawnSync } from 'child_process';
-import * as fs from 'fs';
-import * as path from 'path';
-import * as util from 'util';
-import { TraceModel, videoById, VideoMetaInfo } from './traceModel';
-import type { PageVideoTraceEvent } from '../../trace/traceTypes';
-import { ffmpegExecutable } from '../../utils/binaryPaths';
-
-const fsReadFileAsync = util.promisify(fs.readFile.bind(fs));
-const fsWriteFileAsync = util.promisify(fs.writeFile.bind(fs));
-
-export class VideoTileGenerator {
-  private _traceModel: TraceModel;
-
-  constructor(traceModel: TraceModel) {
-    this._traceModel = traceModel;
-  }
-
-  tilePath(urlPath: string) {
-    const index = urlPath.lastIndexOf('/');
-    const tile = urlPath.substring(index + 1);
-    const videoId = urlPath.substring(0, index);
-    const { context, page } = videoById(this._traceModel, videoId);
-    const videoFilePath = path.join(path.dirname(context.filePath), page.video!.video.fileName);
-    return videoFilePath + '-' + tile;
-  }
-
-  async render(videoId: string): Promise<VideoMetaInfo | undefined> {
-    const { context, page } = videoById(this._traceModel, videoId);
-    const video = page.video!.video;
-    const videoFilePath = path.join(path.dirname(context.filePath), video.fileName);
-    const metaInfoFilePath = videoFilePath + '-metainfo.txt';
-    try {
-      const metaInfo = await fsReadFileAsync(metaInfoFilePath, 'utf8');
-      return metaInfo ? JSON.parse(metaInfo) : undefined;
-    } catch (e) {
-    }
-
-    const ffmpeg = ffmpegExecutable()!;
-    console.log('Generating frames for ' + videoFilePath); // eslint-disable-line no-console
-    // Force output frame rate to 25 fps as otherwise it would produce one image per timebase unit
-    // which is currently 1 / (25 * 1000).
-    const result = spawnSync(ffmpeg, ['-i', videoFilePath, '-r', '25', `${videoFilePath}-%03d.png`]);
-    const metaInfo = parseMetaInfo(result.stderr.toString(), video);
-    await fsWriteFileAsync(metaInfoFilePath, metaInfo ? JSON.stringify(metaInfo) : '');
-    return metaInfo;
-  }
-}
-
-function parseMetaInfo(text: string, video: PageVideoTraceEvent): VideoMetaInfo | undefined {
-  const lines = text.split('\n');
-  let framesLine = lines.find(l => l.startsWith('frame='));
-  if (!framesLine)
-    return;
-  framesLine = framesLine.substring(framesLine.lastIndexOf('frame='));
-  const framesMatch = framesLine.match(/frame=\s+(\d+)/);
-  const outputLineIndex = lines.findIndex(l => l.trim().startsWith('Output #0'));
-  const streamLine = lines.slice(outputLineIndex).find(l => l.trim().startsWith('Stream #0:0'))!;
-  const fpsMatch = streamLine.match(/, (\d+) fps,/);
-  const resolutionMatch = streamLine.match(/, (\d+)x(\d+)\D/);
-  const durationMatch = lines.find(l => l.trim().startsWith('Duration'))!.match(/Duration: (\d+):(\d\d):(\d\d.\d\d)/);
-  const duration = (((parseInt(durationMatch![1], 10) * 60) + parseInt(durationMatch![2], 10)) * 60 + parseFloat(durationMatch![3])) * 1000;
-  return {
-    frames: parseInt(framesMatch![1], 10),
-    width: parseInt(resolutionMatch![1], 10),
-    height: parseInt(resolutionMatch![2], 10),
-    fps: parseInt(fpsMatch![1], 10),
-    startTime: video.timestamp,
-    endTime: video.timestamp + duration
-  };
-}

src/cli/traceViewer/web/index.tsx

@@ -24,7 +24,6 @@ import { applyTheme } from './theme';
 declare global {
   interface Window {
     getTraceModel(): Promise<TraceModel>;
-    getVideoMetaInfo(videoId: string): Promise<VideoMetaInfo | undefined>;
     readFile(filePath: string): Promise<string>;
     renderSnapshot(action: trace.ActionTraceEvent): void;
   }

src/cli/traceViewer/web/ui/filmStrip.css

@@ -1,45 +0,0 @@
-/*
-  Copyright (c) Microsoft Corporation.
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-*/
-
-.film-strip {
-  flex: none;
-  display: flex;
-  flex-direction: column;
-  position: relative;
-}
-
-.film-strip-lane {
-  flex: none;
-  display: flex;
-}
-
-.film-strip-frame {
-  flex: none;
-  pointer-events: none;
-  box-shadow: var(--box-shadow);
-}
-
-.film-strip-hover {
-  position: absolute;
-  top: 0;
-  right: 0;
-  bottom: 0;
-  left: 0;
-  background-color: white;
-  box-shadow: rgba(0, 0, 0, 0.133) 0px 1.6px 3.6px 0px, rgba(0, 0, 0, 0.11) 0px 0.3px 0.9px 0px;
-  box-shadow: rgba(0, 0, 0, 0.133) 0px 1.6px 10px 0px, rgba(0, 0, 0, 0.11) 0px 0.3px 10px 0px;
-  z-index: 10;
-}

src/cli/traceViewer/web/ui/filmStrip.tsx

@@ -1,136 +0,0 @@
-/*
-  Copyright (c) Microsoft Corporation.
-
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-      http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License.
-*/
-
-import { ContextEntry, VideoEntry, VideoMetaInfo } from '../../traceModel';
-import './filmStrip.css';
-import { Boundaries } from '../geometry';
-import * as React from 'react';
-import { useAsyncMemo, useMeasure } from './helpers';
-
-function imageURL(videoId: string, index: number) {
-  const imageURLpadding = '0'.repeat(3 - String(index + 1).length);
-  return `video-tile/${videoId}/${imageURLpadding}${index + 1}.png`;
-}
-
-export const FilmStrip: React.FunctionComponent<{
-  context: ContextEntry,
-  boundaries: Boundaries,
-  previewX?: number,
-}> = ({ context, boundaries, previewX }) => {
-  const [measure, ref] = useMeasure<HTMLDivElement>();
-
-  const videos = React.useMemo(() => {
-    const videos: VideoEntry[] = [];
-    for (const page of context.pages) {
-      if (page.video)
-        videos.push(page.video);
-    }
-    return videos;
-  }, [context]);
-
-  const metaInfos = useAsyncMemo<Map<VideoEntry, VideoMetaInfo | undefined>>(async () => {
-    const infos = new Map<VideoEntry, VideoMetaInfo | undefined>();
-    for (const video of videos)
-      infos.set(video, await window.getVideoMetaInfo(video.videoId));
-    return infos;
-  }, [videos], new Map(), new Map());
-
-  // TODO: pick file from the Y position.
-  const previewVideo = videos[0];
-  const previewMetaInfo = metaInfos.get(previewVideo);
-  let previewIndex = 0;
-  if ((previewX !== undefined) && previewMetaInfo) {
-    const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewX / measure.width;
-    previewIndex = (previewTime - previewMetaInfo.startTime) / (previewMetaInfo.endTime - previewMetaInfo.startTime) * previewMetaInfo.frames | 0;
-  }
-
-  const previewImage = useAsyncMemo<HTMLImageElement | undefined>(async () => {
-    if (!previewMetaInfo || previewIndex < 0 || previewIndex >= previewMetaInfo.frames)
-      return;
-    const idealWidth = previewMetaInfo.width / 2;
-    const idealHeight = previewMetaInfo.height / 2;
-    const ratio = Math.min(1, (measure.width - 20) / idealWidth);
-    const image = new Image((idealWidth * ratio) | 0, (idealHeight * ratio) | 0);
-    image.src = imageURL(previewVideo.videoId, previewIndex);
-    await new Promise(f => image.onload = f);
-    return image;
-  }, [previewMetaInfo, previewIndex, measure.width, previewVideo], undefined);
-
-  return <div className='film-strip' ref={ref}>{
-    videos.map(video => <FilmStripLane
-      boundaries={boundaries}
-      video={video}
-      metaInfo={metaInfos.get(video)}
-      width={measure.width}
-      key={video.videoId}
-    />)
-  }
-  {(previewX !== undefined) && previewMetaInfo && previewImage &&
-    <div className='film-strip-hover' style={{
-      width: previewImage.width + 'px',
-      height: previewImage.height + 'px',
-      top: measure.bottom + 5 + 'px',
-      left: Math.min(previewX, measure.width - previewImage.width - 10) + 'px',
-    }}>
-      <img src={previewImage.src} width={previewImage.width} height={previewImage.height} />
-    </div>
-  }
-  </div>;
-};
-
-const FilmStripLane: React.FunctionComponent<{
-  boundaries: Boundaries,
-  video: VideoEntry,
-  metaInfo: VideoMetaInfo | undefined,
-  width: number,
-}> = ({ boundaries, video, metaInfo, width }) => {
-  const frameHeight = 45;
-  const frameMargin = 2.5;
-
-  if (!metaInfo)
-    return <div className='film-strip-lane' style={{ height: (frameHeight + 2 * frameMargin) + 'px' }}></div>;
-
-  const frameWidth = frameHeight / metaInfo.height * metaInfo.width | 0;
-  const boundariesSize = boundaries.maximum - boundaries.minimum;
-  const gapLeft = (metaInfo.startTime - boundaries.minimum) / boundariesSize * width;
-  const gapRight = (boundaries.maximum - metaInfo.endTime) / boundariesSize * width;
-  const effectiveWidth = (metaInfo.endTime - metaInfo.startTime) / boundariesSize * width;
-
-  const frameCount = effectiveWidth / (frameWidth + 2 * frameMargin) | 0;
-  const frameStep = metaInfo.frames / frameCount;
-  const frameGap = frameCount <= 1 ? 0 : (effectiveWidth - (frameWidth + 2 * frameMargin) * frameCount) / (frameCount - 1);
-
-  const frames: JSX.Element[] = [];
-  for (let i = 0; i < metaInfo.frames; i += frameStep) {
-    let index = i | 0;
-    // Always show last frame.
-    if (Math.floor(i + frameStep) >= metaInfo.frames)
-      index = metaInfo.frames - 1;
-    frames.push(<div className='film-strip-frame' key={i} style={{
-      width: frameWidth + 'px',
-      height: frameHeight + 'px',
-      backgroundImage: `url(${imageURL(video.videoId, index)})`,
-      backgroundSize: `${frameWidth}px ${frameHeight}px`,
-      margin: frameMargin + 'px',
-      marginRight: (frameMargin + frameGap) + 'px',
-    }} />);
-  }
-
-  return <div className='film-strip-lane' style={{
-    marginLeft: gapLeft + 'px',
-    marginRight: gapRight + 'px',
-  }}>{frames}</div>;
-};

src/cli/traceViewer/web/ui/timeline.tsx

@@ -17,7 +17,6 @@
 
 import { ContextEntry, InterestingPageEvent, ActionEntry, trace } from '../../traceModel';
 import './timeline.css';
-import { FilmStrip } from './filmStrip';
 import { Boundaries } from '../geometry';
 import * as React from 'react';
 import { useMeasure } from './helpers';
@@ -189,7 +188,6 @@ export const Timeline: React.FunctionComponent<{
         ></div>;
       })
     }</div>
-    <FilmStrip context={context} boundaries={boundaries} previewX={previewX} />
     <div className='timeline-marker timeline-marker-hover' style={{
       display: (previewX !== undefined) ? 'block' : 'none',
       left: (previewX || 0) + 'px',