chore(har): allow replaying from zip har (#14962)

packages/playwright-core/src/client/harRouter.ts

@@ -18,7 +18,9 @@ import fs from 'fs';
 import type { HAREntry, HARFile, HARResponse } from '../../types/types';
 import { debugLogger } from '../common/debugLogger';
 import { rewriteErrorMessage } from '../utils/stackTrace';
+import { ZipFile } from '../utils/zipFile';
 import type { BrowserContext } from './browserContext';
+import { Events } from './events';
 import type { Route } from './network';
 import type { BrowserContextOptions } from './types';
 
@@ -26,40 +28,79 @@ type HarOptions = NonNullable<BrowserContextOptions['har']>;
 
 export class HarRouter {
   private _pattern: string | RegExp;
-  private _handler: (route: Route) => Promise<void>;
+  private _harFile: HARFile;
+  private _zipFile: ZipFile | null;
+  private _options: HarOptions | undefined;
 
   static async create(options: HarOptions): Promise<HarRouter> {
+    if (options.path.endsWith('.zip')) {
+      const zipFile = new ZipFile(options.path);
+      const har = await zipFile.read('har.har');
+      const harFile = JSON.parse(har.toString()) as HARFile;
+      return new HarRouter(harFile, zipFile, options);
+    }
     const harFile = JSON.parse(await fs.promises.readFile(options.path, 'utf-8')) as HARFile;
-    return new HarRouter(harFile, options);
+    return new HarRouter(harFile, null, options);
   }
 
-  constructor(harFile: HARFile, options?: HarOptions) {
+  constructor(harFile: HARFile, zipFile: ZipFile | null, options?: HarOptions) {
+    this._harFile = harFile;
+    this._zipFile = zipFile;
     this._pattern = options?.urlFilter ?? /.*/;
-    this._handler = async (route: Route) => {
-      let response;
-      try {
-        response = harFindResponse(harFile, {
-          url: route.request().url(),
-          method: route.request().method()
+    this._options = options;
+  }
+
+  private async _handle(route: Route) {
+    let response;
+    try {
+      response = harFindResponse(this._harFile, {
+        url: route.request().url(),
+        method: route.request().method()
+      });
+    } catch (e) {
+      rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`);
+      debugLogger.log('api', e);
+    }
+
+    if (response) {
+      debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`);
+      const sha1 = (response.content as any)._sha1;
+
+      if (this._zipFile && sha1) {
+        const body = await this._zipFile.read(sha1).catch(() => {
+          debugLogger.log('api', `payload ${sha1} for request ${route.request().url()} is not found in archive`);
+          return null;
         });
-      } catch (e) {
-        rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`);
-        debugLogger.log('api', e);
+        if (body) {
+          await route.fulfill({
+            status: response.status,
+            headers: Object.fromEntries(response.headers.map(h => [h.name, h.value])),
+            body
+          });
+          return;
+        }
       }
-      if (response) {
-        debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`);
-        await route.fulfill({ response });
-      } else if (options?.fallback === 'continue') {
-        await route.fallback();
-      } else {
-        debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`);
-        await route.abort();
-      }
-    };
+
+      await route.fulfill({ response });
+      return;
+    }
+
+    if (this._options?.fallback === 'continue') {
+      await route.fallback();
+      return;
+    }
+
+    debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`);
+    await route.abort();
   }
 
   async addRoute(context: BrowserContext) {
-    await context.route(this._pattern, this._handler);
+    await context.route(this._pattern, route => this._handle(route));
+    context.once(Events.BrowserContext.Close, () => this.dispose());
+  }
+
+  dispose() {
+    this._zipFile?.close();
   }
 }
 

packages/playwright-core/src/utils/DEPS.list

@@ -3,3 +3,4 @@
 ../third_party/diff_match_patch
 ../third_party/pixelmatch
 ../utilsBundle.ts
+../zipBundle.ts

packages/playwright-core/src/utils/zipFile.ts

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { yauzl } from '../zipBundle';
+import type { UnzipFile, Entry } from '../zipBundle';
+
+export class ZipFile {
+  private _fileName: string;
+  private _zipFile: UnzipFile | undefined;
+  private _entries = new Map<string, Entry>();
+  private _openedPromise: Promise<void>;
+
+  constructor(fileName: string) {
+    this._fileName = fileName;
+    this._openedPromise = this._open();
+  }
+
+  private async _open() {
+    await new Promise<UnzipFile>((fulfill, reject) => {
+      yauzl.open(this._fileName, { autoClose: false }, (e, z) => {
+        if (e) {
+          reject(e);
+          return;
+        }
+        this._zipFile = z;
+        this._zipFile!.on('entry', (entry: Entry) => {
+          this._entries.set(entry.fileName, entry);
+        });
+        this._zipFile!.on('end', fulfill);
+      });
+    });
+  }
+
+  async entries(): Promise<string[]> {
+    await this._openedPromise;
+    return [...this._entries.keys()];
+  }
+
+  async read(entryPath: string): Promise<Buffer> {
+    await this._openedPromise;
+    const entry = this._entries.get(entryPath)!;
+    if (!entry)
+      throw new Error(`${entryPath} not found in file ${this._fileName}`);
+
+    return new Promise((resolve, reject) => {
+      this._zipFile!.openReadStream(entry, (error, readStream) => {
+        if (error || !readStream) {
+          reject(error || 'Entry not found');
+          return;
+        }
+
+        const buffers: Buffer[] = [];
+        readStream.on('data', data => buffers.push(data));
+        readStream.on('end', () => resolve(Buffer.concat(buffers)));
+      });
+    });
+  }
+
+  close() {
+    this._zipFile?.close();
+  }
+}

packages/playwright-core/src/zipBundle.ts

@@ -17,4 +17,5 @@
 export const yazl: typeof import('../bundles/zip/node_modules/@types/yazl') = require('./zipBundleImpl').yazl;
 export type { ZipFile } from '../bundles/zip/node_modules/@types/yazl';
 export const yauzl: typeof import('../bundles/zip/node_modules/@types/yauzl') = require('./zipBundleImpl').yauzl;
+export type { ZipFile as UnzipFile, Entry } from '../bundles/zip/node_modules/@types/yauzl';
 export const extract: typeof import('../bundles/zip/node_modules/extract-zip') = require('./zipBundleImpl').extract;

tests/config/utils.ts

@@ -16,7 +16,7 @@
 
 import { expect } from '@playwright/test';
 import type { Frame, Page } from 'playwright-core';
-import { ZipFileSystem } from './vfs';
+import { ZipFile } from '../../packages/playwright-core/lib/utils/zipFile';
 
 export async function attachFrame(page: Page, frameId: string, url: string): Promise<Frame> {
   const handle = await page.evaluateHandle(async ({ frameId, url }) => {
@@ -91,7 +91,7 @@ export function suppressCertificateWarning() {
 }
 
 export async function parseTrace(file: string): Promise<{ events: any[], resources: Map<string, Buffer> }> {
-  const zipFS = new ZipFileSystem(file);
+  const zipFS = new ZipFile(file);
   const resources = new Map<string, Buffer>();
   for (const entry of await zipFS.entries())
     resources.set(entry, await zipFS.read(entry));
@@ -113,7 +113,7 @@ export async function parseTrace(file: string): Promise<{ events: any[], resourc
 }
 
 export async function parseHar(file: string): Promise<Map<string, Buffer>> {
-  const zipFS = new ZipFileSystem(file);
+  const zipFS = new ZipFile(file);
   const resources = new Map<string, Buffer>();
   for (const entry of await zipFS.entries())
     resources.set(entry, await zipFS.read(entry));

tests/config/vfs.ts

@@ -1,124 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import path from 'path';
-import fs from 'fs';
-import type stream from 'stream';
-import yauzl from 'yauzl';
-
-export interface VirtualFileSystem {
-  entries(): Promise<string[]>;
-  read(entry: string): Promise<Buffer>;
-  readStream(entryPath: string): Promise<stream.Readable>;
-  close(): void;
-}
-
-abstract class BaseFileSystem {
-
-  abstract readStream(entryPath: string): Promise<stream.Readable>;
-
-  async read(entryPath: string): Promise<Buffer> {
-    const readStream = await this.readStream(entryPath);
-    const buffers: Buffer[] = [];
-    return new Promise(f => {
-      readStream.on('data', d => buffers.push(d));
-      readStream.on('end', () => f(Buffer.concat(buffers)));
-    });
-  }
-
-  close() {
-  }
-}
-
-export class RealFileSystem extends BaseFileSystem implements VirtualFileSystem {
-  private _folder: string;
-
-  constructor(folder: string) {
-    super();
-    this._folder = folder;
-  }
-
-  async entries(): Promise<string[]> {
-    const result: string[] = [];
-    const visit = (dir: string) => {
-      for (const name of fs.readdirSync(dir)) {
-        const fqn = path.join(dir, name);
-        if (fs.statSync(fqn).isDirectory())
-          visit(fqn);
-        if (fs.statSync(fqn).isFile())
-          result.push(fqn);
-      }
-    };
-    visit(this._folder);
-    return result;
-  }
-
-  async readStream(entry: string): Promise<stream.Readable> {
-    return fs.createReadStream(path.join(this._folder, ...entry.split('/')));
-  }
-}
-
-export class ZipFileSystem extends BaseFileSystem implements VirtualFileSystem {
-  private _fileName: string;
-  private _zipFile: yauzl.ZipFile | undefined;
-  private _entries = new Map<string, yauzl.Entry>();
-  private _openedPromise: Promise<void>;
-
-  constructor(fileName: string) {
-    super();
-    this._fileName = fileName;
-    this._openedPromise = this.open();
-  }
-
-  async open() {
-    await new Promise<yauzl.ZipFile>((fulfill, reject) => {
-      yauzl.open(this._fileName, { autoClose: false }, (e, z) => {
-        if (e) {
-          reject(e);
-          return;
-        }
-        this._zipFile = z;
-        this._zipFile!.on('entry', (entry: yauzl.Entry) => {
-          this._entries.set(entry.fileName, entry);
-        });
-        this._zipFile!.on('end', fulfill);
-      });
-    });
-  }
-
-  async entries(): Promise<string[]> {
-    await this._openedPromise;
-    return [...this._entries.keys()];
-  }
-
-  async readStream(entryPath: string): Promise<stream.Readable> {
-    await this._openedPromise;
-    const entry = this._entries.get(entryPath)!;
-    return new Promise((f, r) => {
-      this._zipFile!.openReadStream(entry, (error, readStream) => {
-        if (error || !readStream) {
-          r(error || 'Entry not found');
-          return;
-        }
-        f(readStream);
-      });
-    });
-  }
-
-  override close() {
-    this._zipFile?.close();
-  }
-}

tests/playwright-test/playwright.trace.spec.ts

@@ -15,7 +15,7 @@
  */
 
 import { test, expect } from './playwright-test-fixtures';
-import { ZipFileSystem } from '../config/vfs';
+import { ZipFile } from '../../packages/playwright-core/lib/utils/zipFile';
 import fs from 'fs';
 
 test('should stop tracing with trace: on-first-retry, when not retrying', async ({ runInlineTest }, testInfo) => {
@@ -243,7 +243,7 @@ test('should not override trace file in afterAll', async ({ runInlineTest, serve
 
 
 async function parseTrace(file: string): Promise<Map<string, Buffer>> {
-  const zipFS = new ZipFileSystem(file);
+  const zipFS = new ZipFile(file);
   const resources = new Map<string, Buffer>();
   for (const entry of await zipFS.entries())
     resources.set(entry, await zipFS.read(entry));