From 030e7d211cd0215c24c208713f4e71814d229335 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 17 Jun 2022 16:11:22 -0700 Subject: [PATCH] chore(har): allow replaying from zip har (#14962) --- .../playwright-core/src/client/harRouter.ts | 87 ++++++++---- packages/playwright-core/src/utils/DEPS.list | 1 + packages/playwright-core/src/utils/zipFile.ts | 75 +++++++++++ packages/playwright-core/src/zipBundle.ts | 1 + tests/config/utils.ts | 6 +- tests/config/vfs.ts | 124 ------------------ .../playwright-test/playwright.trace.spec.ts | 4 +- 7 files changed, 146 insertions(+), 152 deletions(-) create mode 100644 packages/playwright-core/src/utils/zipFile.ts delete mode 100644 tests/config/vfs.ts diff --git a/packages/playwright-core/src/client/harRouter.ts b/packages/playwright-core/src/client/harRouter.ts index 0c37b94f90..2acbc5caae 100644 --- a/packages/playwright-core/src/client/harRouter.ts +++ b/packages/playwright-core/src/client/harRouter.ts @@ -18,7 +18,9 @@ import fs from 'fs'; import type { HAREntry, HARFile, HARResponse } from '../../types/types'; import { debugLogger } from '../common/debugLogger'; import { rewriteErrorMessage } from '../utils/stackTrace'; +import { ZipFile } from '../utils/zipFile'; import type { BrowserContext } from './browserContext'; +import { Events } from './events'; import type { Route } from './network'; import type { BrowserContextOptions } from './types'; @@ -26,40 +28,79 @@ type HarOptions = NonNullable; export class HarRouter { private _pattern: string | RegExp; - private _handler: (route: Route) => Promise; + private _harFile: HARFile; + private _zipFile: ZipFile | null; + private _options: HarOptions | undefined; static async create(options: HarOptions): Promise { + if (options.path.endsWith('.zip')) { + const zipFile = new ZipFile(options.path); + const har = await zipFile.read('har.har'); + const harFile = JSON.parse(har.toString()) as HARFile; + return new HarRouter(harFile, zipFile, options); + } const harFile = JSON.parse(await fs.promises.readFile(options.path, 'utf-8')) as HARFile; - return new HarRouter(harFile, options); + return new HarRouter(harFile, null, options); } - constructor(harFile: HARFile, options?: HarOptions) { + constructor(harFile: HARFile, zipFile: ZipFile | null, options?: HarOptions) { + this._harFile = harFile; + this._zipFile = zipFile; this._pattern = options?.urlFilter ?? /.*/; - this._handler = async (route: Route) => { - let response; - try { - response = harFindResponse(harFile, { - url: route.request().url(), - method: route.request().method() + this._options = options; + } + + private async _handle(route: Route) { + let response; + try { + response = harFindResponse(this._harFile, { + url: route.request().url(), + method: route.request().method() + }); + } catch (e) { + rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`); + debugLogger.log('api', e); + } + + if (response) { + debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`); + const sha1 = (response.content as any)._sha1; + + if (this._zipFile && sha1) { + const body = await this._zipFile.read(sha1).catch(() => { + debugLogger.log('api', `payload ${sha1} for request ${route.request().url()} is not found in archive`); + return null; }); - } catch (e) { - rewriteErrorMessage(e, `Error while finding entry for ${route.request().method()} ${route.request().url()} in HAR file:\n${e.message}`); - debugLogger.log('api', e); + if (body) { + await route.fulfill({ + status: response.status, + headers: Object.fromEntries(response.headers.map(h => [h.name, h.value])), + body + }); + return; + } } - if (response) { - debugLogger.log('api', `serving from HAR: ${route.request().method()} ${route.request().url()}`); - await route.fulfill({ response }); - } else if (options?.fallback === 'continue') { - await route.fallback(); - } else { - debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`); - await route.abort(); - } - }; + + await route.fulfill({ response }); + return; + } + + if (this._options?.fallback === 'continue') { + await route.fallback(); + return; + } + + debugLogger.log('api', `request not in HAR, aborting: ${route.request().method()} ${route.request().url()}`); + await route.abort(); } async addRoute(context: BrowserContext) { - await context.route(this._pattern, this._handler); + await context.route(this._pattern, route => this._handle(route)); + context.once(Events.BrowserContext.Close, () => this.dispose()); + } + + dispose() { + this._zipFile?.close(); } } diff --git a/packages/playwright-core/src/utils/DEPS.list b/packages/playwright-core/src/utils/DEPS.list index 25592671eb..e97f2d3792 100644 --- a/packages/playwright-core/src/utils/DEPS.list +++ b/packages/playwright-core/src/utils/DEPS.list @@ -3,3 +3,4 @@ ../third_party/diff_match_patch ../third_party/pixelmatch ../utilsBundle.ts +../zipBundle.ts diff --git a/packages/playwright-core/src/utils/zipFile.ts b/packages/playwright-core/src/utils/zipFile.ts new file mode 100644 index 0000000000..3ae68b1650 --- /dev/null +++ b/packages/playwright-core/src/utils/zipFile.ts @@ -0,0 +1,75 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { yauzl } from '../zipBundle'; +import type { UnzipFile, Entry } from '../zipBundle'; + +export class ZipFile { + private _fileName: string; + private _zipFile: UnzipFile | undefined; + private _entries = new Map(); + private _openedPromise: Promise; + + constructor(fileName: string) { + this._fileName = fileName; + this._openedPromise = this._open(); + } + + private async _open() { + await new Promise((fulfill, reject) => { + yauzl.open(this._fileName, { autoClose: false }, (e, z) => { + if (e) { + reject(e); + return; + } + this._zipFile = z; + this._zipFile!.on('entry', (entry: Entry) => { + this._entries.set(entry.fileName, entry); + }); + this._zipFile!.on('end', fulfill); + }); + }); + } + + async entries(): Promise { + await this._openedPromise; + return [...this._entries.keys()]; + } + + async read(entryPath: string): Promise { + await this._openedPromise; + const entry = this._entries.get(entryPath)!; + if (!entry) + throw new Error(`${entryPath} not found in file ${this._fileName}`); + + return new Promise((resolve, reject) => { + this._zipFile!.openReadStream(entry, (error, readStream) => { + if (error || !readStream) { + reject(error || 'Entry not found'); + return; + } + + const buffers: Buffer[] = []; + readStream.on('data', data => buffers.push(data)); + readStream.on('end', () => resolve(Buffer.concat(buffers))); + }); + }); + } + + close() { + this._zipFile?.close(); + } +} diff --git a/packages/playwright-core/src/zipBundle.ts b/packages/playwright-core/src/zipBundle.ts index b396c7fca8..9c275873a8 100644 --- a/packages/playwright-core/src/zipBundle.ts +++ b/packages/playwright-core/src/zipBundle.ts @@ -17,4 +17,5 @@ export const yazl: typeof import('../bundles/zip/node_modules/@types/yazl') = require('./zipBundleImpl').yazl; export type { ZipFile } from '../bundles/zip/node_modules/@types/yazl'; export const yauzl: typeof import('../bundles/zip/node_modules/@types/yauzl') = require('./zipBundleImpl').yauzl; +export type { ZipFile as UnzipFile, Entry } from '../bundles/zip/node_modules/@types/yauzl'; export const extract: typeof import('../bundles/zip/node_modules/extract-zip') = require('./zipBundleImpl').extract; diff --git a/tests/config/utils.ts b/tests/config/utils.ts index 145ed6d2c1..53e8176186 100644 --- a/tests/config/utils.ts +++ b/tests/config/utils.ts @@ -16,7 +16,7 @@ import { expect } from '@playwright/test'; import type { Frame, Page } from 'playwright-core'; -import { ZipFileSystem } from './vfs'; +import { ZipFile } from '../../packages/playwright-core/lib/utils/zipFile'; export async function attachFrame(page: Page, frameId: string, url: string): Promise { const handle = await page.evaluateHandle(async ({ frameId, url }) => { @@ -91,7 +91,7 @@ export function suppressCertificateWarning() { } export async function parseTrace(file: string): Promise<{ events: any[], resources: Map }> { - const zipFS = new ZipFileSystem(file); + const zipFS = new ZipFile(file); const resources = new Map(); for (const entry of await zipFS.entries()) resources.set(entry, await zipFS.read(entry)); @@ -113,7 +113,7 @@ export async function parseTrace(file: string): Promise<{ events: any[], resourc } export async function parseHar(file: string): Promise> { - const zipFS = new ZipFileSystem(file); + const zipFS = new ZipFile(file); const resources = new Map(); for (const entry of await zipFS.entries()) resources.set(entry, await zipFS.read(entry)); diff --git a/tests/config/vfs.ts b/tests/config/vfs.ts deleted file mode 100644 index f0a4241a36..0000000000 --- a/tests/config/vfs.ts +++ /dev/null @@ -1,124 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import path from 'path'; -import fs from 'fs'; -import type stream from 'stream'; -import yauzl from 'yauzl'; - -export interface VirtualFileSystem { - entries(): Promise; - read(entry: string): Promise; - readStream(entryPath: string): Promise; - close(): void; -} - -abstract class BaseFileSystem { - - abstract readStream(entryPath: string): Promise; - - async read(entryPath: string): Promise { - const readStream = await this.readStream(entryPath); - const buffers: Buffer[] = []; - return new Promise(f => { - readStream.on('data', d => buffers.push(d)); - readStream.on('end', () => f(Buffer.concat(buffers))); - }); - } - - close() { - } -} - -export class RealFileSystem extends BaseFileSystem implements VirtualFileSystem { - private _folder: string; - - constructor(folder: string) { - super(); - this._folder = folder; - } - - async entries(): Promise { - const result: string[] = []; - const visit = (dir: string) => { - for (const name of fs.readdirSync(dir)) { - const fqn = path.join(dir, name); - if (fs.statSync(fqn).isDirectory()) - visit(fqn); - if (fs.statSync(fqn).isFile()) - result.push(fqn); - } - }; - visit(this._folder); - return result; - } - - async readStream(entry: string): Promise { - return fs.createReadStream(path.join(this._folder, ...entry.split('/'))); - } -} - -export class ZipFileSystem extends BaseFileSystem implements VirtualFileSystem { - private _fileName: string; - private _zipFile: yauzl.ZipFile | undefined; - private _entries = new Map(); - private _openedPromise: Promise; - - constructor(fileName: string) { - super(); - this._fileName = fileName; - this._openedPromise = this.open(); - } - - async open() { - await new Promise((fulfill, reject) => { - yauzl.open(this._fileName, { autoClose: false }, (e, z) => { - if (e) { - reject(e); - return; - } - this._zipFile = z; - this._zipFile!.on('entry', (entry: yauzl.Entry) => { - this._entries.set(entry.fileName, entry); - }); - this._zipFile!.on('end', fulfill); - }); - }); - } - - async entries(): Promise { - await this._openedPromise; - return [...this._entries.keys()]; - } - - async readStream(entryPath: string): Promise { - await this._openedPromise; - const entry = this._entries.get(entryPath)!; - return new Promise((f, r) => { - this._zipFile!.openReadStream(entry, (error, readStream) => { - if (error || !readStream) { - r(error || 'Entry not found'); - return; - } - f(readStream); - }); - }); - } - - override close() { - this._zipFile?.close(); - } -} diff --git a/tests/playwright-test/playwright.trace.spec.ts b/tests/playwright-test/playwright.trace.spec.ts index adcc69deb3..417bd4090a 100644 --- a/tests/playwright-test/playwright.trace.spec.ts +++ b/tests/playwright-test/playwright.trace.spec.ts @@ -15,7 +15,7 @@ */ import { test, expect } from './playwright-test-fixtures'; -import { ZipFileSystem } from '../config/vfs'; +import { ZipFile } from '../../packages/playwright-core/lib/utils/zipFile'; import fs from 'fs'; test('should stop tracing with trace: on-first-retry, when not retrying', async ({ runInlineTest }, testInfo) => { @@ -243,7 +243,7 @@ test('should not override trace file in afterAll', async ({ runInlineTest, serve async function parseTrace(file: string): Promise> { - const zipFS = new ZipFileSystem(file); + const zipFS = new ZipFile(file); const resources = new Map(); for (const entry of await zipFS.entries()) resources.set(entry, await zipFS.read(entry));