feat(route): fulfill from har (#14720)

feat(route): fulfill from har

This allows to use pre-recorded HAR file to fulfill routes.

docs/src/api/class-route.md

@@ -216,6 +216,12 @@ Optional response body as text.
 
 Optional response body as raw bytes.
 
+### option: Route.fulfill.har
+- `har` <[path]>
+
+HAR file to extract the response from. If HAR file contains an entry with the matching the url, its headers, status and body will be used. Individual fields such as headers can be overridden using fulfill options. If matching entry is not found, this method will throw.
+If `har` is a relative path, then it is resolved relative to the current working directory.
+
 ### option: Route.fulfill.path
 - `path` <[path]>
 

packages/playwright-core/src/client/browserContext.ts

@@ -337,6 +337,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel>
     if (this._browser)
       this._browser._contexts.delete(this);
     this._browserType?._contexts?.delete(this);
+    this._connection.localUtils()._channel.harClearCache({ cacheKey: this._guid }).catch(() => {});
     this.emit(Events.BrowserContext.Close, this);
   }
 

packages/playwright-core/src/client/localUtils.ts

@@ -21,8 +21,4 @@ export class LocalUtils extends ChannelOwner<channels.LocalUtilsChannel> {
   constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.LocalUtilsInitializer) {
     super(parent, type, guid, initializer);
   }
-
-  async zip(zipFile: string, entries: channels.NameValue[]): Promise<void> {
-    await this._channel.zip({ zipFile, entries });
-  }
 }

packages/playwright-core/src/client/network.ts

@@ -21,7 +21,7 @@ import { Frame } from './frame';
 import type { Headers, RemoteAddr, SecurityDetails, WaitForEventOptions } from './types';
 import fs from 'fs';
 import { mime } from '../utilsBundle';
-import { isString, headersObjectToArray } from '../utils';
+import { isString, headersObjectToArray, headersArrayToObject } from '../utils';
 import { ManualPromise } from '../utils/manualPromise';
 import { Events } from './events';
 import type { Page } from './page';
@@ -140,6 +140,11 @@ export class Request extends ChannelOwner<channels.RequestChannel> implements ap
     return this._provisionalHeaders.headers();
   }
 
+  _context() {
+    // TODO: make sure this works for service worker requests.
+    return this.frame().page().context();
+  }
+
   _actualHeaders(): Promise<RawHeaders> {
     if (!this._actualHeadersPromise) {
       this._actualHeadersPromise = this._wrapApiCall(async () => {
@@ -239,13 +244,34 @@ export class Route extends ChannelOwner<channels.RouteChannel> implements api.Ro
     await this._raceWithPageClose(this._channel.abort({ errorCode }));
   }
 
-  async fulfill(options: { response?: api.APIResponse, status?: number, headers?: Headers, contentType?: string, body?: string | Buffer, path?: string } = {}) {
+  async fulfill(options: { response?: api.APIResponse, status?: number, headers?: Headers, contentType?: string, body?: string | Buffer, path?: string, har?: string } = {}) {
     let fetchResponseUid;
     let { status: statusOption, headers: headersOption, body } = options;
+
+    if (options.har && options.response)
+      throw new Error(`At most one of "har" and "response" options should be present`);
+
+    if (options.har) {
+      const entry = await this._connection.localUtils()._channel.harFindEntry({
+        cacheKey: this.request()._context()._guid,
+        harFile: options.har,
+        url: this.request().url(),
+        needBody: body === undefined,
+      });
+      if (entry.error)
+        throw new Error(entry.error);
+      if (statusOption === undefined)
+        statusOption = entry.status;
+      if (headersOption === undefined && entry.headers)
+        headersOption = headersArrayToObject(entry.headers, false);
+      if (body === undefined && entry.body !== undefined)
+        body = Buffer.from(entry.body, 'base64');
+    }
+
     if (options.response) {
-      statusOption ||= options.response.status();
-      headersOption ||= options.response.headers();
-      if (options.body === undefined && options.path === undefined && options.response instanceof APIResponse) {
+      statusOption ??= options.response.status();
+      headersOption ??= options.response.headers();
+      if (body === undefined && options.path === undefined && options.response instanceof APIResponse) {
         if (options.response._request._connection === this._connection)
           fetchResponseUid = (options.response as APIResponse)._fetchUid();
         else

packages/playwright-core/src/client/tracing.ts

@@ -78,6 +78,6 @@ export class Tracing extends ChannelOwner<channels.TracingChannel> implements ap
 
     // Add local sources to the remote trace if necessary.
     if (result.sourceEntries?.length)
-      await this._connection.localUtils().zip(filePath, result.sourceEntries);
+      await this._connection.localUtils()._channel.zip({ zipFile: filePath, entries: result.sourceEntries });
   }
 }

packages/playwright-core/src/protocol/channels.ts

@@ -378,6 +378,8 @@ export interface LocalUtilsEventTarget {
 export interface LocalUtilsChannel extends LocalUtilsEventTarget, Channel {
   _type_LocalUtils: boolean;
   zip(params: LocalUtilsZipParams, metadata?: Metadata): Promise<LocalUtilsZipResult>;
+  harFindEntry(params: LocalUtilsHarFindEntryParams, metadata?: Metadata): Promise<LocalUtilsHarFindEntryResult>;
+  harClearCache(params: LocalUtilsHarClearCacheParams, metadata?: Metadata): Promise<LocalUtilsHarClearCacheResult>;
 }
 export type LocalUtilsZipParams = {
   zipFile: string,
@@ -387,6 +389,28 @@ export type LocalUtilsZipOptions = {
 
 };
 export type LocalUtilsZipResult = void;
+export type LocalUtilsHarFindEntryParams = {
+  cacheKey: string,
+  harFile: string,
+  url: string,
+  needBody: boolean,
+};
+export type LocalUtilsHarFindEntryOptions = {
+
+};
+export type LocalUtilsHarFindEntryResult = {
+  error?: string,
+  status?: number,
+  headers?: NameValue[],
+  body?: Binary,
+};
+export type LocalUtilsHarClearCacheParams = {
+  cacheKey: string,
+};
+export type LocalUtilsHarClearCacheOptions = {
+
+};
+export type LocalUtilsHarClearCacheResult = void;
 
 export interface LocalUtilsEvents {
 }

packages/playwright-core/src/protocol/protocol.yml

@@ -473,6 +473,26 @@ LocalUtils:
           type: array
           items: NameValue
 
+    harFindEntry:
+      parameters:
+        # HAR file is cached until clearHarCache is called
+        cacheKey: string
+        harFile: string
+        url: string
+        needBody: boolean
+      returns:
+        error: string?
+        status: number?
+        headers:
+          type: array?
+          items: NameValue
+        body: binary?
+
+    harClearCache:
+      parameters:
+        cacheKey: string
+
+
 Root:
   type: interface
 

packages/playwright-core/src/protocol/validator.ts

@@ -205,6 +205,15 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     zipFile: tString,
     entries: tArray(tType('NameValue')),
   });
+  scheme.LocalUtilsHarFindEntryParams = tObject({
+    cacheKey: tString,
+    harFile: tString,
+    url: tString,
+    needBody: tBoolean,
+  });
+  scheme.LocalUtilsHarClearCacheParams = tObject({
+    cacheKey: tString,
+  });
   scheme.RootInitializeParams = tObject({
     sdkLanguage: tString,
   });

packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts

@@ -23,9 +23,12 @@ import { assert, createGuid } from '../../utils';
 import type { DispatcherScope } from './dispatcher';
 import { Dispatcher } from './dispatcher';
 import { yazl, yauzl } from '../../zipBundle';
+import type { Log } from '../har/har';
 
 export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.LocalUtilsChannel> implements channels.LocalUtilsChannel {
   _type_LocalUtils: boolean;
+  private _harCache = new Map<string, Map<string, Log>>();
+
   constructor(scope: DispatcherScope) {
     super(scope, { guid: 'localUtils@' + createGuid() }, 'LocalUtils', {});
     this._type_LocalUtils = true;
@@ -85,4 +88,39 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.
     });
     return promise;
   }
+
+  async harFindEntry(params: channels.LocalUtilsHarFindEntryParams, metadata?: channels.Metadata): Promise<channels.LocalUtilsHarFindEntryResult> {
+    try {
+      let cache = this._harCache.get(params.cacheKey);
+      if (!cache) {
+        cache = new Map();
+        this._harCache.set(params.cacheKey, cache);
+      }
+
+      let harLog = cache.get(params.harFile);
+      if (!harLog) {
+        const contents = await fs.promises.readFile(params.harFile, 'utf-8');
+        harLog = JSON.parse(contents).log as Log;
+        cache.set(params.harFile, harLog);
+      }
+
+      const entry = harLog.entries.find(entry => entry.request.url === params.url);
+      if (!entry)
+        throw new Error(`No entry matching ${params.url}`);
+      let base64body: string | undefined;
+      if (params.needBody && entry.response.content && entry.response.content.text !== undefined) {
+        if (entry.response.content.encoding === 'base64')
+          base64body = entry.response.content.text;
+        else
+          base64body = Buffer.from(entry.response.content.text, 'utf8').toString('base64');
+      }
+      return { status: entry.response.status, headers: entry.response.headers, body: base64body };
+    } catch (e) {
+      return { error: `Error reading HAR file ${params.harFile}: ` + e.message };
+    }
+  }
+
+  async harClearCache(params: channels.LocalUtilsHarClearCacheParams, metadata?: channels.Metadata): Promise<void> {
+    this._harCache.delete(params.cacheKey);
+  }
 }

packages/playwright-core/types/types.d.ts

@@ -14867,6 +14867,14 @@ export interface Route {
      */
     contentType?: string;
 
+    /**
+     * HAR file to extract the response from. If HAR file contains an entry with the matching the url, its headers, status and
+     * body will be used. Individual fields such as headers can be overridden using fulfill options. If matching entry is not
+     * found, this method will throw. If `har` is a relative path, then it is resolved relative to the current working
+     * directory.
+     */
+    har?: string;
+
     /**
      * Response headers. Header values will be converted to a string.
      */

tests/library/har.spec.ts

@@ -287,6 +287,47 @@ it('should filter by regexp', async ({ contextFactory, server }, testInfo) => {
   expect(log.entries[0].request.url.endsWith('har.html')).toBe(true);
 });
 
+it('should fulfill route from har', async ({ contextFactory, server }, testInfo) => {
+  const kCustomCSS = 'body { background-color: rgb(50, 100, 150); }';
+
+  const harPath = testInfo.outputPath('test.har');
+  const harContext = await contextFactory({ baseURL: server.PREFIX, recordHar: { path: harPath, urlFilter: '/*.css' }, ignoreHTTPSErrors: true });
+  const harPage = await harContext.newPage();
+  await harPage.route('**/one-style.css', async route => {
+    // Make sure har content is not what the server returns.
+    await route.fulfill({ body: kCustomCSS });
+  });
+  await harPage.goto('/har.html');
+  await harContext.close();
+
+  const context  = await contextFactory();
+  const page1 = await context.newPage();
+  await page1.route('**/*.css', async route => {
+    // Fulfulling from har should give expected CSS.
+    await route.fulfill({ har: harPath });
+  });
+  const [response1] = await Promise.all([
+    page1.waitForResponse('**/one-style.css'),
+    page1.goto(server.PREFIX + '/one-style.html'),
+  ]);
+  expect(await response1.text()).toBe(kCustomCSS);
+  await expect(page1.locator('body')).toHaveCSS('background-color', 'rgb(50, 100, 150)');
+  await page1.close();
+
+  const page2 = await context.newPage();
+  await page2.route('**/*.css', async route => {
+    // Overriding status should make CSS not apply.
+    await route.fulfill({ har: harPath, status: 404 });
+  });
+  const [response2] = await Promise.all([
+    page2.waitForResponse('**/one-style.css'),
+    page2.goto(server.PREFIX + '/one-style.html'),
+  ]);
+  expect(response2.status()).toBe(404);
+  await expect(page2.locator('body')).toHaveCSS('background-color', 'rgba(0, 0, 0, 0)');
+  await page2.close();
+});
+
 it('should include sizes', async ({ contextFactory, server, asset }, testInfo) => {
   const { page, getLog } = await pageWithHar(contextFactory, testInfo);
   await page.goto(server.PREFIX + '/har.html');

tests/page/page-request-fulfill.spec.ts

@@ -321,3 +321,38 @@ it('headerValue should return set-cookie from intercepted response', async ({ pa
   const response = await page.goto(server.EMPTY_PAGE);
   expect(await response.headerValue('Set-Cookie')).toBe('a=b');
 });
+
+it('should complain about bad har', async ({ page, server }, testInfo) => {
+  const harPath = testInfo.outputPath('test.har');
+  fs.writeFileSync(harPath, JSON.stringify({ log: {} }), 'utf-8');
+  let error;
+  await page.route('**/*.css', async route => {
+    error = await route.fulfill({ har: harPath }).catch(e => e);
+    await route.continue();
+  });
+  await page.goto(server.PREFIX + '/one-style.html');
+  expect(error.message).toContain(`Error reading HAR file ${harPath}: Cannot read`);
+});
+
+it('should complain about no entry found in har', async ({ page, server }, testInfo) => {
+  const harPath = testInfo.outputPath('test.har');
+  fs.writeFileSync(harPath, JSON.stringify({ log: { entries: [] } }), 'utf-8');
+  let error;
+  await page.route('**/*.css', async route => {
+    error = await route.fulfill({ har: harPath }).catch(e => e);
+    await route.continue();
+  });
+  await page.goto(server.PREFIX + '/one-style.html');
+  expect(error.message).toBe(`Error reading HAR file ${harPath}: No entry matching ${server.PREFIX + '/one-style.css'}`);
+});
+
+it('should complain about har + response options', async ({ page, server }, testInfo) => {
+  let error;
+  await page.route('**/*.css', async route => {
+    const response = await page.request.fetch(route.request());
+    error = await route.fulfill({ har: 'har', response }).catch(e => e);
+    await route.continue();
+  });
+  await page.goto(server.PREFIX + '/one-style.html');
+  expect(error.message).toBe(`At most one of "har" and "response" options should be present`);
+});