feat(har): disambiguate requests by post data (#14993)

packages/playwright-core/src/cli/cli.ts

@@ -471,6 +471,7 @@ async function launchContext(options: Options, headless: boolean, executablePath
     contextOptions.recordHar = { path: path.resolve(process.cwd(), options.saveHar) };
     if (options.saveHarGlob)
       contextOptions.recordHar.urlFilter = options.saveHarGlob;
+    contextOptions.serviceWorkers = 'block';
   }
 
   // Close app when the last window closes.

packages/playwright-core/src/client/harRouter.ts

@@ -44,11 +44,14 @@ export class HarRouter {
   }
 
   private async _handle(route: Route) {
+    const request = route.request();
     const response = await this._localUtils._channel.harLookup({
       harId: this._harId,
-      url: route.request().url(),
+      url: request.url(),
-      method: route.request().method(),
+      method: request.method(),
-      isNavigationRequest: route.request().isNavigationRequest()
+      headers: (await request.headersArray()),
+      postData: request.postDataBuffer()?.toString('base64'),
+      isNavigationRequest: request.isNavigationRequest()
     });
 
     if (response.action === 'redirect') {
@@ -61,7 +64,7 @@ export class HarRouter {
       await route.fulfill({
         status: response.status,
         headers: Object.fromEntries(response.headers!.map(h => [h.name, h.value])),
-        body: response.base64Encoded ? Buffer.from(response.body!, 'base64') : response.body
+        body: Buffer.from(response.body!, 'base64')
       });
       return;
     }

packages/playwright-core/src/protocol/channels.ts

@@ -404,10 +404,12 @@ export type LocalUtilsHarLookupParams = {
   harId: string,
   url: string,
   method: string,
+  headers: NameValue[],
+  postData?: string,
   isNavigationRequest: boolean,
 };
 export type LocalUtilsHarLookupOptions = {
-
+  postData?: string,
 };
 export type LocalUtilsHarLookupResult = {
   action: 'error' | 'redirect' | 'fulfill' | 'noentry',
@@ -416,7 +418,6 @@ export type LocalUtilsHarLookupResult = {
   status?: number,
   headers?: NameValue[],
   body?: string,
-  base64Encoded?: boolean,
 };
 export type LocalUtilsHarCloseParams = {
   harId: string,

packages/playwright-core/src/protocol/protocol.yml

@@ -490,6 +490,10 @@ LocalUtils:
         harId: string
         url: string
         method: string
+        headers:
+          type: array
+          items: NameValue
+        postData: string?
         isNavigationRequest: boolean
       returns:
         action:
@@ -506,7 +510,6 @@ LocalUtils:
           type: array?
           items: NameValue
         body: string?
-        base64Encoded: boolean?
 
     harClose:
       parameters:

packages/playwright-core/src/protocol/validator.ts

@@ -212,6 +212,8 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     harId: tString,
     url: tString,
     method: tString,
+    headers: tArray(tType('NameValue')),
+    postData: tOptional(tString),
     isNavigationRequest: tBoolean,
   });
   scheme.LocalUtilsHarCloseParams = tObject({

packages/playwright-core/src/server/dispatchers/localUtilsDispatcher.ts

@@ -114,7 +114,7 @@ export class LocalUtilsDispatcher extends Dispatcher<{ guid: string }, channels.
     const harBackend = this._harBakends.get(params.harId);
     if (!harBackend)
       return { action: 'error', message: `Internal error: har was not opened` };
-    return await harBackend.lookup(params.url, params.method, params.isNavigationRequest);
+    return await harBackend.lookup(params.url, params.method, params.headers, params.postData ? Buffer.from(params.postData, 'base64') : undefined, params.isNavigationRequest);
   }
 
   async harClose(params: channels.LocalUtilsHarCloseParams, metadata?: channels.Metadata): Promise<void> {
@@ -140,7 +140,7 @@ class HarBackend {
     this._zipFile = zipFile;
   }
 
-  async lookup(url: string, method: string, isNavigationRequest: boolean): Promise<{
+  async lookup(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined, isNavigationRequest: boolean): Promise<{
       action: 'error' | 'redirect' | 'fulfill' | 'noentry',
       message?: string,
       redirectURL?: string,
@@ -150,7 +150,7 @@ class HarBackend {
       base64Encoded?: boolean }> {
     let entry;
     try {
-      entry = this._harFindResponse(url, method);
+      entry = await this._harFindResponse(url, method, headers, postData);
     } catch (e) {
       return { action: 'error', message: 'HAR error: ' + e.message };
     }
@@ -163,45 +163,72 @@ class HarBackend {
       return { action: 'redirect', redirectURL: entry.request.url };
 
     const response = entry.response;
-    const sha1 = (response.content as any)._sha1;
-    let body = response.content.text;
-    let base64Encoded = response.content.encoding === 'base64';
-
-    if (sha1) {
-      let buffer: Buffer;
     try {
-        if (this._zipFile)
+      const buffer = await this._loadContent(response.content);
-          buffer = await this._zipFile.read(sha1);
-        else
-          buffer = await fs.promises.readFile(path.resolve(this._baseDir!, sha1));
-      } catch (e) {
-        return { action: 'error', message: e.message };
-      }
-
-      body = buffer.toString('base64');
-      base64Encoded = true;
-    }
-
       return {
         action: 'fulfill',
         status: response.status,
         headers: response.headers,
-      body,
+        body: buffer.toString('base64'),
-      base64Encoded
       };
+    } catch (e) {
+      return { action: 'error', message: e.message };
+    }
   }
 
-  private _harFindResponse(url: string, method: string): HAREntry | undefined {
+  private async _loadContent(content: { text?: string, encoding?: string, _sha1?: string }): Promise<Buffer> {
+    const sha1 = content._sha1;
+    let buffer: Buffer;
+    if (sha1) {
+      if (this._zipFile)
+        buffer = await this._zipFile.read(sha1);
+      else
+        buffer = await fs.promises.readFile(path.resolve(this._baseDir!, sha1));
+    } else {
+      buffer = Buffer.from(content.text || '', content.encoding === 'base64' ? 'base64' : 'utf-8');
+    }
+    return buffer;
+  }
+
+  private async _harFindResponse(url: string, method: string, headers: HeadersArray, postData: Buffer | undefined): Promise<HAREntry | undefined> {
     const harLog = this._harFile.log;
     const visited = new Set<HAREntry>();
     while (true) {
-      const entry = harLog.entries.find(entry => entry.request.url === url && entry.request.method === method);
+      const entries = harLog.entries.filter(entry => entry.request.url === url && entry.request.method === method);
-      if (!entry)
+      if (!entries.length)
         return;
+
+      let entry: HAREntry | undefined;
+
+      if (entries.length > 1) {
+        // Disambiguating requests
+
+        // 1. Disambiguate by postData - this covers GraphQL
+        if (!entry && postData) {
+          for (const candidate of entries) {
+            if (!candidate.request.postData)
+              continue;
+            const buffer = await this._loadContent(candidate.request.postData);
+            if (buffer.equals(postData)) {
+              entry = candidate;
+              break;
+            }
+          }
+        }
+
+        // TODO: disambiguate by headers.
+      }
+
+      // Fall back to first entry.
+      if (!entry)
+        entry = entries[0];
+
       if (visited.has(entry))
         throw new Error(`Found redirect cycle for ${url}`);
+
       visited.add(entry);
 
+      // Follow redirects.
       const locationHeader = entry.response.headers.find(h => h.name.toLowerCase() === 'location');
       if (redirectStatus.includes(entry.response.status) && locationHeader) {
         const locationURL = new URL(locationHeader.value, url);