feat(tracing): trace context APIRequest calls (#10684)

packages/playwright-core/src/server/browserContext.ts

@@ -78,11 +78,12 @@ export abstract class BrowserContext extends SdkObject {
     // Create instrumentation per context.
     this.instrumentation = createInstrumentation();
 
+    this.fetchRequest = new BrowserContextAPIRequestContext(this);
+
     if (this._options.recordHar)
       this._harRecorder = new HarRecorder(this, { ...this._options.recordHar, path: path.join(this._browser.options.artifactsDir, `${createGuid()}.har`) });
 
     this.tracing = new Tracing(this);
-    this.fetchRequest = new BrowserContextAPIRequestContext(this);
   }
 
   isPersistentContext(): boolean {

packages/playwright-core/src/server/fetch.ts

@@ -44,9 +44,31 @@ type FetchRequestOptions = {
   baseURL?: string;
 };
 
+export type APIRequestEvent = {
+  url: URL,
+  method: string,
+  headers: { [name: string]: string },
+  cookies: types.NameValueList,
+  postData?: Buffer
+};
+
+export type APIRequestFinishedEvent = {
+  requestEvent: APIRequestEvent,
+  httpVersion: string;
+  headers: http.IncomingHttpHeaders;
+  cookies: types.NetworkCookie[];
+  rawHeaders: string[];
+  statusCode: number;
+  statusMessage: string;
+  body?: Buffer;
+};
+
 export abstract class APIRequestContext extends SdkObject {
   static Events = {
     Dispose: 'dispose',
+
+    Request: 'request',
+    RequestFinished: 'requestfinished',
   };
 
   readonly fetchResponses: Map<string, Buffer> = new Map();
@@ -166,7 +188,9 @@ export abstract class APIRequestContext extends SdkObject {
     return { ...fetchResponse, fetchUid };
   }
 
-  private async _updateCookiesFromHeader(responseUrl: string, setCookie: string[]) {
+  private _parseSetCookieHeader(responseUrl: string, setCookie: string[] | undefined): types.NetworkCookie[] {
+    if (!setCookie)
+      return [];
     const url = new URL(responseUrl);
     // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
     const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
@@ -188,8 +212,7 @@ export abstract class APIRequestContext extends SdkObject {
         cookie.path = defaultPath;
       cookies.push(cookie);
     }
-    if (cookies.length)
+    return cookies;
-      await this._addCookies(cookies);
   }
 
   private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) {
@@ -204,15 +227,43 @@ export abstract class APIRequestContext extends SdkObject {
 
   private async _sendRequest(progress: Progress, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.APIResponse>{
     await this._updateRequestCookieHeader(url, options);
+
+    const requestCookies = (options.headers!['cookie'] as (string | undefined))?.split(';').map(p => {
+      const [name, value] = p.split('=').map(v => v.trim());
+      return { name, value };
+    }) || [];
+    const requestEvent: APIRequestEvent = {
+      url,
+      method: options.method!,
+      headers: options.headers as { [name: string]: string },
+      cookies: requestCookies,
+      postData
+    };
+    this.emit(APIRequestContext.Events.Request, requestEvent);
+
     return new Promise<types.APIResponse>((fulfill, reject) => {
       const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
         = (url.protocol === 'https:' ? https : http).request;
       const request = requestConstructor(url, options, async response => {
+        const notifyRequestFinished = (body?: Buffer) => {
+          const requestFinishedEvent: APIRequestFinishedEvent = {
+            requestEvent,
+            statusCode: -1,
+            statusMessage: '',
+            ...response,
+            cookies,
+            body
+          };
+          this.emit(APIRequestContext.Events.RequestFinished, requestFinishedEvent);
+        };
         progress.log(`← ${response.statusCode} ${response.statusMessage}`);
         for (const [name, value] of Object.entries(response.headers))
           progress.log(`  ${name}: ${value}`);
-        if (response.headers['set-cookie'])
+
-          await this._updateCookiesFromHeader(response.url || url.toString(), response.headers['set-cookie']);
+        const cookies = this._parseSetCookieHeader(response.url || url.toString(), response.headers['set-cookie']) ;
+        if (cookies.length)
+          await this._addCookies(cookies);
+
         if (redirectStatus.includes(response.statusCode!)) {
           if (!options.maxRedirects) {
             reject(new Error('Max redirect count exceeded'));
@@ -251,6 +302,7 @@ export abstract class APIRequestContext extends SdkObject {
           // HTTP-redirect fetch step 4: If locationURL is null, then return response.
           if (response.headers.location) {
             const locationURL = new URL(response.headers.location, url);
+            notifyRequestFinished();
             fulfill(this._sendRequest(progress, locationURL, redirectOptions, postData));
             request.destroy();
             return;
@@ -263,6 +315,7 @@ export abstract class APIRequestContext extends SdkObject {
             const { username, password } = credentials;
             const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
             options.headers!['authorization'] = `Basic ${encoded}`;
+            notifyRequestFinished();
             fulfill(this._sendRequest(progress, url, options, postData));
             request.destroy();
             return;
@@ -294,6 +347,7 @@ export abstract class APIRequestContext extends SdkObject {
         body.on('data', chunk => chunks.push(chunk));
         body.on('end', () => {
           const body = Buffer.concat(chunks);
+          notifyRequestFinished(body);
           fulfill({
             url: response.url || url.toString(),
             status: response.statusCode || 0,

packages/playwright-core/src/server/supplements/har/harTracer.ts

@@ -15,6 +15,7 @@
  */
 
 import { BrowserContext } from '../../browserContext';
+import { APIRequestContext, APIRequestEvent, APIRequestFinishedEvent } from '../../fetch';
 import { helper } from '../../helper';
 import * as network from '../../network';
 import { Page } from '../../page';
@@ -64,10 +65,12 @@ export class HarTracer {
       eventsHelper.addEventListener(this._context, BrowserContext.Events.Request, (request: network.Request) => this._onRequest(request)),
       eventsHelper.addEventListener(this._context, BrowserContext.Events.RequestFinished, ({ request, response }) => this._onRequestFinished(request, response).catch(() => {})),
       eventsHelper.addEventListener(this._context, BrowserContext.Events.Response, (response: network.Response) => this._onResponse(response)),
+      eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.Request, (event: APIRequestEvent) => this._onAPIRequest(event)),
+      eventsHelper.addEventListener(this._context.fetchRequest, APIRequestContext.Events.RequestFinished, (event: APIRequestFinishedEvent) => this._onAPIRequestFinished(event)),
     ];
   }
 
-  private _entryForRequest(request: network.Request): har.Entry | undefined {
+  private _entryForRequest(request: network.Request | APIRequestEvent): har.Entry | undefined {
     return (request as any)[this._entrySymbol];
   }
 
@@ -132,6 +135,49 @@ export class HarTracer {
     this._barrierPromises.add(race);
   }
 
+  private _onAPIRequest(event: APIRequestEvent) {
+    const harEntry = createHarEntry(event.method, event.url, '', '');
+    harEntry.request.cookies = event.cookies;
+    harEntry.request.headers = Object.entries(event.headers).map(([name, value]) => ({ name, value }));
+    harEntry.request.postData = postDataForBuffer(event.postData || null, event.headers['content-type'],  this._options.content);
+    harEntry.request.bodySize = event.postData?.length || 0;
+    (event as any)[this._entrySymbol] = harEntry;
+    if (this._started)
+      this._delegate.onEntryStarted(harEntry);
+  }
+
+  private _onAPIRequestFinished(event: APIRequestFinishedEvent): void {
+    const harEntry = this._entryForRequest(event.requestEvent);
+    if (!harEntry)
+      return;
+
+    harEntry.response.status = event.statusCode;
+    harEntry.response.statusText = event.statusMessage;
+    harEntry.response.httpVersion = event.httpVersion;
+    harEntry.response.redirectURL = event.headers.location || '';
+    for (let i = 0; i < event.rawHeaders.length; i += 2) {
+      harEntry.response.headers.push({
+        name: event.rawHeaders[i],
+        value: event.rawHeaders[i + 1]
+      });
+    }
+    harEntry.response.cookies = event.cookies.map(c => {
+      return {
+        ...c,
+        expires: c.expires === -1 ? undefined : new Date(c.expires)
+      };
+    });
+
+    const content = harEntry.response.content;
+    const contentType = event.headers['content-type'];
+    if (contentType)
+      content.mimeType = contentType;
+    this._storeResponseContent(event.body, content);
+
+    if (this._started)
+      this._delegate.onEntryFinished(harEntry);
+  }
+
   private _onRequest(request: network.Request) {
     const page = request.frame()._page;
     const url = network.parsedURL(request.url());
@@ -139,49 +185,10 @@ export class HarTracer {
       return;
 
     const pageEntry = this._ensurePageEntry(page);
-    const harEntry: har.Entry = {
+    const harEntry = createHarEntry(request.method(), url, request.guid, request.frame().guid);
-      pageref: pageEntry.id,
+    harEntry.pageref = pageEntry.id;
-      _requestref: request.guid,
+    harEntry.request.postData = postDataForRequest(request, this._options.content);
-      _frameref: request.frame().guid,
+    harEntry.request.bodySize = request.bodySize();
-      _monotonicTime: monotonicTime(),
-      startedDateTime: new Date(),
-      time: -1,
-      request: {
-        method: request.method(),
-        url: request.url(),
-        httpVersion: FALLBACK_HTTP_VERSION,
-        cookies: [],
-        headers: [],
-        queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
-        postData: postDataForHar(request, this._options.content),
-        headersSize: -1,
-        bodySize: request.bodySize(),
-      },
-      response: {
-        status: -1,
-        statusText: '',
-        httpVersion: FALLBACK_HTTP_VERSION,
-        cookies: [],
-        headers: [],
-        content: {
-          size: -1,
-          mimeType: request.headerValue('content-type') || 'x-unknown',
-        },
-        headersSize: -1,
-        bodySize: -1,
-        redirectURL: '',
-        _transferSize: -1
-      },
-      cache: {
-        beforeRequest: null,
-        afterRequest: null,
-      },
-      timings: {
-        send: -1,
-        wait: -1,
-        receive: -1
-      },
-    };
     if (request.redirectedFrom()) {
       const fromEntry = this._entryForRequest(request.redirectedFrom()!);
       if (fromEntry)
@@ -232,18 +239,8 @@ export class HarTracer {
       }
 
       const content = harEntry.response.content;
-      content.size = buffer.length;
       compressionCalculationBarrier.setDecodedBodySize(buffer.length);
-      if (buffer && buffer.length > 0) {
+      this._storeResponseContent(buffer, content);
-        if (this._options.content === 'embedded') {
-          content.text = buffer.toString('base64');
-          content.encoding = 'base64';
-        } else if (this._options.content === 'sha1') {
-          content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
-          if (this._started)
-            this._delegate.onContentBlob(content._sha1, buffer);
-        }
-      }
     }).catch(() => {
       compressionCalculationBarrier.setDecodedBodySize(0);
     }).then(() => {
@@ -267,6 +264,22 @@ export class HarTracer {
     }));
   }
 
+  private _storeResponseContent(buffer: Buffer | undefined, content: har.Content) {
+    if (!buffer) {
+      content.size = 0;
+      return;
+    }
+    content.size = buffer.length;
+    if (this._options.content === 'embedded') {
+      content.text = buffer.toString('base64');
+      content.encoding = 'base64';
+    } else if (this._options.content === 'sha1') {
+      content._sha1 = calculateSha1(buffer) + '.' + (mime.getExtension(content.mimeType) || 'dat');
+      if (this._started)
+        this._delegate.onContentBlob(content._sha1, buffer);
+    }
+  }
+
   private _onResponse(response: network.Response) {
     const page = response.frame()._page;
     const pageEntry = this._ensurePageEntry(page);
@@ -275,7 +288,7 @@ export class HarTracer {
       return;
     const request = response.request();
 
-    harEntry.request.postData = postDataForHar(request, this._options.content);
+    harEntry.request.postData = postDataForRequest(request, this._options.content);
 
     harEntry.response = {
       status: response.status(),
@@ -372,12 +385,66 @@ export class HarTracer {
   }
 }
 
-function postDataForHar(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
+function createHarEntry(method: string, url: URL, requestref: string, frameref: string): har.Entry {
+  const harEntry: har.Entry = {
+    _requestref: requestref,
+    _frameref: frameref,
+    _monotonicTime: monotonicTime(),
+    startedDateTime: new Date(),
+    time: -1,
+    request: {
+      method: method,
+      url: url.toString(),
+      httpVersion: FALLBACK_HTTP_VERSION,
+      cookies: [],
+      headers: [],
+      queryString: [...url.searchParams].map(e => ({ name: e[0], value: e[1] })),
+      headersSize: -1,
+      bodySize: 0,
+    },
+    response: {
+      status: -1,
+      statusText: '',
+      httpVersion: FALLBACK_HTTP_VERSION,
+      cookies: [],
+      headers: [],
+      content: {
+        size: -1,
+        mimeType: 'x-unknown',
+      },
+      headersSize: -1,
+      bodySize: -1,
+      redirectURL: '',
+      _transferSize: -1
+    },
+    cache: {
+      beforeRequest: null,
+      afterRequest: null,
+    },
+    timings: {
+      send: -1,
+      wait: -1,
+      receive: -1
+    },
+  };
+  return harEntry;
+}
+
+function postDataForRequest(request: network.Request, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
   const postData = request.postDataBuffer();
   if (!postData)
     return;
 
-  const contentType = request.headerValue('content-type') || 'application/octet-stream';
+  const contentType = request.headerValue('content-type');
+  return postDataForBuffer(postData, contentType, content);
+}
+
+function postDataForBuffer(postData: Buffer | null, contentType: string | undefined, content: 'omit' | 'sha1' | 'embedded'): har.PostData | undefined {
+  if (!postData)
+    return;
+
+  contentType ??= 'application/octet-stream';
+
   const result: har.PostData = {
     mimeType: contentType,
     text: '',

tests/har.spec.ts

@@ -634,3 +634,40 @@ it('should include _requestref for redirects', async ({ contextFactory, server }
   expect(entryEmptyPage.request.url).toBe(server.EMPTY_PAGE);
   expect(entryEmptyPage._requestref).toBe(requests.get(entryEmptyPage.request.url));
 });
+
+it('should include API request', async ({ contextFactory, server }, testInfo) => {
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  const url = server.PREFIX + '/simple.json';
+  const response = await page.request.post(url, {
+    headers: { cookie: 'a=b; c=d' },
+    data: { foo: 'bar' }
+  });
+  const responseBody = await response.body();
+  const log = await getLog();
+  expect(log.entries.length).toBe(1);
+  const entry = log.entries[0];
+  expect(entry.request.url).toBe(url);
+  expect(entry.request.method).toBe('POST');
+  expect(entry.request.httpVersion).toBe('HTTP/1.1');
+  expect(entry.request.cookies).toEqual([
+    {
+      'name': 'a',
+      'value': 'b'
+    },
+    {
+      'name': 'c',
+      'value': 'd'
+    }
+  ]);
+  expect(entry.request.headers.length).toBeGreaterThan(1);
+  expect(entry.request.headers.find(h => h.name.toLowerCase() === 'user-agent')).toBeTruthy();
+  expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toBe('application/json');
+  expect(entry.request.headers.find(h => h.name.toLowerCase() === 'content-length')?.value).toBe('13');
+  expect(entry.request.bodySize).toBe(13);
+
+  expect(entry.response.status).toBe(200);
+  expect(entry.response.headers.find(h => h.name.toLowerCase() === 'content-type')?.value).toContain('application/json');
+  expect(entry.response.content.size).toBe(15);
+  expect(entry.response.content.text).toBe(responseBody.toString('base64'));
+});
+

tests/tracing.spec.ts

@@ -85,6 +85,19 @@ test('should exclude internal pages', async ({ browserName, context, page, serve
   expect(pageIds.size).toBe(1);
 });
 
+test('should include context API requests', async ({ browserName, context, page, server }, testInfo) => {
+  await context.tracing.start({ snapshots: true });
+  await page.request.post(server.PREFIX + '/simple.json', { data: { foo: 'bar' } });
+  await context.tracing.stop({ path: testInfo.outputPath('trace.zip') });
+  const { events } = await parseTrace(testInfo.outputPath('trace.zip'));
+  const postEvent = events.find(e => e.metadata?.apiName === 'apiRequestContext.post');
+  expect(postEvent).toBeTruthy();
+  const harEntry = events.find(e => e.type === 'resource-snapshot');
+  expect(harEntry).toBeTruthy();
+  expect(harEntry.snapshot.request.url).toBe(server.PREFIX + '/simple.json');
+  expect(harEntry.snapshot.response.status).toBe(200);
+});
+
 test('should collect two traces', async ({ context, page, server }, testInfo) => {
   await context.tracing.start({ screenshots: true, snapshots: true });
   await page.goto(server.EMPTY_PAGE);