feat(har): record remote IP:PORT and SSL details (#6631)

src/server/chromium/crNetworkManager.ts

@@ -296,7 +296,22 @@ export class CRNetworkManager {
         responseStart: -1,
       };
     }
-    return new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody);
+    const response = new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody);
+    if (responsePayload?.remoteIPAddress && typeof responsePayload?.remotePort === 'number')
+      response._serverAddrFinished({
+        ipAddress: responsePayload.remoteIPAddress,
+        port: responsePayload.remotePort,
+      });
+    else
+      response._serverAddrFinished();
+    response._securityDetailsFinished({
+      protocol: responsePayload?.securityDetails?.protocol,
+      subjectName: responsePayload?.securityDetails?.subjectName,
+      issuer: responsePayload?.securityDetails?.issuer,
+      validFrom: responsePayload?.securityDetails?.validFrom,
+      validTo: responsePayload?.securityDetails?.validTo,
+    });
+    return response;
   }
 
   _handleRequestRedirect(request: InterceptableRequest, responsePayload: Protocol.Network.Response, timestamp: number) {

src/server/firefox/ffNetworkManager.ts

@@ -89,6 +89,20 @@ export class FFNetworkManager {
       responseStart: this._relativeTiming(event.timing.responseStart),
     };
     const response = new network.Response(request.request, event.status, event.statusText, event.headers, timing, getResponseBody);
+    if (event?.remoteIPAddress && typeof event?.remotePort === 'number')
+      response._serverAddrFinished({
+        ipAddress: event.remoteIPAddress,
+        port: event.remotePort,
+      });
+    else
+      response._serverAddrFinished()
+    response._securityDetailsFinished({
+      protocol: event?.securityDetails?.protocol,
+      subjectName: event?.securityDetails?.subjectName,
+      issuer: event?.securityDetails?.issuer,
+      validFrom: event?.securityDetails?.validFrom,
+      validTo: event?.securityDetails?.validTo,
+    });
     this._page._frameManager.requestReceivedResponse(response);
   }
 

src/server/network.ts

@@ -263,6 +263,19 @@ export type ResourceTiming = {
   responseStart: number;
 };
 
+export type RemoteAddr = {
+  ipAddress: string;
+  port: number;
+}
+
+export type SecurityDetails = {
+    protocol?: string;
+    subjectName?: string;
+    issuer?: string;
+    validFrom?: number;
+    validTo?: number;
+};
+
 export class Response extends SdkObject {
   private _request: Request;
   private _contentPromise: Promise<Buffer> | null = null;
@@ -275,6 +288,10 @@ export class Response extends SdkObject {
   private _headersMap = new Map<string, string>();
   private _getResponseBodyCallback: GetResponseBodyCallback;
   private _timing: ResourceTiming;
+  private _serverAddrPromise: Promise<RemoteAddr|undefined>;
+  private _serverAddrPromiseCallback: (arg?: RemoteAddr) => void = () => {};
+  private _securityDetailsPromise: Promise<SecurityDetails|undefined>;
+  private _securityDetailsPromiseCallback: (arg?: SecurityDetails) => void = () => {};
 
   constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback) {
     super(request.frame(), 'response');
@@ -287,12 +304,26 @@ export class Response extends SdkObject {
     for (const { name, value } of this._headers)
       this._headersMap.set(name.toLowerCase(), value);
     this._getResponseBodyCallback = getResponseBodyCallback;
+    this._serverAddrPromise = new Promise(f => {
+      this._serverAddrPromiseCallback = f;
+    });
+    this._securityDetailsPromise = new Promise(f => {
+      this._securityDetailsPromiseCallback = f;
+    });
     this._finishedPromise = new Promise(f => {
       this._finishedPromiseCallback = f;
     });
     this._request._setResponse(this);
   }
 
+  _serverAddrFinished(addr?: RemoteAddr) {
+    this._serverAddrPromiseCallback(addr);
+  }
+
+  _securityDetailsFinished(securityDetails?: SecurityDetails) {
+    this._securityDetailsPromiseCallback(securityDetails);
+  }
+
   _requestFinished(responseEndTiming: number, error?: string) {
     this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart);
     this._finishedPromiseCallback({ error });
@@ -326,6 +357,14 @@ export class Response extends SdkObject {
     return this._timing;
   }
 
+  async serverAddr(): Promise<RemoteAddr|null> {
+    return await this._serverAddrPromise || null;
+  }
+
+  async securityDetails(): Promise<SecurityDetails|null> {
+    return await this._securityDetailsPromise || null;
+  }
+
   body(): Promise<Buffer> {
     if (!this._contentPromise) {
       this._contentPromise = this._finishedPromise.then(async ({ error }) => {

src/server/supplements/har/har.ts

@@ -55,6 +55,8 @@ export type Entry = {
   timings: Timings;
   serverIPAddress?: string;
   connection?: string;
+  _serverPort?: number;
+  _securityDetails?: SecurityDetails;
 };
 
 export type Request = {
@@ -144,3 +146,11 @@ export type Timings = {
   receive: number;
   ssl?: number;
 };
+
+export type SecurityDetails = {
+  protocol?: string;
+  subjectName?: string;
+  issuer?: string;
+  validFrom?: number;
+  validTo?: number;
+};

src/server/supplements/har/harTracer.ts

@@ -209,6 +209,18 @@ export class HarTracer {
       receive,
     };
     harEntry.time = [dns, connect, ssl, wait, receive].reduce((pre, cur) => cur > 0 ? cur + pre : pre, 0);
+
+    this._addBarrier(page, response.serverAddr().then(server => {
+      if (server?.ipAddress)
+        harEntry.serverIPAddress = server.ipAddress;
+      if (server?.port)
+        harEntry._serverPort = server.port;
+    }));
+    this._addBarrier(page, response.securityDetails().then(details => {
+      if (details)
+        harEntry._securityDetails = details;
+    }));
+
     if (!this._options.omitContent && response.status() === 200) {
       const promise = response.body().then(buffer => {
         harEntry.response.content.text = buffer.toString('base64');

src/server/webkit/wkPage.ts

@@ -69,6 +69,7 @@ export class WKPage implements PageDelegate {
   _firstNonInitialNavigationCommittedReject = (e: Error) => {};
   private _lastConsoleMessage: { derivedType: string, text: string, handles: JSHandle[]; count: number, location: types.ConsoleMessageLocation; } | null = null;
 
+  private readonly _requestIdToResponseReceivedPayloadEvent = new Map<string, Protocol.Network.responseReceivedPayload>();
   // Holds window features for the next popup being opened via window.open,
   // until the popup page proxy arrives.
   private _nextWindowOpenPopupFeatures?: string[];
@@ -953,6 +954,8 @@ export class WKPage implements PageDelegate {
 
   private _handleRequestRedirect(request: WKInterceptableRequest, responsePayload: Protocol.Network.Response, timestamp: number) {
     const response = request.createResponse(responsePayload);
+    response._securityDetailsFinished();
+    response._serverAddrFinished();
     response._requestFinished(responsePayload.timing ? helper.secondsToRoundishMillis(timestamp - request._timestamp) : -1, 'Response body is unavailable for redirect responses');
     this._requestIdToRequest.delete(request._requestId);
     this._page._frameManager.requestReceivedResponse(response);
@@ -979,6 +982,7 @@ export class WKPage implements PageDelegate {
     // FileUpload sends a response without a matching request.
     if (!request)
       return;
+    this._requestIdToResponseReceivedPayloadEvent.set(request._requestId, event);
     const response = request.createResponse(event.response);
     if (event.response.requestHeaders && Object.keys(event.response.requestHeaders).length)
       request.request.updateWithRawHeaders(headersObjectToArray(event.response.requestHeaders));
@@ -1003,8 +1007,19 @@ export class WKPage implements PageDelegate {
     // Under certain conditions we never get the Network.responseReceived
     // event from protocol. @see https://crbug.com/883475
     const response = request.request._existingResponse();
-    if (response)
+    if (response) {
+      const responseReceivedPayload = this._requestIdToResponseReceivedPayloadEvent.get(request._requestId);
+      response._serverAddrFinished(parseRemoteAddress(event?.metrics?.remoteAddress));
+      response._securityDetailsFinished({
+        protocol: isLoadedSecurely(response.url(), response.timing()) ? event.metrics?.securityConnection?.protocol : undefined,
+        subjectName: responseReceivedPayload?.response.security?.certificate?.subject,
+        validFrom: responseReceivedPayload?.response.security?.certificate?.validFrom,
+        validTo: responseReceivedPayload?.response.security?.certificate?.validUntil,
+      });
       response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp));
+    }
+
+    this._requestIdToResponseReceivedPayloadEvent.delete(request._requestId);
     this._requestIdToRequest.delete(request._requestId);
     this._page._frameManager.requestFinished(request.request);
   }
@@ -1016,8 +1031,11 @@ export class WKPage implements PageDelegate {
     if (!request)
       return;
     const response = request.request._existingResponse();
-    if (response)
+    if (response) {
+      response._serverAddrFinished();
+      response._securityDetailsFinished();
       response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp));
+    }
     this._requestIdToRequest.delete(request._requestId);
     request.request._setFailureText(event.errorText);
     this._page._frameManager.requestFailed(request.request, event.errorText.includes('cancelled'));
@@ -1047,3 +1065,63 @@ function webkitWorldName(world: types.World) {
     case 'utility': return UTILITY_WORLD_NAME;
   }
 }
+
+/**
+ * WebKit Remote Addresses look like:
+ *
+ * macOS:
+ * ::1.8911
+ * 2606:2800:220:1:248:1893:25c8:1946.443
+ * 127.0.0.1:8000
+ *
+ * ubuntu:
+ * ::1:8907
+ * 127.0.0.1:8000
+ *
+ * NB: They look IPv4 and IPv6's with ports but use an alternative notation.
+ */
+function parseRemoteAddress(value?: string) {
+  if (!value)
+    return;
+
+  try {
+    const colon = value.lastIndexOf(':');
+    const dot = value.lastIndexOf('.');
+    if (dot < 0) { // IPv6ish:port
+      return {
+        ipAddress: `[${value.slice(0, colon)}]`,
+        port: +value.slice(colon + 1)
+      };
+    }
+
+    if (colon > dot) { // IPv4:port
+      const [address, port] = value.split(':');
+      return {
+        ipAddress: address,
+        port: +port,
+      };
+    } else { // IPv6ish.port
+      const [address, port] = value.split('.');
+      return {
+        ipAddress: `[${address}]`,
+        port: +port,
+      };
+    }
+  } catch (_) {}
+}
+
+
+/**
+ * Adapted from Source/WebInspectorUI/UserInterface/Models/Resource.js in
+ * WebKit codebase.
+ */
+function isLoadedSecurely(url: string, timing: network.ResourceTiming) {
+  try {
+    const u = new URL(url);
+    if (u.protocol !== 'https:' && u.protocol !== 'wss:' && u.protocol !== 'sftp:')
+      return false;
+    if (timing.secureConnectionStart === -1 && timing.connectStart !== -1)
+      return false;
+    return true;
+  } catch (_) {}
+}

tests/har.spec.ts

@@ -301,3 +301,65 @@ it('should not contain internal pages', async ({ browserName, contextFactory, se
   const log = await getLog();
   expect(log.pages.length).toBe(1);
 });
+
+it('should have connection details', async ({ contextFactory, server, browserName, platform }, testInfo) => {
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.EMPTY_PAGE);
+  const log = await getLog();
+  const { serverIPAddress, _serverPort: port, _securityDetails: securityDetails } = log.entries[0];
+  expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
+  expect(port).toBe(server.PORT);
+  expect(securityDetails).toEqual({});
+});
+
+it('should have security details', async ({ contextFactory, httpsServer, browserName, platform }, testInfo) => {
+  it.fail(browserName === 'webkit' && platform === 'linux', 'https://github.com/microsoft/playwright/issues/6759');
+
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(httpsServer.EMPTY_PAGE);
+  const log = await getLog();
+  const { serverIPAddress, _serverPort: port, _securityDetails: securityDetails } = log.entries[0];
+  expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
+  expect(port).toBe(httpsServer.PORT);
+  if (browserName === 'webkit' && platform === 'win32')
+    expect(securityDetails).toEqual({subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: -1});
+  else if (browserName === 'webkit')
+    expect(securityDetails).toEqual({protocol: 'TLS 1.3', subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: 33086084863});
+  else
+    expect(securityDetails).toEqual({issuer: 'puppeteer-tests', protocol: 'TLS 1.3', subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: 33086084863});
+});
+
+it('should have connection details for redirects', async ({ contextFactory, server, browserName }, testInfo) => {
+  server.setRedirect('/foo.html', '/empty.html');
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.PREFIX + '/foo.html');
+  const log = await getLog();
+  expect(log.entries.length).toBe(2);
+
+  const detailsFoo = log.entries[0];
+
+  if (browserName === 'webkit') {
+    expect(detailsFoo.serverIPAddress).toBeUndefined();
+    expect(detailsFoo._serverPort).toBeUndefined();
+  } else {
+    expect(detailsFoo.serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
+    expect(detailsFoo._serverPort).toBe(server.PORT);
+  }
+
+  const detailsEmpty = log.entries[1];
+  expect(detailsEmpty.serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
+  expect(detailsEmpty._serverPort).toBe(server.PORT);
+});
+
+it('should have connection details for failed requests', async ({ contextFactory, server, browserName, platform }, testInfo) => {
+  server.setRoute('/one-style.css', (_, res) => {
+    res.setHeader('Content-Type', 'text/css');
+    res.connection.destroy();
+  });
+  const { page, getLog } = await pageWithHar(contextFactory, testInfo);
+  await page.goto(server.PREFIX + '/one-style.html');
+  const log = await getLog();
+  const { serverIPAddress, _serverPort: port } = log.entries[0];
+  expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/);
+  expect(port).toBe(server.PORT);
+});