From 195eab8787e4452ec7ad5024c69d033b5950cb0c Mon Sep 17 00:00:00 2001 From: Ross Wollman Date: Tue, 15 Jun 2021 00:48:08 -0700 Subject: [PATCH] feat(har): record remote IP:PORT and SSL details (#6631) --- src/server/chromium/crNetworkManager.ts | 17 ++++- src/server/firefox/ffNetworkManager.ts | 14 +++++ src/server/network.ts | 39 ++++++++++++ src/server/supplements/har/har.ts | 10 +++ src/server/supplements/har/harTracer.ts | 12 ++++ src/server/webkit/wkPage.ts | 82 ++++++++++++++++++++++++- tests/har.spec.ts | 62 +++++++++++++++++++ 7 files changed, 233 insertions(+), 3 deletions(-) diff --git a/src/server/chromium/crNetworkManager.ts b/src/server/chromium/crNetworkManager.ts index 7c008a22cd..075b573ff5 100644 --- a/src/server/chromium/crNetworkManager.ts +++ b/src/server/chromium/crNetworkManager.ts @@ -296,7 +296,22 @@ export class CRNetworkManager { responseStart: -1, }; } - return new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody); + const response = new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody); + if (responsePayload?.remoteIPAddress && typeof responsePayload?.remotePort === 'number') + response._serverAddrFinished({ + ipAddress: responsePayload.remoteIPAddress, + port: responsePayload.remotePort, + }); + else + response._serverAddrFinished(); + response._securityDetailsFinished({ + protocol: responsePayload?.securityDetails?.protocol, + subjectName: responsePayload?.securityDetails?.subjectName, + issuer: responsePayload?.securityDetails?.issuer, + validFrom: responsePayload?.securityDetails?.validFrom, + validTo: responsePayload?.securityDetails?.validTo, + }); + return response; } _handleRequestRedirect(request: InterceptableRequest, responsePayload: Protocol.Network.Response, timestamp: number) { diff --git a/src/server/firefox/ffNetworkManager.ts b/src/server/firefox/ffNetworkManager.ts index f2fee87e41..7c4c6721c8 100644 --- a/src/server/firefox/ffNetworkManager.ts +++ b/src/server/firefox/ffNetworkManager.ts @@ -89,6 +89,20 @@ export class FFNetworkManager { responseStart: this._relativeTiming(event.timing.responseStart), }; const response = new network.Response(request.request, event.status, event.statusText, event.headers, timing, getResponseBody); + if (event?.remoteIPAddress && typeof event?.remotePort === 'number') + response._serverAddrFinished({ + ipAddress: event.remoteIPAddress, + port: event.remotePort, + }); + else + response._serverAddrFinished() + response._securityDetailsFinished({ + protocol: event?.securityDetails?.protocol, + subjectName: event?.securityDetails?.subjectName, + issuer: event?.securityDetails?.issuer, + validFrom: event?.securityDetails?.validFrom, + validTo: event?.securityDetails?.validTo, + }); this._page._frameManager.requestReceivedResponse(response); } diff --git a/src/server/network.ts b/src/server/network.ts index b66a7aac79..3b4fd46a0f 100644 --- a/src/server/network.ts +++ b/src/server/network.ts @@ -263,6 +263,19 @@ export type ResourceTiming = { responseStart: number; }; +export type RemoteAddr = { + ipAddress: string; + port: number; +} + +export type SecurityDetails = { + protocol?: string; + subjectName?: string; + issuer?: string; + validFrom?: number; + validTo?: number; +}; + export class Response extends SdkObject { private _request: Request; private _contentPromise: Promise | null = null; @@ -275,6 +288,10 @@ export class Response extends SdkObject { private _headersMap = new Map(); private _getResponseBodyCallback: GetResponseBodyCallback; private _timing: ResourceTiming; + private _serverAddrPromise: Promise; + private _serverAddrPromiseCallback: (arg?: RemoteAddr) => void = () => {}; + private _securityDetailsPromise: Promise; + private _securityDetailsPromiseCallback: (arg?: SecurityDetails) => void = () => {}; constructor(request: Request, status: number, statusText: string, headers: types.HeadersArray, timing: ResourceTiming, getResponseBodyCallback: GetResponseBodyCallback) { super(request.frame(), 'response'); @@ -287,12 +304,26 @@ export class Response extends SdkObject { for (const { name, value } of this._headers) this._headersMap.set(name.toLowerCase(), value); this._getResponseBodyCallback = getResponseBodyCallback; + this._serverAddrPromise = new Promise(f => { + this._serverAddrPromiseCallback = f; + }); + this._securityDetailsPromise = new Promise(f => { + this._securityDetailsPromiseCallback = f; + }); this._finishedPromise = new Promise(f => { this._finishedPromiseCallback = f; }); this._request._setResponse(this); } + _serverAddrFinished(addr?: RemoteAddr) { + this._serverAddrPromiseCallback(addr); + } + + _securityDetailsFinished(securityDetails?: SecurityDetails) { + this._securityDetailsPromiseCallback(securityDetails); + } + _requestFinished(responseEndTiming: number, error?: string) { this._request._responseEndTiming = Math.max(responseEndTiming, this._timing.responseStart); this._finishedPromiseCallback({ error }); @@ -326,6 +357,14 @@ export class Response extends SdkObject { return this._timing; } + async serverAddr(): Promise { + return await this._serverAddrPromise || null; + } + + async securityDetails(): Promise { + return await this._securityDetailsPromise || null; + } + body(): Promise { if (!this._contentPromise) { this._contentPromise = this._finishedPromise.then(async ({ error }) => { diff --git a/src/server/supplements/har/har.ts b/src/server/supplements/har/har.ts index b8d2c458ae..e06b1c9cf9 100644 --- a/src/server/supplements/har/har.ts +++ b/src/server/supplements/har/har.ts @@ -55,6 +55,8 @@ export type Entry = { timings: Timings; serverIPAddress?: string; connection?: string; + _serverPort?: number; + _securityDetails?: SecurityDetails; }; export type Request = { @@ -144,3 +146,11 @@ export type Timings = { receive: number; ssl?: number; }; + +export type SecurityDetails = { + protocol?: string; + subjectName?: string; + issuer?: string; + validFrom?: number; + validTo?: number; +}; diff --git a/src/server/supplements/har/harTracer.ts b/src/server/supplements/har/harTracer.ts index cad34a866a..b34c3fdb38 100644 --- a/src/server/supplements/har/harTracer.ts +++ b/src/server/supplements/har/harTracer.ts @@ -209,6 +209,18 @@ export class HarTracer { receive, }; harEntry.time = [dns, connect, ssl, wait, receive].reduce((pre, cur) => cur > 0 ? cur + pre : pre, 0); + + this._addBarrier(page, response.serverAddr().then(server => { + if (server?.ipAddress) + harEntry.serverIPAddress = server.ipAddress; + if (server?.port) + harEntry._serverPort = server.port; + })); + this._addBarrier(page, response.securityDetails().then(details => { + if (details) + harEntry._securityDetails = details; + })); + if (!this._options.omitContent && response.status() === 200) { const promise = response.body().then(buffer => { harEntry.response.content.text = buffer.toString('base64'); diff --git a/src/server/webkit/wkPage.ts b/src/server/webkit/wkPage.ts index 5bb11a27c1..25193b63c4 100644 --- a/src/server/webkit/wkPage.ts +++ b/src/server/webkit/wkPage.ts @@ -69,6 +69,7 @@ export class WKPage implements PageDelegate { _firstNonInitialNavigationCommittedReject = (e: Error) => {}; private _lastConsoleMessage: { derivedType: string, text: string, handles: JSHandle[]; count: number, location: types.ConsoleMessageLocation; } | null = null; + private readonly _requestIdToResponseReceivedPayloadEvent = new Map(); // Holds window features for the next popup being opened via window.open, // until the popup page proxy arrives. private _nextWindowOpenPopupFeatures?: string[]; @@ -953,6 +954,8 @@ export class WKPage implements PageDelegate { private _handleRequestRedirect(request: WKInterceptableRequest, responsePayload: Protocol.Network.Response, timestamp: number) { const response = request.createResponse(responsePayload); + response._securityDetailsFinished(); + response._serverAddrFinished(); response._requestFinished(responsePayload.timing ? helper.secondsToRoundishMillis(timestamp - request._timestamp) : -1, 'Response body is unavailable for redirect responses'); this._requestIdToRequest.delete(request._requestId); this._page._frameManager.requestReceivedResponse(response); @@ -979,6 +982,7 @@ export class WKPage implements PageDelegate { // FileUpload sends a response without a matching request. if (!request) return; + this._requestIdToResponseReceivedPayloadEvent.set(request._requestId, event); const response = request.createResponse(event.response); if (event.response.requestHeaders && Object.keys(event.response.requestHeaders).length) request.request.updateWithRawHeaders(headersObjectToArray(event.response.requestHeaders)); @@ -1003,8 +1007,19 @@ export class WKPage implements PageDelegate { // Under certain conditions we never get the Network.responseReceived // event from protocol. @see https://crbug.com/883475 const response = request.request._existingResponse(); - if (response) + if (response) { + const responseReceivedPayload = this._requestIdToResponseReceivedPayloadEvent.get(request._requestId); + response._serverAddrFinished(parseRemoteAddress(event?.metrics?.remoteAddress)); + response._securityDetailsFinished({ + protocol: isLoadedSecurely(response.url(), response.timing()) ? event.metrics?.securityConnection?.protocol : undefined, + subjectName: responseReceivedPayload?.response.security?.certificate?.subject, + validFrom: responseReceivedPayload?.response.security?.certificate?.validFrom, + validTo: responseReceivedPayload?.response.security?.certificate?.validUntil, + }); response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp)); + } + + this._requestIdToResponseReceivedPayloadEvent.delete(request._requestId); this._requestIdToRequest.delete(request._requestId); this._page._frameManager.requestFinished(request.request); } @@ -1016,8 +1031,11 @@ export class WKPage implements PageDelegate { if (!request) return; const response = request.request._existingResponse(); - if (response) + if (response) { + response._serverAddrFinished(); + response._securityDetailsFinished(); response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp)); + } this._requestIdToRequest.delete(request._requestId); request.request._setFailureText(event.errorText); this._page._frameManager.requestFailed(request.request, event.errorText.includes('cancelled')); @@ -1047,3 +1065,63 @@ function webkitWorldName(world: types.World) { case 'utility': return UTILITY_WORLD_NAME; } } + +/** + * WebKit Remote Addresses look like: + * + * macOS: + * ::1.8911 + * 2606:2800:220:1:248:1893:25c8:1946.443 + * 127.0.0.1:8000 + * + * ubuntu: + * ::1:8907 + * 127.0.0.1:8000 + * + * NB: They look IPv4 and IPv6's with ports but use an alternative notation. + */ +function parseRemoteAddress(value?: string) { + if (!value) + return; + + try { + const colon = value.lastIndexOf(':'); + const dot = value.lastIndexOf('.'); + if (dot < 0) { // IPv6ish:port + return { + ipAddress: `[${value.slice(0, colon)}]`, + port: +value.slice(colon + 1) + }; + } + + if (colon > dot) { // IPv4:port + const [address, port] = value.split(':'); + return { + ipAddress: address, + port: +port, + }; + } else { // IPv6ish.port + const [address, port] = value.split('.'); + return { + ipAddress: `[${address}]`, + port: +port, + }; + } + } catch (_) {} +} + + +/** + * Adapted from Source/WebInspectorUI/UserInterface/Models/Resource.js in + * WebKit codebase. + */ +function isLoadedSecurely(url: string, timing: network.ResourceTiming) { + try { + const u = new URL(url); + if (u.protocol !== 'https:' && u.protocol !== 'wss:' && u.protocol !== 'sftp:') + return false; + if (timing.secureConnectionStart === -1 && timing.connectStart !== -1) + return false; + return true; + } catch (_) {} +} diff --git a/tests/har.spec.ts b/tests/har.spec.ts index 78caaa7fff..4c9c66cf1e 100644 --- a/tests/har.spec.ts +++ b/tests/har.spec.ts @@ -301,3 +301,65 @@ it('should not contain internal pages', async ({ browserName, contextFactory, se const log = await getLog(); expect(log.pages.length).toBe(1); }); + +it('should have connection details', async ({ contextFactory, server, browserName, platform }, testInfo) => { + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.EMPTY_PAGE); + const log = await getLog(); + const { serverIPAddress, _serverPort: port, _securityDetails: securityDetails } = log.entries[0]; + expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/); + expect(port).toBe(server.PORT); + expect(securityDetails).toEqual({}); +}); + +it('should have security details', async ({ contextFactory, httpsServer, browserName, platform }, testInfo) => { + it.fail(browserName === 'webkit' && platform === 'linux', 'https://github.com/microsoft/playwright/issues/6759'); + + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(httpsServer.EMPTY_PAGE); + const log = await getLog(); + const { serverIPAddress, _serverPort: port, _securityDetails: securityDetails } = log.entries[0]; + expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/); + expect(port).toBe(httpsServer.PORT); + if (browserName === 'webkit' && platform === 'win32') + expect(securityDetails).toEqual({subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: -1}); + else if (browserName === 'webkit') + expect(securityDetails).toEqual({protocol: 'TLS 1.3', subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: 33086084863}); + else + expect(securityDetails).toEqual({issuer: 'puppeteer-tests', protocol: 'TLS 1.3', subjectName: 'puppeteer-tests', validFrom: 1550084863, validTo: 33086084863}); +}); + +it('should have connection details for redirects', async ({ contextFactory, server, browserName }, testInfo) => { + server.setRedirect('/foo.html', '/empty.html'); + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.PREFIX + '/foo.html'); + const log = await getLog(); + expect(log.entries.length).toBe(2); + + const detailsFoo = log.entries[0]; + + if (browserName === 'webkit') { + expect(detailsFoo.serverIPAddress).toBeUndefined(); + expect(detailsFoo._serverPort).toBeUndefined(); + } else { + expect(detailsFoo.serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/); + expect(detailsFoo._serverPort).toBe(server.PORT); + } + + const detailsEmpty = log.entries[1]; + expect(detailsEmpty.serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/); + expect(detailsEmpty._serverPort).toBe(server.PORT); +}); + +it('should have connection details for failed requests', async ({ contextFactory, server, browserName, platform }, testInfo) => { + server.setRoute('/one-style.css', (_, res) => { + res.setHeader('Content-Type', 'text/css'); + res.connection.destroy(); + }); + const { page, getLog } = await pageWithHar(contextFactory, testInfo); + await page.goto(server.PREFIX + '/one-style.html'); + const log = await getLog(); + const { serverIPAddress, _serverPort: port } = log.entries[0]; + expect(serverIPAddress).toMatch(/^127\.0\.0\.1|\[::1\]/); + expect(port).toBe(server.PORT); +});