playwright/src/server/chromium/crNetworkManager.ts

512 lines
23 KiB
TypeScript

/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { CRSession } from './crConnection';
import { Page } from '../page';
import { helper } from '../helper';
import { eventsHelper, RegisteredListener } from '../../utils/eventsHelper';
import { Protocol } from './protocol';
import * as network from '../network';
import * as frames from '../frames';
import * as types from '../types';
import { CRPage } from './crPage';
import { assert, headersObjectToArray } from '../../utils/utils';
export class CRNetworkManager {
private _client: CRSession;
private _page: Page;
private _parentManager: CRNetworkManager | null;
private _requestIdToRequest = new Map<string, InterceptableRequest>();
private _requestIdToRequestWillBeSentEvent = new Map<string, Protocol.Network.requestWillBeSentPayload>();
private _credentials: {username: string, password: string} | null = null;
private _attemptedAuthentications = new Set<string>();
private _userRequestInterceptionEnabled = false;
private _protocolRequestInterceptionEnabled = false;
private _requestIdToRequestPausedEvent = new Map<string, Protocol.Fetch.requestPausedPayload>();
private _eventListeners: RegisteredListener[];
private _requestIdToExtraInfo = new Map<string, Protocol.Network.requestWillBeSentExtraInfoPayload>();
constructor(client: CRSession, page: Page, parentManager: CRNetworkManager | null) {
this._client = client;
this._page = page;
this._parentManager = parentManager;
this._eventListeners = this.instrumentNetworkEvents(client);
}
instrumentNetworkEvents(session: CRSession, workerFrame?: frames.Frame): RegisteredListener[] {
return [
eventsHelper.addEventListener(session, 'Fetch.requestPaused', this._onRequestPaused.bind(this, workerFrame)),
eventsHelper.addEventListener(session, 'Fetch.authRequired', this._onAuthRequired.bind(this)),
eventsHelper.addEventListener(session, 'Network.requestWillBeSent', this._onRequestWillBeSent.bind(this, workerFrame)),
eventsHelper.addEventListener(session, 'Network.requestWillBeSentExtraInfo', this._onRequestWillBeSentExtraInfo.bind(this)),
eventsHelper.addEventListener(session, 'Network.responseReceived', this._onResponseReceived.bind(this)),
eventsHelper.addEventListener(session, 'Network.loadingFinished', this._onLoadingFinished.bind(this)),
eventsHelper.addEventListener(session, 'Network.loadingFailed', this._onLoadingFailed.bind(this)),
eventsHelper.addEventListener(session, 'Network.webSocketCreated', e => this._page._frameManager.onWebSocketCreated(e.requestId, e.url)),
eventsHelper.addEventListener(session, 'Network.webSocketWillSendHandshakeRequest', e => this._page._frameManager.onWebSocketRequest(e.requestId)),
eventsHelper.addEventListener(session, 'Network.webSocketHandshakeResponseReceived', e => this._page._frameManager.onWebSocketResponse(e.requestId, e.response.status, e.response.statusText)),
eventsHelper.addEventListener(session, 'Network.webSocketFrameSent', e => e.response.payloadData && this._page._frameManager.onWebSocketFrameSent(e.requestId, e.response.opcode, e.response.payloadData)),
eventsHelper.addEventListener(session, 'Network.webSocketFrameReceived', e => e.response.payloadData && this._page._frameManager.webSocketFrameReceived(e.requestId, e.response.opcode, e.response.payloadData)),
eventsHelper.addEventListener(session, 'Network.webSocketClosed', e => this._page._frameManager.webSocketClosed(e.requestId)),
eventsHelper.addEventListener(session, 'Network.webSocketFrameError', e => this._page._frameManager.webSocketError(e.requestId, e.errorMessage)),
];
}
async initialize() {
await this._client.send('Network.enable');
}
dispose() {
eventsHelper.removeEventListeners(this._eventListeners);
}
async authenticate(credentials: types.Credentials | null) {
this._credentials = credentials;
await this._updateProtocolRequestInterception();
}
async setOffline(offline: boolean) {
await this._client.send('Network.emulateNetworkConditions', {
offline,
// values of 0 remove any active throttling. crbug.com/456324#c9
latency: 0,
downloadThroughput: -1,
uploadThroughput: -1
});
}
async setRequestInterception(value: boolean) {
this._userRequestInterceptionEnabled = value;
await this._updateProtocolRequestInterception();
}
async _updateProtocolRequestInterception() {
const enabled = this._userRequestInterceptionEnabled || !!this._credentials;
if (enabled === this._protocolRequestInterceptionEnabled)
return;
this._protocolRequestInterceptionEnabled = enabled;
if (enabled) {
await Promise.all([
this._client.send('Network.setCacheDisabled', { cacheDisabled: true }),
this._client.send('Fetch.enable', {
handleAuthRequests: true,
patterns: [{urlPattern: '*', requestStage: 'Request'}, {urlPattern: '*', requestStage: 'Response'}],
}),
]);
} else {
await Promise.all([
this._client.send('Network.setCacheDisabled', { cacheDisabled: false }),
this._client.send('Fetch.disable')
]);
}
}
_onRequestWillBeSent(workerFrame: frames.Frame | undefined, event: Protocol.Network.requestWillBeSentPayload) {
// Request interception doesn't happen for data URLs with Network Service.
if (this._protocolRequestInterceptionEnabled && !event.request.url.startsWith('data:')) {
const requestId = event.requestId;
const requestPausedEvent = this._requestIdToRequestPausedEvent.get(requestId);
if (requestPausedEvent) {
this._onRequest(workerFrame, event, requestPausedEvent);
this._requestIdToRequestPausedEvent.delete(requestId);
} else {
this._requestIdToRequestWillBeSentEvent.set(event.requestId, event);
}
} else {
this._onRequest(workerFrame, event, null);
}
const extraInfo = this._requestIdToExtraInfo.get(event.requestId);
if (extraInfo)
this._onRequestWillBeSentExtraInfo(extraInfo);
}
_onRequestWillBeSentExtraInfo(event: Protocol.Network.requestWillBeSentExtraInfoPayload) {
const request = this._requestIdToRequest.get(event.requestId);
if (request) {
request.request.updateWithRawHeaders(headersObjectToArray(event.headers));
this._requestIdToExtraInfo.delete(event.requestId);
} else {
this._requestIdToExtraInfo.set(event.requestId, event);
}
}
_onAuthRequired(event: Protocol.Fetch.authRequiredPayload) {
let response: 'Default' | 'CancelAuth' | 'ProvideCredentials' = 'Default';
if (this._attemptedAuthentications.has(event.requestId)) {
response = 'CancelAuth';
} else if (this._credentials) {
response = 'ProvideCredentials';
this._attemptedAuthentications.add(event.requestId);
}
const {username, password} = this._credentials || {username: undefined, password: undefined};
this._client._sendMayFail('Fetch.continueWithAuth', {
requestId: event.requestId,
authChallengeResponse: { response, username, password },
});
}
_onRequestPaused(workerFrame: frames.Frame | undefined, event: Protocol.Fetch.requestPausedPayload) {
if (!this._userRequestInterceptionEnabled && this._protocolRequestInterceptionEnabled) {
this._client._sendMayFail('Fetch.continueRequest', {
requestId: event.requestId
});
}
if (!event.networkId) {
// Fetch without networkId means that request was not recongnized by inspector, and
// it will never receive Network.requestWillBeSent. Most likely, this is an internal request
// that we can safely fail.
this._client._sendMayFail('Fetch.failRequest', {
requestId: event.requestId,
errorReason: 'Aborted',
});
return;
}
if (event.request.url.startsWith('data:'))
return;
if (event.responseStatusCode || event.responseHeaders || event.responseErrorReason) {
const request = this._requestIdToRequest.get(event.networkId);
if (!request || !request._onInterceptedResponse) {
this._client._sendMayFail('Fetch.continueRequest', {
requestId: event.requestId
});
return;
}
request._onInterceptedResponse!(event);
return;
}
const requestId = event.networkId;
const requestWillBeSentEvent = this._requestIdToRequestWillBeSentEvent.get(requestId);
if (requestWillBeSentEvent) {
this._onRequest(workerFrame, requestWillBeSentEvent, event);
this._requestIdToRequestWillBeSentEvent.delete(requestId);
} else {
this._requestIdToRequestPausedEvent.set(requestId, event);
}
}
_onRequest(workerFrame: frames.Frame | undefined, requestWillBeSentEvent: Protocol.Network.requestWillBeSentPayload, requestPausedEvent: Protocol.Fetch.requestPausedPayload | null) {
if (requestWillBeSentEvent.request.url.startsWith('data:'))
return;
let redirectedFrom: network.Request | null = null;
if (requestWillBeSentEvent.redirectResponse) {
const request = this._requestIdToRequest.get(requestWillBeSentEvent.requestId);
// If we connect late to the target, we could have missed the requestWillBeSent event.
if (request) {
this._handleRequestRedirect(request, requestWillBeSentEvent.redirectResponse, requestWillBeSentEvent.timestamp);
redirectedFrom = request.request;
}
}
let frame = requestWillBeSentEvent.frameId ? this._page._frameManager.frame(requestWillBeSentEvent.frameId) : workerFrame;
// Requests from workers lack frameId, because we receive Network.requestWillBeSent
// on the worker target. However, we receive Fetch.requestPaused on the page target,
// and lack workerFrame there. Luckily, Fetch.requestPaused provides a frameId.
if (!frame && requestPausedEvent && requestPausedEvent.frameId)
frame = this._page._frameManager.frame(requestPausedEvent.frameId);
// Check if it's main resource request interception (targetId === main frame id).
if (!frame && requestWillBeSentEvent.frameId === (this._page._delegate as CRPage)._targetId) {
// Main resource request for the page is being intercepted so the Frame is not created
// yet. Precreate it here for the purposes of request interception. It will be updated
// later as soon as the request continues and we receive frame tree from the page.
frame = this._page._frameManager.frameAttached(requestWillBeSentEvent.frameId, null);
}
// CORS options request is generated by the network stack. If interception is enabled,
// we accept all CORS options, assuming that this was intended when setting route.
//
// Note: it would be better to match the URL against interception patterns, but
// that information is only available to the client. Perhaps we can just route to the client?
if (requestPausedEvent && requestPausedEvent.request.method === 'OPTIONS' && this._page._needsRequestInterception()) {
const requestHeaders = requestPausedEvent.request.headers;
const responseHeaders: Protocol.Fetch.HeaderEntry[] = [
{ name: 'Access-Control-Allow-Origin', value: requestHeaders['Origin'] || '*' },
{ name: 'Access-Control-Allow-Methods', value: requestHeaders['Access-Control-Request-Method'] || 'GET, POST, OPTIONS, DELETE' },
{ name: 'Access-Control-Allow-Credentials', value: 'true' }
];
if (requestHeaders['Access-Control-Request-Headers'])
responseHeaders.push({ name: 'Access-Control-Allow-Headers', value: requestHeaders['Access-Control-Request-Headers'] });
this._client._sendMayFail('Fetch.fulfillRequest', {
requestId: requestPausedEvent.requestId,
responseCode: 204,
responsePhrase: network.STATUS_TEXTS['204'],
responseHeaders,
body: '',
});
return;
}
if (!frame) {
if (requestPausedEvent)
this._client._sendMayFail('Fetch.continueRequest', { requestId: requestPausedEvent.requestId });
return;
}
let allowInterception = this._userRequestInterceptionEnabled;
if (redirectedFrom) {
allowInterception = false;
// We do not support intercepting redirects.
if (requestPausedEvent)
this._client._sendMayFail('Fetch.continueRequest', { requestId: requestPausedEvent.requestId });
}
const isNavigationRequest = requestWillBeSentEvent.requestId === requestWillBeSentEvent.loaderId && requestWillBeSentEvent.type === 'Document';
const documentId = isNavigationRequest ? requestWillBeSentEvent.loaderId : undefined;
const request = new InterceptableRequest({
client: this._client,
frame,
documentId,
allowInterception,
requestWillBeSentEvent,
requestPausedEvent,
redirectedFrom
});
this._requestIdToRequest.set(requestWillBeSentEvent.requestId, request);
this._page._frameManager.requestStarted(request.request);
}
_createResponse(request: InterceptableRequest, responsePayload: Protocol.Network.Response): network.Response {
const getResponseBody = async () => {
const response = await this._client.send('Network.getResponseBody', { requestId: request._requestId });
return Buffer.from(response.body, response.base64Encoded ? 'base64' : 'utf8');
};
const timingPayload = responsePayload.timing!;
let timing: network.ResourceTiming;
if (timingPayload) {
timing = {
startTime: (timingPayload.requestTime - request._timestamp + request._wallTime) * 1000,
domainLookupStart: timingPayload.dnsStart,
domainLookupEnd: timingPayload.dnsEnd,
connectStart: timingPayload.connectStart,
secureConnectionStart: timingPayload.sslStart,
connectEnd: timingPayload.connectEnd,
requestStart: timingPayload.sendStart,
responseStart: timingPayload.receiveHeadersEnd,
};
} else {
timing = {
startTime: request._wallTime * 1000,
domainLookupStart: -1,
domainLookupEnd: -1,
connectStart: -1,
secureConnectionStart: -1,
connectEnd: -1,
requestStart: -1,
responseStart: -1,
};
}
const response = new network.Response(request.request, responsePayload.status, responsePayload.statusText, headersObjectToArray(responsePayload.headers), timing, getResponseBody);
if (responsePayload?.remoteIPAddress && typeof responsePayload?.remotePort === 'number') {
response._serverAddrFinished({
ipAddress: responsePayload.remoteIPAddress,
port: responsePayload.remotePort,
});
} else {
response._serverAddrFinished();
}
response._securityDetailsFinished({
protocol: responsePayload?.securityDetails?.protocol,
subjectName: responsePayload?.securityDetails?.subjectName,
issuer: responsePayload?.securityDetails?.issuer,
validFrom: responsePayload?.securityDetails?.validFrom,
validTo: responsePayload?.securityDetails?.validTo,
});
return response;
}
_handleRequestRedirect(request: InterceptableRequest, responsePayload: Protocol.Network.Response, timestamp: number) {
const response = this._createResponse(request, responsePayload);
response._requestFinished((timestamp - request._timestamp) * 1000, 'Response body is unavailable for redirect responses');
this._requestIdToRequest.delete(request._requestId);
if (request._interceptionId)
this._attemptedAuthentications.delete(request._interceptionId);
this._page._frameManager.requestReceivedResponse(response);
this._page._frameManager.requestFinished(request.request);
}
_onResponseReceived(event: Protocol.Network.responseReceivedPayload) {
const request = this._requestIdToRequest.get(event.requestId);
// FileUpload sends a response without a matching request.
if (!request)
return;
const response = this._createResponse(request, event.response);
this._page._frameManager.requestReceivedResponse(response);
}
_onLoadingFinished(event: Protocol.Network.loadingFinishedPayload) {
let request = this._requestIdToRequest.get(event.requestId);
if (!request)
request = this._maybeAdoptMainRequest(event.requestId);
// For certain requestIds we never receive requestWillBeSent event.
// @see https://crbug.com/750469
if (!request)
return;
// Under certain conditions we never get the Network.responseReceived
// event from protocol. @see https://crbug.com/883475
const response = request.request._existingResponse();
if (response)
response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp));
this._requestIdToRequest.delete(request._requestId);
if (request._interceptionId)
this._attemptedAuthentications.delete(request._interceptionId);
this._page._frameManager.requestFinished(request.request);
}
_onLoadingFailed(event: Protocol.Network.loadingFailedPayload) {
let request = this._requestIdToRequest.get(event.requestId);
if (!request)
request = this._maybeAdoptMainRequest(event.requestId);
// For certain requestIds we never receive requestWillBeSent event.
// @see https://crbug.com/750469
if (!request)
return;
const response = request.request._existingResponse();
if (response)
response._requestFinished(helper.secondsToRoundishMillis(event.timestamp - request._timestamp));
this._requestIdToRequest.delete(request._requestId);
if (request._interceptionId)
this._attemptedAuthentications.delete(request._interceptionId);
request.request._setFailureText(event.errorText);
this._page._frameManager.requestFailed(request.request, !!event.canceled);
}
private _maybeAdoptMainRequest(requestId: Protocol.Network.RequestId): InterceptableRequest | undefined {
// OOPIF has a main request that starts in the parent session but finishes in the child session.
if (!this._parentManager)
return;
const request = this._parentManager._requestIdToRequest.get(requestId);
// Main requests have matching loaderId and requestId.
if (!request || request._documentId !== requestId)
return;
this._requestIdToRequest.set(requestId, request);
this._parentManager._requestIdToRequest.delete(requestId);
if (request._interceptionId && this._parentManager._attemptedAuthentications.has(request._interceptionId)) {
this._parentManager._attemptedAuthentications.delete(request._interceptionId);
this._attemptedAuthentications.add(request._interceptionId);
}
return request;
}
}
class InterceptableRequest implements network.RouteDelegate {
readonly request: network.Request;
_requestId: string;
_interceptionId: string | null;
_documentId: string | undefined;
private readonly _client: CRSession;
_timestamp: number;
_wallTime: number;
_onInterceptedResponse: ((event: Protocol.Fetch.requestPausedPayload) => void) | null = null;
constructor(options: {
client: CRSession;
frame: frames.Frame;
documentId?: string;
allowInterception: boolean;
requestWillBeSentEvent: Protocol.Network.requestWillBeSentPayload;
requestPausedEvent: Protocol.Fetch.requestPausedPayload | null;
redirectedFrom: network.Request | null;
}) {
const { client, frame, documentId, allowInterception, requestWillBeSentEvent, requestPausedEvent, redirectedFrom } = options;
this._client = client;
this._timestamp = requestWillBeSentEvent.timestamp;
this._wallTime = requestWillBeSentEvent.wallTime;
this._requestId = requestWillBeSentEvent.requestId;
this._interceptionId = requestPausedEvent && requestPausedEvent.requestId;
this._documentId = documentId;
const {
headers,
method,
url,
postDataEntries = null,
} = requestPausedEvent ? requestPausedEvent.request : requestWillBeSentEvent.request;
const type = (requestWillBeSentEvent.type || '').toLowerCase();
let postDataBuffer = null;
if (postDataEntries && postDataEntries.length && postDataEntries[0].bytes)
postDataBuffer = Buffer.from(postDataEntries[0].bytes, 'base64');
this.request = new network.Request(allowInterception ? this : null, frame, redirectedFrom, documentId, url, type, method, postDataBuffer, headersObjectToArray(headers));
}
async responseBody(): Promise<Buffer> {
const response = await this._client.send('Fetch.getResponseBody', { requestId: this._interceptionId! });
return Buffer.from(response.body, response.base64Encoded ? 'base64' : 'utf8');
}
async continue(overrides: types.NormalizedContinueOverrides): Promise<network.InterceptedResponse|null> {
const interceptPromise = overrides.interceptResponse ? new Promise<Protocol.Fetch.requestPausedPayload>(resolve => this._onInterceptedResponse = resolve) : null;
// In certain cases, protocol will return error if the request was already canceled
// or the page was closed. We should tolerate these errors.
await this._client._sendMayFail('Fetch.continueRequest', {
requestId: this._interceptionId!,
url: overrides.url,
headers: overrides.headers,
method: overrides.method,
postData: overrides.postData ? overrides.postData.toString('base64') : undefined
});
if (!interceptPromise)
return null;
const event = await interceptPromise;
return new network.InterceptedResponse(this.request, event.responseStatusCode!, event.responseErrorReason!, event.responseHeaders!);
}
async fulfill(response: types.NormalizedFulfillResponse) {
const body = response.isBase64 ? response.body : Buffer.from(response.body).toString('base64');
// In certain cases, protocol will return error if the request was already canceled
// or the page was closed. We should tolerate these errors.
await this._client._sendMayFail('Fetch.fulfillRequest', {
requestId: this._interceptionId!,
responseCode: response.status,
responsePhrase: network.STATUS_TEXTS[String(response.status)],
responseHeaders: response.headers,
body,
});
}
async abort(errorCode: string = 'failed') {
const errorReason = errorReasons[errorCode];
assert(errorReason, 'Unknown error code: ' + errorCode);
// In certain cases, protocol will return error if the request was already canceled
// or the page was closed. We should tolerate these errors.
await this._client._sendMayFail('Fetch.failRequest', {
requestId: this._interceptionId!,
errorReason
});
}
}
const errorReasons: { [reason: string]: Protocol.Network.ErrorReason } = {
'aborted': 'Aborted',
'accessdenied': 'AccessDenied',
'addressunreachable': 'AddressUnreachable',
'blockedbyclient': 'BlockedByClient',
'blockedbyresponse': 'BlockedByResponse',
'connectionaborted': 'ConnectionAborted',
'connectionclosed': 'ConnectionClosed',
'connectionfailed': 'ConnectionFailed',
'connectionrefused': 'ConnectionRefused',
'connectionreset': 'ConnectionReset',
'internetdisconnected': 'InternetDisconnected',
'namenotresolved': 'NameNotResolved',
'timedout': 'TimedOut',
'failed': 'Failed',
};