/** * Copyright (c) Microsoft Corporation. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ import { URLSearchParams } from 'url'; import type * as channels from '@protocol/channels'; import { ChannelOwner } from './channelOwner'; import { Frame } from './frame'; import { Worker } from './worker'; import type { Headers, RemoteAddr, SecurityDetails, WaitForEventOptions } from './types'; import fs from 'fs'; import { mime } from '../utilsBundle'; import { assert, isString, headersObjectToArray, isRegExp } from '../utils'; import { ManualPromise } from '../utils/manualPromise'; import { Events } from './events'; import type { Page } from './page'; import { Waiter } from './waiter'; import type * as api from '../../types/types'; import type { HeadersArray, URLMatch } from '../common/types'; import { urlMatches } from '../utils/network'; import { MultiMap } from '../utils/multimap'; import { APIResponse } from './fetch'; import type { Serializable } from '../../types/structs'; import type { BrowserContext } from './browserContext'; import { HarRouter } from './harRouter'; import { kBrowserOrContextClosedError } from '../common/errors'; export type NetworkCookie = { name: string, value: string, domain: string, path: string, expires: number, httpOnly: boolean, secure: boolean, sameSite: 'Strict' | 'Lax' | 'None' }; export type SetNetworkCookieParam = { name: string, value: string, url?: string, domain?: string, path?: string, expires?: number, httpOnly?: boolean, secure?: boolean, sameSite?: 'Strict' | 'Lax' | 'None' }; type SerializedFallbackOverrides = { url?: string; method?: string; headers?: Headers; postDataBuffer?: Buffer; }; type FallbackOverrides = { url?: string; method?: string; headers?: Headers; postData?: string | Buffer | Serializable; }; export class Request extends ChannelOwner implements api.Request { private _redirectedFrom: Request | null = null; private _redirectedTo: Request | null = null; _failureText: string | null = null; private _provisionalHeaders: RawHeaders; private _actualHeadersPromise: Promise | undefined; _timing: ResourceTiming; private _fallbackOverrides: SerializedFallbackOverrides = {}; static from(request: channels.RequestChannel): Request { return (request as any)._object; } static fromNullable(request: channels.RequestChannel | undefined): Request | null { return request ? Request.from(request) : null; } constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.RequestInitializer) { super(parent, type, guid, initializer); this._redirectedFrom = Request.fromNullable(initializer.redirectedFrom); if (this._redirectedFrom) this._redirectedFrom._redirectedTo = this; this._provisionalHeaders = new RawHeaders(initializer.headers); this._fallbackOverrides.postDataBuffer = initializer.postData; this._timing = { startTime: 0, domainLookupStart: -1, domainLookupEnd: -1, connectStart: -1, secureConnectionStart: -1, connectEnd: -1, requestStart: -1, responseStart: -1, responseEnd: -1, }; } url(): string { return this._fallbackOverrides.url || this._initializer.url; } resourceType(): string { return this._initializer.resourceType; } method(): string { return this._fallbackOverrides.method || this._initializer.method; } postData(): string | null { return this._fallbackOverrides.postDataBuffer?.toString('utf-8') || null; } postDataBuffer(): Buffer | null { return this._fallbackOverrides.postDataBuffer || null; } postDataJSON(): Object | null { const postData = this.postData(); if (!postData) return null; const contentType = this.headers()['content-type']; if (contentType === 'application/x-www-form-urlencoded') { const entries: Record = {}; const parsed = new URLSearchParams(postData); for (const [k, v] of parsed.entries()) entries[k] = v; return entries; } try { return JSON.parse(postData); } catch (e) { throw new Error('POST data is not a valid JSON object: ' + postData); } } /** * @deprecated */ headers(): Headers { if (this._fallbackOverrides.headers) return RawHeaders._fromHeadersObjectLossy(this._fallbackOverrides.headers).headers(); return this._provisionalHeaders.headers(); } _context() { // TODO: make sure this works for service worker requests. return this.frame().page().context(); } _actualHeaders(): Promise { if (this._fallbackOverrides.headers) return Promise.resolve(RawHeaders._fromHeadersObjectLossy(this._fallbackOverrides.headers)); if (!this._actualHeadersPromise) { this._actualHeadersPromise = this._wrapApiCall(async () => { return new RawHeaders((await this._channel.rawRequestHeaders()).headers); }); } return this._actualHeadersPromise; } async allHeaders(): Promise { return (await this._actualHeaders()).headers(); } async headersArray(): Promise { return (await this._actualHeaders()).headersArray(); } async headerValue(name: string): Promise { return (await this._actualHeaders()).get(name); } async response(): Promise { return Response.fromNullable((await this._channel.response()).response); } async _internalResponse(): Promise { return this._wrapApiCall(async () => { return Response.fromNullable((await this._channel.response()).response); }, true); } frame(): Frame { if (!this._initializer.frame) { assert(this.serviceWorker()); throw new Error('Service Worker requests do not have an associated frame.'); } return Frame.from(this._initializer.frame); } serviceWorker(): Worker | null { return this._initializer.serviceWorker ? Worker.from(this._initializer.serviceWorker) : null; } isNavigationRequest(): boolean { return this._initializer.isNavigationRequest; } redirectedFrom(): Request | null { return this._redirectedFrom; } redirectedTo(): Request | null { return this._redirectedTo; } failure(): { errorText: string; } | null { if (this._failureText === null) return null; return { errorText: this._failureText }; } timing(): ResourceTiming { return this._timing; } async sizes(): Promise { const response = await this.response(); if (!response) throw new Error('Unable to fetch sizes for failed request'); return (await response._channel.sizes()).sizes; } _setResponseEndTiming(responseEndTiming: number) { this._timing.responseEnd = responseEndTiming; if (this._timing.responseStart === -1) this._timing.responseStart = responseEndTiming; } _finalRequest(): Request { return this._redirectedTo ? this._redirectedTo._finalRequest() : this; } _applyFallbackOverrides(overrides: FallbackOverrides) { if (overrides.url) this._fallbackOverrides.url = overrides.url; if (overrides.method) this._fallbackOverrides.method = overrides.method; if (overrides.headers) this._fallbackOverrides.headers = overrides.headers; if (isString(overrides.postData)) this._fallbackOverrides.postDataBuffer = Buffer.from(overrides.postData, 'utf-8'); else if (overrides.postData instanceof Buffer) this._fallbackOverrides.postDataBuffer = overrides.postData; else if (overrides.postData) this._fallbackOverrides.postDataBuffer = Buffer.from(JSON.stringify(overrides.postData), 'utf-8'); } _fallbackOverridesForContinue() { return this._fallbackOverrides; } _targetClosedPromise(): Promise { return this.serviceWorker()?._closedPromise || this.frame()._page?._closedOrCrashedPromise || new Promise(() => {}); } } export class Route extends ChannelOwner implements api.Route { private _handlingPromise: ManualPromise | null = null; static from(route: channels.RouteChannel): Route { return (route as any)._object; } constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.RouteInitializer) { super(parent, type, guid, initializer); } request(): Request { return Request.from(this._initializer.request); } private _raceWithTargetClose(promise: Promise): Promise { // When page closes or crashes, we catch any potential rejects from this Route. // Note that page could be missing when routing popup's initial request that // does not have a Page initialized just yet. return Promise.race([ promise, this.request()._targetClosedPromise(), ]); } _startHandling(): Promise { this._handlingPromise = new ManualPromise(); return this._handlingPromise; } async fallback(options: FallbackOverrides = {}) { this._checkNotHandled(); this.request()._applyFallbackOverrides(options); this._reportHandled(false); } async abort(errorCode?: string) { this._checkNotHandled(); await this._raceWithTargetClose(this._channel.abort({ errorCode })); this._reportHandled(true); } async _redirectNavigationRequest(url: string) { this._checkNotHandled(); await this._raceWithTargetClose(this._channel.redirectNavigationRequest({ url })); this._reportHandled(true); } async fetch(options: FallbackOverrides = {}) { return await this._wrapApiCall(async () => { const context = this.request()._context(); return context.request._innerFetch({ request: this.request(), data: options.postData, ...options }); }); } async fulfill(options: { response?: api.APIResponse, status?: number, headers?: Headers, contentType?: string, body?: string | Buffer, json?: any, path?: string } = {}) { this._checkNotHandled(); await this._wrapApiCall(async () => { await this._innerFulfill(options); this._reportHandled(true); }); } private async _innerFulfill(options: { response?: api.APIResponse, status?: number, headers?: Headers, contentType?: string, body?: string | Buffer, json?: any, path?: string } = {}): Promise { let fetchResponseUid; let { status: statusOption, headers: headersOption, body } = options; if (options.json !== undefined) { assert(options.body === undefined, 'Can specify either body or json parameters'); body = JSON.stringify(options.json); } if (options.response instanceof APIResponse) { statusOption ??= options.response.status(); headersOption ??= options.response.headers(); if (body === undefined && options.path === undefined) { if (options.response._request._connection === this._connection) fetchResponseUid = (options.response as APIResponse)._fetchUid(); else body = await options.response.body(); } } let isBase64 = false; let length = 0; if (options.path) { const buffer = await fs.promises.readFile(options.path); body = buffer.toString('base64'); isBase64 = true; length = buffer.length; } else if (isString(body)) { isBase64 = false; length = Buffer.byteLength(body); } else if (body) { length = body.length; body = body.toString('base64'); isBase64 = true; } const headers: Headers = {}; for (const header of Object.keys(headersOption || {})) headers[header.toLowerCase()] = String(headersOption![header]); if (options.contentType) headers['content-type'] = String(options.contentType); else if (options.json) headers['content-type'] = 'application/json'; else if (options.path) headers['content-type'] = mime.getType(options.path) || 'application/octet-stream'; if (length && !('content-length' in headers)) headers['content-length'] = String(length); await this._raceWithTargetClose(this._channel.fulfill({ status: statusOption || 200, headers: headersObjectToArray(headers), body, isBase64, fetchResponseUid })); } async continue(options: FallbackOverrides = {}) { this._checkNotHandled(); this.request()._applyFallbackOverrides(options); await this._innerContinue(); this._reportHandled(true); } _checkNotHandled() { if (!this._handlingPromise) throw new Error('Route is already handled!'); } _reportHandled(done: boolean) { const chain = this._handlingPromise!; this._handlingPromise = null; chain.resolve(done); } async _innerContinue(internal = false) { const options = this.request()._fallbackOverridesForContinue(); return await this._wrapApiCall(async () => { await this._raceWithTargetClose(this._channel.continue({ url: options.url, method: options.method, headers: options.headers ? headersObjectToArray(options.headers) : undefined, postData: options.postDataBuffer, })); }, !!internal); } } export type RouteHandlerCallback = (route: Route, request: Request) => Promise | void; export type ResourceTiming = { startTime: number; domainLookupStart: number; domainLookupEnd: number; connectStart: number; secureConnectionStart: number; connectEnd: number; requestStart: number; responseStart: number; responseEnd: number; }; export type RequestSizes = { requestBodySize: number; requestHeadersSize: number; responseBodySize: number; responseHeadersSize: number; }; export class Response extends ChannelOwner implements api.Response { private _provisionalHeaders: RawHeaders; private _actualHeadersPromise: Promise | undefined; private _request: Request; readonly _finishedPromise = new ManualPromise(); static from(response: channels.ResponseChannel): Response { return (response as any)._object; } static fromNullable(response: channels.ResponseChannel | undefined): Response | null { return response ? Response.from(response) : null; } constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.ResponseInitializer) { super(parent, type, guid, initializer); this._provisionalHeaders = new RawHeaders(initializer.headers); this._request = Request.from(this._initializer.request); Object.assign(this._request._timing, this._initializer.timing); } url(): string { return this._initializer.url; } ok(): boolean { // Status 0 is for file:// URLs return this._initializer.status === 0 || (this._initializer.status >= 200 && this._initializer.status <= 299); } status(): number { return this._initializer.status; } statusText(): string { return this._initializer.statusText; } fromServiceWorker(): boolean { return this._initializer.fromServiceWorker; } /** * @deprecated */ headers(): Headers { return this._provisionalHeaders.headers(); } async _actualHeaders(): Promise { if (!this._actualHeadersPromise) { this._actualHeadersPromise = (async () => { return new RawHeaders((await this._channel.rawResponseHeaders()).headers); })(); } return this._actualHeadersPromise; } async allHeaders(): Promise { return (await this._actualHeaders()).headers(); } async headersArray(): Promise { return (await this._actualHeaders()).headersArray().slice(); } async headerValue(name: string): Promise { return (await this._actualHeaders()).get(name); } async headerValues(name: string): Promise { return (await this._actualHeaders()).getAll(name); } async finished(): Promise { return Promise.race([ this._finishedPromise.then(() => null), this.request()._targetClosedPromise().then(() => { throw new Error(kBrowserOrContextClosedError); }), ]); } async body(): Promise { return (await this._channel.body()).binary; } async text(): Promise { const content = await this.body(); return content.toString('utf8'); } async json(): Promise { const content = await this.text(); return JSON.parse(content); } request(): Request { return this._request; } frame(): Frame { return this._request.frame(); } async serverAddr(): Promise { return (await this._channel.serverAddr()).value || null; } async securityDetails(): Promise { return (await this._channel.securityDetails()).value || null; } } export class WebSocket extends ChannelOwner implements api.WebSocket { private _page: Page; private _isClosed: boolean; static from(webSocket: channels.WebSocketChannel): WebSocket { return (webSocket as any)._object; } constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.WebSocketInitializer) { super(parent, type, guid, initializer); this._isClosed = false; this._page = parent as Page; this._channel.on('frameSent', event => { if (event.opcode === 1) this.emit(Events.WebSocket.FrameSent, { payload: event.data }); else if (event.opcode === 2) this.emit(Events.WebSocket.FrameSent, { payload: Buffer.from(event.data, 'base64') }); }); this._channel.on('frameReceived', event => { if (event.opcode === 1) this.emit(Events.WebSocket.FrameReceived, { payload: event.data }); else if (event.opcode === 2) this.emit(Events.WebSocket.FrameReceived, { payload: Buffer.from(event.data, 'base64') }); }); this._channel.on('socketError', ({ error }) => this.emit(Events.WebSocket.Error, error)); this._channel.on('close', () => { this._isClosed = true; this.emit(Events.WebSocket.Close, this); }); } url(): string { return this._initializer.url; } isClosed(): boolean { return this._isClosed; } async waitForEvent(event: string, optionsOrPredicate: WaitForEventOptions = {}): Promise { return this._wrapApiCall(async () => { const timeout = this._page._timeoutSettings.timeout(typeof optionsOrPredicate === 'function' ? {} : optionsOrPredicate); const predicate = typeof optionsOrPredicate === 'function' ? optionsOrPredicate : optionsOrPredicate.predicate; const waiter = Waiter.createForEvent(this, event); waiter.rejectOnTimeout(timeout, `Timeout ${timeout}ms exceeded while waiting for event "${event}"`); if (event !== Events.WebSocket.Error) waiter.rejectOnEvent(this, Events.WebSocket.Error, new Error('Socket error')); if (event !== Events.WebSocket.Close) waiter.rejectOnEvent(this, Events.WebSocket.Close, new Error('Socket closed')); waiter.rejectOnEvent(this._page, Events.Page.Close, new Error('Page closed')); const result = await waiter.waitForEvent(this, event, predicate as any); waiter.dispose(); return result; }); } } export function validateHeaders(headers: Headers) { for (const key of Object.keys(headers)) { const value = headers[key]; if (!Object.is(value, undefined) && !isString(value)) throw new Error(`Expected value of header "${key}" to be String, but "${typeof value}" is found.`); } } export class NetworkRouter { private _owner: Page | BrowserContext; private _baseURL: string | undefined; private _routes: RouteHandler[] = []; constructor(owner: Page | BrowserContext, baseURL: string | undefined) { this._owner = owner; this._baseURL = baseURL; } async route(url: URLMatch, handler: RouteHandlerCallback, options: { times?: number } = {}): Promise { this._routes.unshift(new RouteHandler(this._baseURL, url, handler, options.times)); await this._updateInterception(); } async routeFromHAR(har: string, options: { url?: string | RegExp, notFound?: 'abort' | 'fallback' } = {}): Promise { const harRouter = await HarRouter.create(this._owner._connection.localUtils(), har, options.notFound || 'abort'); await this.route(options.url || '**/*', route => harRouter.handleRoute(route)); this._owner.once('close', () => harRouter.dispose()); } async unroute(url: URLMatch, handler?: RouteHandlerCallback): Promise { this._routes = this._routes.filter(route => route.url !== url || (handler && route.handler !== handler)); await this._updateInterception(); } async handleRoute(route: Route) { const routeHandlers = this._routes.slice(); for (const routeHandler of routeHandlers) { if (!routeHandler.matches(route.request().url())) continue; if (routeHandler.willExpire()) this._routes.splice(this._routes.indexOf(routeHandler), 1); const handled = await routeHandler.handle(route); if (!this._routes.length) this._owner._wrapApiCall(() => this._updateInterception(), true).catch(() => {}); if (handled) return true; } return false; } private async _updateInterception() { const patterns: channels.BrowserContextSetNetworkInterceptionPatternsParams['patterns'] = []; let all = false; for (const handler of this._routes) { if (isString(handler.url)) patterns.push({ glob: handler.url }); else if (isRegExp(handler.url)) patterns.push({ regexSource: handler.url.source, regexFlags: handler.url.flags }); else all = true; } await this._owner._channel.setNetworkInterceptionPatterns(all ? { patterns: [{ glob: '**/*' }] } : { patterns }); } } class RouteHandler { private handledCount = 0; private readonly _baseURL: string | undefined; private readonly _times: number; readonly url: URLMatch; readonly handler: RouteHandlerCallback; constructor(baseURL: string | undefined, url: URLMatch, handler: RouteHandlerCallback, times: number = Number.MAX_SAFE_INTEGER) { this._baseURL = baseURL; this._times = times; this.url = url; this.handler = handler; } public matches(requestURL: string): boolean { return urlMatches(this._baseURL, requestURL, this.url); } public async handle(route: Route): Promise { ++this.handledCount; const handledPromise = route._startHandling(); // Extract handler into a variable to avoid [RouteHandler.handler] in the stack. const handler = this.handler; const [handled] = await Promise.all([ handledPromise, handler(route, route.request()), ]); return handled; } public willExpire(): boolean { return this.handledCount + 1 >= this._times; } } export class RawHeaders { private _headersArray: HeadersArray; private _headersMap = new MultiMap(); static _fromHeadersObjectLossy(headers: Headers): RawHeaders { const headersArray: HeadersArray = Object.entries(headers).map(([name, value]) => ({ name, value })).filter(header => header.value !== undefined); return new RawHeaders(headersArray); } constructor(headers: HeadersArray) { this._headersArray = headers; for (const header of headers) this._headersMap.set(header.name.toLowerCase(), header.value); } get(name: string): string | null { const values = this.getAll(name); if (!values || !values.length) return null; return values.join(name.toLowerCase() === 'set-cookie' ? '\n' : ', '); } getAll(name: string): string[] { return [...this._headersMap.get(name.toLowerCase())]; } headers(): Headers { const result: Headers = {}; for (const name of this._headersMap.keys()) result[name] = this.get(name)!; return result; } headersArray(): HeadersArray { return this._headersArray; } }