diff --git a/src/client/browserContext.ts b/src/client/browserContext.ts index 9dd5706be3..e7898afa3b 100644 --- a/src/client/browserContext.ts +++ b/src/client/browserContext.ts @@ -69,7 +69,7 @@ export class BrowserContext extends ChannelOwner this._onBinding(BindingCall.from(binding))); this._channel.on('close', () => this._onClose()); diff --git a/src/client/connection.ts b/src/client/connection.ts index 500581d041..4d52be1105 100644 --- a/src/client/connection.ts +++ b/src/client/connection.ts @@ -39,6 +39,7 @@ import { ParsedStackTrace } from '../utils/stackTrace'; import { Artifact } from './artifact'; import { EventEmitter } from 'events'; import { JsonPipe } from './jsonPipe'; +import { FetchRequest } from './fetch'; class Root extends ChannelOwner { constructor(connection: Connection) { @@ -216,6 +217,9 @@ export class Connection extends EventEmitter { case 'ElementHandle': result = new ElementHandle(parent, type, guid, initializer); break; + case 'FetchRequest': + result = new FetchRequest(parent, type, guid, initializer); + break; case 'Frame': result = new Frame(parent, type, guid, initializer); break; diff --git a/src/client/fetch.ts b/src/client/fetch.ts index 6239f000d4..ad4d57d4a4 100644 --- a/src/client/fetch.ts +++ b/src/client/fetch.ts @@ -18,7 +18,7 @@ import * as api from '../../types/types'; import { HeadersArray } from '../common/types'; import * as channels from '../protocol/channels'; import { assert, headersObjectToArray, isString, objectToArray } from '../utils/utils'; -import { BrowserContext } from './browserContext'; +import { ChannelOwner } from './channelOwner'; import * as network from './network'; import { RawHeaders } from './network'; import { Headers } from './types'; @@ -32,11 +32,13 @@ export type FetchOptions = { failOnStatusCode?: boolean, }; -export class FetchRequest implements api.FetchRequest { - private _context: BrowserContext; +export class FetchRequest extends ChannelOwner implements api.FetchRequest { + static from(channel: channels.FetchRequestChannel): FetchRequest { + return (channel as any)._object; + } - constructor(context: BrowserContext) { - this._context = context; + constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.FetchRequestInitializer) { + super(parent, type, guid, initializer); } async get( @@ -69,7 +71,7 @@ export class FetchRequest implements api.FetchRequest { } async fetch(urlOrRequest: string | api.Request, options: FetchOptions = {}): Promise { - return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => { + return this._wrapApiCall(async (channel: channels.FetchRequestChannel) => { const request: network.Request | undefined = (urlOrRequest instanceof network.Request) ? urlOrRequest as network.Request : undefined; assert(request || typeof urlOrRequest === 'string', 'First argument must be either URL string or Request'); const url = request ? request.url() : urlOrRequest as string; @@ -93,7 +95,7 @@ export class FetchRequest implements api.FetchRequest { }); if (result.error) throw new Error(`Request failed: ${result.error}`); - return new FetchResponse(this._context, result.response!); + return new FetchResponse(this, result.response!); }); } } @@ -101,10 +103,10 @@ export class FetchRequest implements api.FetchRequest { export class FetchResponse implements api.FetchResponse { private readonly _initializer: channels.FetchResponse; private readonly _headers: RawHeaders; - private readonly _context: BrowserContext; + private readonly _request: FetchRequest; - constructor(context: BrowserContext, initializer: channels.FetchResponse) { - this._context = context; + constructor(context: FetchRequest, initializer: channels.FetchResponse) { + this._request = context; this._initializer = initializer; this._headers = new RawHeaders(this._initializer.headers); } @@ -134,7 +136,7 @@ export class FetchResponse implements api.FetchResponse { } async body(): Promise { - return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => { + return this._request._wrapApiCall(async (channel: channels.FetchRequestChannel) => { const result = await channel.fetchResponseBody({ fetchUid: this._fetchUid() }); if (!result.binary) throw new Error('Response has been disposed'); @@ -153,7 +155,7 @@ export class FetchResponse implements api.FetchResponse { } async dispose(): Promise { - return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => { + return this._request._wrapApiCall(async (channel: channels.FetchRequestChannel) => { await channel.disposeFetchResponse({ fetchUid: this._fetchUid() }); }); } diff --git a/src/client/page.ts b/src/client/page.ts index b70956f66e..ad9d80e013 100644 --- a/src/client/page.ts +++ b/src/client/page.ts @@ -102,7 +102,7 @@ export class Page extends ChannelOwner { + return await this._wrapApiCall(async (channel: channels.PlaywrightChannel) => { + return FetchRequest.from((await channel.newRequest({})).request); + }); + } + _enablePortForwarding(redirectPortForTest?: number) { this._redirectPortForTest = redirectPortForTest; this._channel.on('socksRequested', ({ uid, host, port }) => this._onSocksRequested(uid, host, port)); diff --git a/src/dispatchers/browserContextDispatcher.ts b/src/dispatchers/browserContextDispatcher.ts index dbb57c8044..ecc85439aa 100644 --- a/src/dispatchers/browserContextDispatcher.ts +++ b/src/dispatchers/browserContextDispatcher.ts @@ -15,12 +15,11 @@ */ import { BrowserContext } from '../server/browserContext'; -import { Dispatcher, DispatcherScope, lookupDispatcher } from './dispatcher'; +import { Dispatcher, DispatcherScope, existingDispatcher, lookupDispatcher } from './dispatcher'; import { PageDispatcher, BindingCallDispatcher, WorkerDispatcher } from './pageDispatcher'; -import { playwrightFetch } from '../server/fetch'; import { FrameDispatcher } from './frameDispatcher'; import * as channels from '../protocol/channels'; -import { RouteDispatcher, RequestDispatcher, ResponseDispatcher } from './networkDispatchers'; +import { RouteDispatcher, RequestDispatcher, ResponseDispatcher, FetchRequestDispatcher } from './networkDispatchers'; import { CRBrowserContext } from '../server/chromium/crBrowser'; import { CDPSessionDispatcher } from './cdpSessionDispatcher'; import { RecorderSupplement } from '../server/supplements/recorderSupplement'; @@ -28,13 +27,15 @@ import { CallMetadata } from '../server/instrumentation'; import { ArtifactDispatcher } from './artifactDispatcher'; import { Artifact } from '../server/artifact'; import { Request, Response } from '../server/network'; -import { arrayToObject, headersArrayToObject } from '../utils/utils'; export class BrowserContextDispatcher extends Dispatcher implements channels.BrowserContextChannel { private _context: BrowserContext; constructor(scope: DispatcherScope, context: BrowserContext) { - super(scope, context, 'BrowserContext', { isChromium: context._browser.options.isChromium }, true); + super(scope, context, 'BrowserContext', { + isChromium: context._browser.options.isChromium, + fetchRequest: FetchRequestDispatcher.from(scope, context.fetchRequest), + }, true); this._context = context; // Note: when launching persistent context, dispatcher is created very late, // so we can already have pages, videos and everything else. @@ -57,6 +58,10 @@ export class BrowserContextDispatcher extends Dispatcher { this._dispatchEvent('close'); this._dispose(); + const fetch = existingDispatcher(this._context.fetchRequest); + // FetchRequestDispatcher is created in the browser rather then context scope but its + // lifetime is bound to the context dispatcher, so we manually dispose it here. + fetch._disposeDispatcher(); }); if (context._browser.options.name === 'chromium') { @@ -107,38 +112,6 @@ export class BrowserContextDispatcher extends Dispatcher { - const { fetchResponse, error } = await playwrightFetch(this._context, { - url: params.url, - params: arrayToObject(params.params), - method: params.method, - headers: params.headers ? headersArrayToObject(params.headers, false) : undefined, - postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined, - timeout: params.timeout, - failOnStatusCode: params.failOnStatusCode, - }); - let response; - if (fetchResponse) { - response = { - url: fetchResponse.url, - status: fetchResponse.status, - statusText: fetchResponse.statusText, - headers: fetchResponse.headers, - fetchUid: fetchResponse.fetchUid - }; - } - return { response, error }; - } - - async fetchResponseBody(params: channels.BrowserContextFetchResponseBodyParams): Promise { - const buffer = this._context.fetchResponses.get(params.fetchUid); - return { binary: buffer ? buffer.toString('base64') : undefined }; - } - - async disposeFetchResponse(params: channels.BrowserContextDisposeFetchResponseParams): Promise { - this._context.fetchResponses.delete(params.fetchUid); - } - async newPage(params: channels.BrowserContextNewPageParams, metadata: CallMetadata): Promise { return { page: lookupDispatcher(await this._context.newPage(metadata)) }; } diff --git a/src/dispatchers/networkDispatchers.ts b/src/dispatchers/networkDispatchers.ts index db3fa616d3..0e5516cf15 100644 --- a/src/dispatchers/networkDispatchers.ts +++ b/src/dispatchers/networkDispatchers.ts @@ -19,6 +19,8 @@ import * as channels from '../protocol/channels'; import { Dispatcher, DispatcherScope, lookupNullableDispatcher, existingDispatcher } from './dispatcher'; import { FrameDispatcher } from './frameDispatcher'; import { CallMetadata } from '../server/instrumentation'; +import { FetchRequest } from '../server/fetch'; +import { arrayToObject, headersArrayToObject } from '../utils/utils'; export class RequestDispatcher extends Dispatcher implements channels.RequestChannel { @@ -156,3 +158,56 @@ export class WebSocketDispatcher extends Dispatcher this._dispatchEvent('close', {})); } } + +export class FetchRequestDispatcher extends Dispatcher implements channels.FetchRequestChannel { + static from(scope: DispatcherScope, request: FetchRequest): FetchRequestDispatcher { + const result = existingDispatcher(request); + return result || new FetchRequestDispatcher(scope, request); + } + + static fromNullable(scope: DispatcherScope, request: FetchRequest | null): FetchRequestDispatcher | undefined { + return request ? FetchRequestDispatcher.from(scope, request) : undefined; + } + + private constructor(scope: DispatcherScope, request: FetchRequest) { + super(scope, request, 'FetchRequest', {}, true); + } + + async fetch(params: channels.FetchRequestFetchParams, metadata?: channels.Metadata): Promise { + const { fetchResponse, error } = await this._object.fetch({ + url: params.url, + params: arrayToObject(params.params), + method: params.method, + headers: params.headers ? headersArrayToObject(params.headers, false) : undefined, + postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined, + timeout: params.timeout, + failOnStatusCode: params.failOnStatusCode, + }); + let response; + if (fetchResponse) { + response = { + url: fetchResponse.url, + status: fetchResponse.status, + statusText: fetchResponse.statusText, + headers: fetchResponse.headers, + fetchUid: fetchResponse.fetchUid + }; + } + return { response, error }; + } + + async fetchResponseBody(params: channels.FetchRequestFetchResponseBodyParams, metadata?: channels.Metadata): Promise { + const buffer = this._object.fetchResponses.get(params.fetchUid); + return { binary: buffer ? buffer.toString('base64') : undefined }; + } + + async disposeFetchResponse(params: channels.FetchRequestDisposeFetchResponseParams, metadata?: channels.Metadata): Promise { + this._object.fetchResponses.delete(params.fetchUid); + } + + _disposeDispatcher() { + if (!this._disposed) + super._dispose(); + } +} + diff --git a/src/dispatchers/playwrightDispatcher.ts b/src/dispatchers/playwrightDispatcher.ts index e80b5869f3..4defffdbf3 100644 --- a/src/dispatchers/playwrightDispatcher.ts +++ b/src/dispatchers/playwrightDispatcher.ts @@ -26,6 +26,8 @@ import * as types from '../server/types'; import { SocksConnection, SocksConnectionClient } from '../utils/socksProxy'; import { createGuid } from '../utils/utils'; import { debugLogger } from '../utils/debugLogger'; +import { GlobalFetchRequest } from '../server/fetch'; +import { FetchRequestDispatcher } from './networkDispatchers'; export class PlaywrightDispatcher extends Dispatcher implements channels.PlaywrightChannel { private _socksProxy: SocksProxy | undefined; @@ -71,6 +73,11 @@ export class PlaywrightDispatcher extends Dispatcher { this._socksProxy?.sendSocketEnd(params); } + + async newRequest(params: channels.PlaywrightNewRequestParams, metadata?: channels.Metadata): Promise { + const request = new GlobalFetchRequest(this._object); + return { request: FetchRequestDispatcher.from(this._scope, request) }; + } } class SocksProxy implements SocksConnectionClient { diff --git a/src/protocol/channels.ts b/src/protocol/channels.ts index 9643ffbc5b..ad9b69f607 100644 --- a/src/protocol/channels.ts +++ b/src/protocol/channels.ts @@ -150,6 +150,54 @@ export type InterceptedResponse = { headers: NameValue[], }; +// ----------- FetchRequest ----------- +export type FetchRequestInitializer = {}; +export interface FetchRequestChannel extends Channel { + fetch(params: FetchRequestFetchParams, metadata?: Metadata): Promise; + fetchResponseBody(params: FetchRequestFetchResponseBodyParams, metadata?: Metadata): Promise; + disposeFetchResponse(params: FetchRequestDisposeFetchResponseParams, metadata?: Metadata): Promise; +} +export type FetchRequestFetchParams = { + url: string, + params?: NameValue[], + method?: string, + headers?: NameValue[], + postData?: Binary, + timeout?: number, + failOnStatusCode?: boolean, +}; +export type FetchRequestFetchOptions = { + params?: NameValue[], + method?: string, + headers?: NameValue[], + postData?: Binary, + timeout?: number, + failOnStatusCode?: boolean, +}; +export type FetchRequestFetchResult = { + response?: FetchResponse, + error?: string, +}; +export type FetchRequestFetchResponseBodyParams = { + fetchUid: string, +}; +export type FetchRequestFetchResponseBodyOptions = { + +}; +export type FetchRequestFetchResponseBodyResult = { + binary?: Binary, +}; +export type FetchRequestDisposeFetchResponseParams = { + fetchUid: string, +}; +export type FetchRequestDisposeFetchResponseOptions = { + +}; +export type FetchRequestDisposeFetchResponseResult = void; + +export interface FetchRequestEvents { +} + export type FetchResponse = { fetchUid: string, url: string, @@ -213,6 +261,7 @@ export interface PlaywrightChannel extends Channel { socksData(params: PlaywrightSocksDataParams, metadata?: Metadata): Promise; socksError(params: PlaywrightSocksErrorParams, metadata?: Metadata): Promise; socksEnd(params: PlaywrightSocksEndParams, metadata?: Metadata): Promise; + newRequest(params: PlaywrightNewRequestParams, metadata?: Metadata): Promise; } export type PlaywrightSocksRequestedEvent = { uid: string, @@ -266,6 +315,15 @@ export type PlaywrightSocksEndOptions = { }; export type PlaywrightSocksEndResult = void; +export type PlaywrightNewRequestParams = { + ignoreHTTPSErrors?: boolean, +}; +export type PlaywrightNewRequestOptions = { + ignoreHTTPSErrors?: boolean, +}; +export type PlaywrightNewRequestResult = { + request: FetchRequestChannel, +}; export interface PlaywrightEvents { 'socksRequested': PlaywrightSocksRequestedEvent; @@ -733,6 +791,7 @@ export interface EventTargetEvents { // ----------- BrowserContext ----------- export type BrowserContextInitializer = { isChromium: boolean, + fetchRequest: FetchRequestChannel, }; export interface BrowserContextChannel extends EventTargetChannel { on(event: 'bindingCall', callback: (params: BrowserContextBindingCallEvent) => void): this; @@ -753,9 +812,6 @@ export interface BrowserContextChannel extends EventTargetChannel { close(params?: BrowserContextCloseParams, metadata?: Metadata): Promise; cookies(params: BrowserContextCookiesParams, metadata?: Metadata): Promise; exposeBinding(params: BrowserContextExposeBindingParams, metadata?: Metadata): Promise; - fetch(params: BrowserContextFetchParams, metadata?: Metadata): Promise; - fetchResponseBody(params: BrowserContextFetchResponseBodyParams, metadata?: Metadata): Promise; - disposeFetchResponse(params: BrowserContextDisposeFetchResponseParams, metadata?: Metadata): Promise; grantPermissions(params: BrowserContextGrantPermissionsParams, metadata?: Metadata): Promise; newPage(params?: BrowserContextNewPageParams, metadata?: Metadata): Promise; setDefaultNavigationTimeoutNoReply(params: BrowserContextSetDefaultNavigationTimeoutNoReplyParams, metadata?: Metadata): Promise; @@ -855,43 +911,6 @@ export type BrowserContextExposeBindingOptions = { needsHandle?: boolean, }; export type BrowserContextExposeBindingResult = void; -export type BrowserContextFetchParams = { - url: string, - params?: NameValue[], - method?: string, - headers?: NameValue[], - postData?: Binary, - timeout?: number, - failOnStatusCode?: boolean, -}; -export type BrowserContextFetchOptions = { - params?: NameValue[], - method?: string, - headers?: NameValue[], - postData?: Binary, - timeout?: number, - failOnStatusCode?: boolean, -}; -export type BrowserContextFetchResult = { - response?: FetchResponse, - error?: string, -}; -export type BrowserContextFetchResponseBodyParams = { - fetchUid: string, -}; -export type BrowserContextFetchResponseBodyOptions = { - -}; -export type BrowserContextFetchResponseBodyResult = { - binary?: Binary, -}; -export type BrowserContextDisposeFetchResponseParams = { - fetchUid: string, -}; -export type BrowserContextDisposeFetchResponseOptions = { - -}; -export type BrowserContextDisposeFetchResponseResult = void; export type BrowserContextGrantPermissionsParams = { permissions: string[], origin?: string, diff --git a/src/protocol/protocol.yml b/src/protocol/protocol.yml index f529eb5269..4ce6e5df03 100644 --- a/src/protocol/protocol.yml +++ b/src/protocol/protocol.yml @@ -217,6 +217,39 @@ InterceptedResponse: items: NameValue +FetchRequest: + type: interface + + commands: + + fetch: + parameters: + url: string + params: + type: array? + items: NameValue + method: string? + headers: + type: array? + items: NameValue + postData: binary? + timeout: number? + failOnStatusCode: boolean? + returns: + response: FetchResponse? + error: string? + + fetchResponseBody: + parameters: + fetchUid: string + returns: + binary?: binary + + disposeFetchResponse: + parameters: + fetchUid: string + + FetchResponse: type: object properties: @@ -417,6 +450,12 @@ Playwright: parameters: uid: string + newRequest: + parameters: + ignoreHTTPSErrors: boolean? + returns: + request: FetchRequest + events: socksRequested: parameters: @@ -579,6 +618,7 @@ BrowserContext: initializer: isChromium: boolean + fetchRequest: FetchRequest commands: @@ -613,33 +653,6 @@ BrowserContext: name: string needsHandle: boolean? - fetch: - parameters: - url: string - params: - type: array? - items: NameValue - method: string? - headers: - type: array? - items: NameValue - postData: binary? - timeout: number? - failOnStatusCode: boolean? - returns: - response: FetchResponse? - error: string? - - fetchResponseBody: - parameters: - fetchUid: string - returns: - binary?: binary - - disposeFetchResponse: - parameters: - fetchUid: string - grantPermissions: parameters: permissions: diff --git a/src/protocol/validator.ts b/src/protocol/validator.ts index fdedf9c9b7..111732db1b 100644 --- a/src/protocol/validator.ts +++ b/src/protocol/validator.ts @@ -147,6 +147,21 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { statusText: tString, headers: tArray(tType('NameValue')), }); + scheme.FetchRequestFetchParams = tObject({ + url: tString, + params: tOptional(tArray(tType('NameValue'))), + method: tOptional(tString), + headers: tOptional(tArray(tType('NameValue'))), + postData: tOptional(tBinary), + timeout: tOptional(tNumber), + failOnStatusCode: tOptional(tBoolean), + }); + scheme.FetchRequestFetchResponseBodyParams = tObject({ + fetchUid: tString, + }); + scheme.FetchRequestDisposeFetchResponseParams = tObject({ + fetchUid: tString, + }); scheme.FetchResponse = tObject({ fetchUid: tString, url: tString, @@ -177,6 +192,9 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { scheme.PlaywrightSocksEndParams = tObject({ uid: tString, }); + scheme.PlaywrightNewRequestParams = tObject({ + ignoreHTTPSErrors: tOptional(tBoolean), + }); scheme.SelectorsRegisterParams = tObject({ name: tString, source: tString, @@ -392,21 +410,6 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { name: tString, needsHandle: tOptional(tBoolean), }); - scheme.BrowserContextFetchParams = tObject({ - url: tString, - params: tOptional(tArray(tType('NameValue'))), - method: tOptional(tString), - headers: tOptional(tArray(tType('NameValue'))), - postData: tOptional(tBinary), - timeout: tOptional(tNumber), - failOnStatusCode: tOptional(tBoolean), - }); - scheme.BrowserContextFetchResponseBodyParams = tObject({ - fetchUid: tString, - }); - scheme.BrowserContextDisposeFetchResponseParams = tObject({ - fetchUid: tString, - }); scheme.BrowserContextGrantPermissionsParams = tObject({ permissions: tArray(tString), origin: tOptional(tString), diff --git a/src/server/browserContext.ts b/src/server/browserContext.ts index c3c3d05e05..89bbfe9d5b 100644 --- a/src/server/browserContext.ts +++ b/src/server/browserContext.ts @@ -34,6 +34,7 @@ import { Tracing } from './trace/recorder/tracing'; import { HarRecorder } from './supplements/har/harRecorder'; import { RecorderSupplement } from './supplements/recorderSupplement'; import * as consoleApiSource from '../generated/consoleApiSource'; +import { BrowserContextFetchRequest } from './fetch'; export abstract class BrowserContext extends SdkObject { static Events = { @@ -63,7 +64,7 @@ export abstract class BrowserContext extends SdkObject { private _origins = new Set(); readonly _harRecorder: HarRecorder | undefined; readonly tracing: Tracing; - readonly fetchResponses: Map = new Map(); + readonly fetchRequest = new BrowserContextFetchRequest(this); constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) { super(browser, 'browser-context'); @@ -133,7 +134,7 @@ export abstract class BrowserContext extends SdkObject { this._closedStatus = 'closed'; this._deleteAllDownloads(); this._downloads.clear(); - this.fetchResponses.clear(); + this.fetchRequest.dispose(); if (this._isPersistentContext) this._onClosePersistent(); this._closePromiseFulfill!(new Error('Context closed')); @@ -382,12 +383,6 @@ export abstract class BrowserContext extends SdkObject { this.on(BrowserContext.Events.Page, installInPage); return Promise.all(this.pages().map(installInPage)); } - - storeFetchResponseBody(body: Buffer): string { - const uid = createGuid(); - this.fetchResponses.set(uid, body); - return uid; - } } export function assertBrowserContextIsNotOwned(context: BrowserContext) { diff --git a/src/server/fetch.ts b/src/server/fetch.ts index a15dc9dcde..0baa80c1d5 100644 --- a/src/server/fetch.ts +++ b/src/server/fetch.ts @@ -22,209 +22,303 @@ import * as https from 'https'; import { BrowserContext } from './browserContext'; import * as types from './types'; import { pipeline, Readable, Transform } from 'stream'; -import { monotonicTime } from '../utils/utils'; +import { createGuid, monotonicTime } from '../utils/utils'; +import { SdkObject } from './instrumentation'; +import { Playwright } from './playwright'; +import { HeadersArray, ProxySettings } from './types'; +import { HTTPCredentials } from '../../types/types'; +import { TimeoutSettings } from '../utils/timeoutSettings'; -export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: Omit & { fetchUid: string }, error?: string}> { - try { - const headers: { [name: string]: string } = {}; - if (params.headers) { - for (const [name, value] of Object.entries(params.headers)) - headers[name.toLowerCase()] = value; - } - headers['user-agent'] ??= context._options.userAgent || context._browser.userAgent(); - headers['accept'] ??= '*/*'; - headers['accept-encoding'] ??= 'gzip,deflate,br'; - if (context._options.extraHTTPHeaders) { - for (const {name, value} of context._options.extraHTTPHeaders) - headers[name.toLowerCase()] = value; - } +type FetchRequestOptions = { + userAgent: string; + extraHTTPHeaders?: HeadersArray; + httpCredentials?: HTTPCredentials; + proxy?: ProxySettings; + timeoutSettings: TimeoutSettings; + ignoreHTTPSErrors?: boolean; + baseURL?: string; +}; - const method = params.method?.toUpperCase() || 'GET'; - const proxy = context._options.proxy || context._browser.options.proxy; - let agent; - if (proxy) { - // TODO: support bypass proxy - const proxyOpts = url.parse(proxy.server); - if (proxy.username) - proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`; - agent = new HttpsProxyAgent(proxyOpts); - } +export abstract class FetchRequest extends SdkObject { + readonly fetchResponses: Map = new Map(); - const timeout = context._timeoutSettings.timeout(params); - const deadline = monotonicTime() + timeout; - - const options: https.RequestOptions & { maxRedirects: number, deadline: number } = { - method, - headers, - agent, - maxRedirects: 20, - timeout, - deadline - }; - // rejectUnauthorized = undefined is treated as true in node 12. - if (context._options.ignoreHTTPSErrors) - options.rejectUnauthorized = false; - - const requestUrl = new URL(params.url, context._options.baseURL); - if (params.params) { - for (const [name, value] of Object.entries(params.params)) - requestUrl.searchParams.set(name, value); - } - - const fetchResponse = await sendRequest(context, requestUrl, options, params.postData); - const fetchUid = context.storeFetchResponseBody(fetchResponse.body); - if (params.failOnStatusCode && (fetchResponse.status < 200 || fetchResponse.status >= 400)) - return { error: `${fetchResponse.status} ${fetchResponse.statusText}` }; - return { fetchResponse: { ...fetchResponse, fetchUid } }; - } catch (e) { - return { error: String(e) }; + constructor(parent: SdkObject) { + super(parent, 'fetchRequest'); } -} -async function updateCookiesFromHeader(context: BrowserContext, responseUrl: string, setCookie: string[]) { - const url = new URL(responseUrl); - // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4 - const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/'); - const cookies: types.SetNetworkCookieParam[] = []; - for (const header of setCookie) { - // Decode cookie value? - const cookie: types.SetNetworkCookieParam | null = parseCookie(header); - if (!cookie) - continue; - if (!cookie.domain) - cookie.domain = url.hostname; - if (!canSetCookie(cookie.domain!, url.hostname)) - continue; - // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4 - if (!cookie.path || !cookie.path.startsWith('/')) - cookie.path = defaultPath; - cookies.push(cookie); + dispose() { + this.fetchResponses.clear(); } - if (cookies.length) - await context.addCookies(cookies); -} -async function updateRequestCookieHeader(context: BrowserContext, url: URL, options: http.RequestOptions) { - if (options.headers!['cookie'] !== undefined) - return; - const cookies = await context.cookies(url.toString()); - if (cookies.length) { - const valueArray = cookies.map(c => `${c.name}=${c.value}`); - options.headers!['cookie'] = valueArray.join('; '); + abstract _defaultOptions(): FetchRequestOptions; + abstract _addCookies(cookies: types.SetNetworkCookieParam[]): Promise; + abstract _cookies(url: string): Promise; + + private _storeResponseBody(body: Buffer): string { + const uid = createGuid(); + this.fetchResponses.set(uid, body); + return uid; } -} -async function sendRequest(context: BrowserContext, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise{ - await updateRequestCookieHeader(context, url, options); - return new Promise((fulfill, reject) => { - const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest) - = (url.protocol === 'https:' ? https : http).request; - const request = requestConstructor(url, options, async response => { - if (response.headers['set-cookie']) - await updateCookiesFromHeader(context, response.url || url.toString(), response.headers['set-cookie']); - if (redirectStatus.includes(response.statusCode!)) { - if (!options.maxRedirects) { - reject(new Error('Max redirect count exceeded')); - request.abort(); - return; - } - const headers = { ...options.headers }; - delete headers[`cookie`]; + async fetch(params: types.FetchOptions): Promise<{fetchResponse?: Omit & { fetchUid: string }, error?: string}> { + try { + const headers: { [name: string]: string } = {}; + const defaults = this._defaultOptions(); + headers['user-agent'] = defaults.userAgent; + headers['accept'] = '*/*'; + headers['accept-encoding'] = 'gzip,deflate,br'; - // HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch) - const status = response.statusCode!; - let method = options.method!; - if ((status === 301 || status === 302) && method === 'POST' || - status === 303 && !['GET', 'HEAD'].includes(method)) { - method = 'GET'; - postData = undefined; - delete headers[`content-encoding`]; - delete headers[`content-language`]; - delete headers[`content-location`]; - delete headers[`content-type`]; - } - - const redirectOptions: http.RequestOptions & { maxRedirects: number, deadline: number } = { - method, - headers, - agent: options.agent, - maxRedirects: options.maxRedirects - 1, - timeout: options.timeout, - deadline: options.deadline - }; - - // HTTP-redirect fetch step 4: If locationURL is null, then return response. - if (response.headers.location) { - const locationURL = new URL(response.headers.location, url); - fulfill(sendRequest(context, locationURL, redirectOptions, postData)); - request.abort(); - return; - } - } - if (response.statusCode === 401 && !options.headers!['authorization']) { - const auth = response.headers['www-authenticate']; - const credentials = context._options.httpCredentials; - if (auth?.trim().startsWith('Basic ') && credentials) { - const {username, password} = credentials; - const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64'); - options.headers!['authorization'] = `Basic ${encoded}`; - fulfill(sendRequest(context, url, options, postData)); - request.abort(); - return; - } - } - response.on('aborted', () => reject(new Error('aborted'))); - - let body: Readable = response; - let transform: Transform | undefined; - const encoding = response.headers['content-encoding']; - if (encoding === 'gzip' || encoding === 'x-gzip') { - transform = zlib.createGunzip({ - flush: zlib.constants.Z_SYNC_FLUSH, - finishFlush: zlib.constants.Z_SYNC_FLUSH - }); - } else if (encoding === 'br') { - transform = zlib.createBrotliDecompress(); - } else if (encoding === 'deflate') { - transform = zlib.createInflate(); - } - if (transform) { - body = pipeline(response, transform, e => { - if (e) - reject(new Error(`failed to decompress '${encoding}' encoding: ${e}`)); - }); + if (defaults.extraHTTPHeaders) { + for (const {name, value} of defaults.extraHTTPHeaders) + headers[name.toLowerCase()] = value; } - const chunks: Buffer[] = []; - body.on('data', chunk => chunks.push(chunk)); - body.on('end', () => { - const body = Buffer.concat(chunks); - fulfill({ - url: response.url || url.toString(), - status: response.statusCode || 0, - statusText: response.statusMessage || '', - headers: toHeadersArray(response.rawHeaders), - body - }); - }); - body.on('error',reject); - }); - request.on('error', reject); - const rejectOnTimeout = () => { - reject(new Error(`Request timed out after ${options.timeout}ms`)); - request.abort(); - }; - const remaining = options.deadline - monotonicTime(); - if (remaining <= 0) { - rejectOnTimeout(); + if (params.headers) { + for (const [name, value] of Object.entries(params.headers)) + headers[name.toLowerCase()] = value; + } + + const method = params.method?.toUpperCase() || 'GET'; + const proxy = defaults.proxy; + let agent; + if (proxy) { + // TODO: support bypass proxy + const proxyOpts = url.parse(proxy.server); + if (proxy.username) + proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`; + agent = new HttpsProxyAgent(proxyOpts); + } + + const timeout = defaults.timeoutSettings.timeout(params); + const deadline = monotonicTime() + timeout; + + const options: https.RequestOptions & { maxRedirects: number, deadline: number } = { + method, + headers, + agent, + maxRedirects: 20, + timeout, + deadline + }; + // rejectUnauthorized = undefined is treated as true in node 12. + if (defaults.ignoreHTTPSErrors) + options.rejectUnauthorized = false; + + const requestUrl = new URL(params.url, defaults.baseURL); + if (params.params) { + for (const [name, value] of Object.entries(params.params)) + requestUrl.searchParams.set(name, value); + } + + const fetchResponse = await this._sendRequest(requestUrl, options, params.postData); + const fetchUid = this._storeResponseBody(fetchResponse.body); + if (params.failOnStatusCode && (fetchResponse.status < 200 || fetchResponse.status >= 400)) + return { error: `${fetchResponse.status} ${fetchResponse.statusText}` }; + return { fetchResponse: { ...fetchResponse, fetchUid } }; + } catch (e) { + return { error: String(e) }; + } + } + + private async _updateCookiesFromHeader(responseUrl: string, setCookie: string[]) { + const url = new URL(responseUrl); + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4 + const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/'); + const cookies: types.SetNetworkCookieParam[] = []; + for (const header of setCookie) { + // Decode cookie value? + const cookie: types.SetNetworkCookieParam | null = parseCookie(header); + if (!cookie) + continue; + if (!cookie.domain) + cookie.domain = url.hostname; + if (!canSetCookie(cookie.domain!, url.hostname)) + continue; + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4 + if (!cookie.path || !cookie.path.startsWith('/')) + cookie.path = defaultPath; + cookies.push(cookie); + } + if (cookies.length) + await this._addCookies(cookies); + } + + private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) { + if (options.headers!['cookie'] !== undefined) return; + const cookies = await this._cookies(url.toString()); + if (cookies.length) { + const valueArray = cookies.map(c => `${c.name}=${c.value}`); + options.headers!['cookie'] = valueArray.join('; '); } - request.setTimeout(remaining, rejectOnTimeout); - if (postData) - request.write(postData); - request.end(); - }); + } + + private async _sendRequest(url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise{ + await this._updateRequestCookieHeader(url, options); + return new Promise((fulfill, reject) => { + const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest) + = (url.protocol === 'https:' ? https : http).request; + const request = requestConstructor(url, options, async response => { + if (response.headers['set-cookie']) + await this._updateCookiesFromHeader(response.url || url.toString(), response.headers['set-cookie']); + if (redirectStatus.includes(response.statusCode!)) { + if (!options.maxRedirects) { + reject(new Error('Max redirect count exceeded')); + request.abort(); + return; + } + const headers = { ...options.headers }; + delete headers[`cookie`]; + + // HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch) + const status = response.statusCode!; + let method = options.method!; + if ((status === 301 || status === 302) && method === 'POST' || + status === 303 && !['GET', 'HEAD'].includes(method)) { + method = 'GET'; + postData = undefined; + delete headers[`content-encoding`]; + delete headers[`content-language`]; + delete headers[`content-location`]; + delete headers[`content-type`]; + } + + const redirectOptions: http.RequestOptions & { maxRedirects: number, deadline: number } = { + method, + headers, + agent: options.agent, + maxRedirects: options.maxRedirects - 1, + timeout: options.timeout, + deadline: options.deadline + }; + + // HTTP-redirect fetch step 4: If locationURL is null, then return response. + if (response.headers.location) { + const locationURL = new URL(response.headers.location, url); + fulfill(this._sendRequest(locationURL, redirectOptions, postData)); + request.abort(); + return; + } + } + if (response.statusCode === 401 && !options.headers!['authorization']) { + const auth = response.headers['www-authenticate']; + const credentials = this._defaultOptions().httpCredentials; + if (auth?.trim().startsWith('Basic ') && credentials) { + const {username, password} = credentials; + const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64'); + options.headers!['authorization'] = `Basic ${encoded}`; + fulfill(this._sendRequest(url, options, postData)); + request.abort(); + return; + } + } + response.on('aborted', () => reject(new Error('aborted'))); + + let body: Readable = response; + let transform: Transform | undefined; + const encoding = response.headers['content-encoding']; + if (encoding === 'gzip' || encoding === 'x-gzip') { + transform = zlib.createGunzip({ + flush: zlib.constants.Z_SYNC_FLUSH, + finishFlush: zlib.constants.Z_SYNC_FLUSH + }); + } else if (encoding === 'br') { + transform = zlib.createBrotliDecompress(); + } else if (encoding === 'deflate') { + transform = zlib.createInflate(); + } + if (transform) { + body = pipeline(response, transform, e => { + if (e) + reject(new Error(`failed to decompress '${encoding}' encoding: ${e}`)); + }); + } + + const chunks: Buffer[] = []; + body.on('data', chunk => chunks.push(chunk)); + body.on('end', () => { + const body = Buffer.concat(chunks); + fulfill({ + url: response.url || url.toString(), + status: response.statusCode || 0, + statusText: response.statusMessage || '', + headers: toHeadersArray(response.rawHeaders), + body + }); + }); + body.on('error',reject); + }); + request.on('error', reject); + const rejectOnTimeout = () => { + reject(new Error(`Request timed out after ${options.timeout}ms`)); + request.abort(); + }; + const remaining = options.deadline - monotonicTime(); + if (remaining <= 0) { + rejectOnTimeout(); + return; + } + request.setTimeout(remaining, rejectOnTimeout); + if (postData) + request.write(postData); + request.end(); + }); + } +} + +export class BrowserContextFetchRequest extends FetchRequest { + private readonly _context: BrowserContext; + + constructor(context: BrowserContext) { + super(context); + this._context = context; + } + + _defaultOptions(): FetchRequestOptions { + return { + userAgent: this._context._options.userAgent || this._context._browser.userAgent(), + extraHTTPHeaders: this._context._options.extraHTTPHeaders, + httpCredentials: this._context._options.httpCredentials, + proxy: this._context._options.proxy || this._context._browser.options.proxy, + timeoutSettings: this._context._timeoutSettings, + ignoreHTTPSErrors: this._context._options.ignoreHTTPSErrors, + baseURL: this._context._options.baseURL, + }; + } + + async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise { + await this._context.addCookies(cookies); + } + + async _cookies(url: string): Promise { + return await this._context.cookies(url); + } +} + + +export class GlobalFetchRequest extends FetchRequest { + constructor(playwright: Playwright) { + super(playwright); + } + + _defaultOptions(): FetchRequestOptions { + return { + userAgent: '', + extraHTTPHeaders: undefined, + proxy: undefined, + timeoutSettings: new TimeoutSettings(), + ignoreHTTPSErrors: false, + baseURL: undefined, + }; + } + + async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise { + } + + async _cookies(url: string): Promise { + return []; + } } function toHeadersArray(rawHeaders: string[]): types.HeadersArray { diff --git a/src/server/network.ts b/src/server/network.ts index b3890d2f8f..58c13b33b1 100644 --- a/src/server/network.ts +++ b/src/server/network.ts @@ -228,7 +228,7 @@ export class Route extends SdkObject { if (body === undefined) { if (overrides.fetchResponseUid) { const context = this._request.frame()._page._browserContext; - const buffer = context.fetchResponses.get(overrides.fetchResponseUid); + const buffer = context.fetchRequest.fetchResponses.get(overrides.fetchResponseUid); assert(buffer, 'Fetch response has been disposed'); body = buffer.toString('utf8'); isBase64 = false; diff --git a/tests/browsercontext-fetch.spec.ts b/tests/browsercontext-fetch.spec.ts index 298937c76b..2b2db8c1f8 100644 --- a/tests/browsercontext-fetch.spec.ts +++ b/tests/browsercontext-fetch.spec.ts @@ -40,6 +40,19 @@ it.afterAll(() => { http.globalAgent = prevAgent; }); +it('global get should work', async ({playwright, context, server}) => { + const request = await playwright._newRequest(); + const response = await request.get(server.PREFIX + '/simple.json'); + expect(response.url()).toBe(server.PREFIX + '/simple.json'); + expect(response.status()).toBe(200); + expect(response.statusText()).toBe('OK'); + expect(response.ok()).toBeTruthy(); + expect(response.url()).toBe(server.PREFIX + '/simple.json'); + expect(response.headers()['content-type']).toBe('application/json; charset=utf-8'); + expect(response.headersArray()).toContainEqual({ name: 'Content-Type', value: 'application/json; charset=utf-8' }); + expect(await response.text()).toBe('{"foo": "bar"}\n'); +}); + it('get should work', async ({context, server}) => { const response = await context._request.get(server.PREFIX + '/simple.json'); expect(response.url()).toBe(server.PREFIX + '/simple.json'); diff --git a/tests/channels.spec.ts b/tests/channels.spec.ts index 3b35283b51..5206a41a6b 100644 --- a/tests/channels.spec.ts +++ b/tests/channels.spec.ts @@ -57,6 +57,7 @@ it('should scope context handles', async ({browserType, browserOptions, server}) { _guid: 'request', objects: [] }, { _guid: 'response', objects: [] }, ]}, + { _guid: 'fetchRequest', objects: [] } ] }, ] }, { _guid: 'electron', objects: [] }, @@ -140,7 +141,8 @@ it('should scope browser handles', async ({browserType, browserOptions}) => { { _guid: 'browser-type', objects: [ { _guid: 'browser', objects: [ - { _guid: 'browser-context', objects: [] } + { _guid: 'browser-context', objects: [] }, + { _guid: 'fetchRequest', objects: [] } ] }, ] diff --git a/types/types.d.ts b/types/types.d.ts index 0219cdf053..90ff4695b9 100644 --- a/types/types.d.ts +++ b/types/types.d.ts @@ -10668,6 +10668,7 @@ export const firefox: BrowserType; export const webkit: BrowserType; export const _electron: Electron; export const _android: Android; +export const _newRequest: () => Promise; // This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459 export {}; diff --git a/utils/generate_types/overrides.d.ts b/utils/generate_types/overrides.d.ts index d179892a9b..0bd921c2cb 100644 --- a/utils/generate_types/overrides.d.ts +++ b/utils/generate_types/overrides.d.ts @@ -348,6 +348,7 @@ export const firefox: BrowserType; export const webkit: BrowserType; export const _electron: Electron; export const _android: Android; +export const _newRequest: () => Promise; // This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459 export {};