yujunjun/playwright
1
0
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-06-26 21:40:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(fetch): introduce global fetch request (#8927)

Browse Source
This commit is contained in:
Yury Semikhatsky 2021-09-14 18:31:35 -07:00 committed by GitHub
parent 5253a7eb54
commit c58f34fb2e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 520 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/client/browserContext.ts
View File

@ -69,7 +69,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
this._browser = parent;
this._isChromium = this._browser?._name === 'chromium';
this.tracing = new Tracing(this);
this._request = new FetchRequest(this);
this._request = FetchRequest.from(initializer.fetchRequest);
this._channel.on('bindingCall', ({binding}) => this._onBinding(BindingCall.from(binding)));
this._channel.on('close', () => this._onClose());

4
src/client/connection.ts
View File

@ -39,6 +39,7 @@ import { ParsedStackTrace } from '../utils/stackTrace';
import { Artifact } from './artifact';
import { EventEmitter } from 'events';
import { JsonPipe } from './jsonPipe';
import { FetchRequest } from './fetch';
class Root extends ChannelOwner<channels.RootChannel, {}> {
constructor(connection: Connection) {
@ -216,6 +217,9 @@ export class Connection extends EventEmitter {
case 'ElementHandle':
result = new ElementHandle(parent, type, guid, initializer);
break;
case 'FetchRequest':
result = new FetchRequest(parent, type, guid, initializer);
break;
case 'Frame':
result = new Frame(parent, type, guid, initializer);
break;

26
src/client/fetch.ts
View File

@ -18,7 +18,7 @@ import * as api from '../../types/types';
import { HeadersArray } from '../common/types';
import * as channels from '../protocol/channels';
import { assert, headersObjectToArray, isString, objectToArray } from '../utils/utils';
import { BrowserContext } from './browserContext';
import { ChannelOwner } from './channelOwner';
import * as network from './network';
import { RawHeaders } from './network';
import { Headers } from './types';
@ -32,11 +32,13 @@ export type FetchOptions = {
failOnStatusCode?: boolean,
};
export class FetchRequest implements api.FetchRequest {
private _context: BrowserContext;
export class FetchRequest extends ChannelOwner<channels.FetchRequestChannel, channels.FetchRequestInitializer> implements api.FetchRequest {
static from(channel: channels.FetchRequestChannel): FetchRequest {
return (channel as any)._object;
}
constructor(context: BrowserContext) {
this._context = context;
constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.FetchRequestInitializer) {
super(parent, type, guid, initializer);
}
async get(
@ -69,7 +71,7 @@ export class FetchRequest implements api.FetchRequest {
}
async fetch(urlOrRequest: string | api.Request, options: FetchOptions = {}): Promise<FetchResponse> {
return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
return this._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
const request: network.Request | undefined = (urlOrRequest instanceof network.Request) ? urlOrRequest as network.Request : undefined;
assert(request || typeof urlOrRequest === 'string', 'First argument must be either URL string or Request');
const url = request ? request.url() : urlOrRequest as string;
@ -93,7 +95,7 @@ export class FetchRequest implements api.FetchRequest {
});
if (result.error)
throw new Error(`Request failed: ${result.error}`);
return new FetchResponse(this._context, result.response!);
return new FetchResponse(this, result.response!);
});
}
}
@ -101,10 +103,10 @@ export class FetchRequest implements api.FetchRequest {
export class FetchResponse implements api.FetchResponse {
private readonly _initializer: channels.FetchResponse;
private readonly _headers: RawHeaders;
private readonly _context: BrowserContext;
private readonly _request: FetchRequest;
constructor(context: BrowserContext, initializer: channels.FetchResponse) {
this._context = context;
constructor(context: FetchRequest, initializer: channels.FetchResponse) {
this._request = context;
this._initializer = initializer;
this._headers = new RawHeaders(this._initializer.headers);
}
@ -134,7 +136,7 @@ export class FetchResponse implements api.FetchResponse {
}
async body(): Promise<Buffer> {
return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
return this._request._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
const result = await channel.fetchResponseBody({ fetchUid: this._fetchUid() });
if (!result.binary)
throw new Error('Response has been disposed');
@ -153,7 +155,7 @@ export class FetchResponse implements api.FetchResponse {
}
async dispose(): Promise<void> {
return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
return this._request._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
await channel.disposeFetchResponse({ fetchUid: this._fetchUid() });
});
}

2
src/client/page.ts
View File

@ -102,7 +102,7 @@ export class Page extends ChannelOwner<channels.PageChannel, channels.PageInitia
this.accessibility = new Accessibility(this._channel);
this.keyboard = new Keyboard(this);
this.mouse = new Mouse(this);
this._request = new FetchRequest(this._browserContext);
this._request = this._browserContext._request;
this.touchscreen = new Touchscreen(this);
this._mainFrame = Frame.from(initializer.mainFrame);

7
src/client/playwright.ts
View File

@ -24,6 +24,7 @@ import { Android } from './android';
import { BrowserType } from './browserType';
import { ChannelOwner } from './channelOwner';
import { Electron } from './electron';
import { FetchRequest } from './fetch';
import { Selectors, SelectorsOwner, sharedSelectors } from './selectors';
import { Size } from './types';
const dnsLookupAsync = util.promisify(dns.lookup);
@ -68,6 +69,12 @@ export class Playwright extends ChannelOwner<channels.PlaywrightChannel, channel
this.selectors._addChannel(this._selectorsOwner);
}
async _newRequest(options?: {}): Promise<FetchRequest> {
return await this._wrapApiCall(async (channel: channels.PlaywrightChannel) => {
return FetchRequest.from((await channel.newRequest({})).request);
});
}
_enablePortForwarding(redirectPortForTest?: number) {
this._redirectPortForTest = redirectPortForTest;
this._channel.on('socksRequested', ({ uid, host, port }) => this._onSocksRequested(uid, host, port));

47
src/dispatchers/browserContextDispatcher.ts
View File

@ -15,12 +15,11 @@
*/
import { BrowserContext } from '../server/browserContext';
import { Dispatcher, DispatcherScope, lookupDispatcher } from './dispatcher';
import { Dispatcher, DispatcherScope, existingDispatcher, lookupDispatcher } from './dispatcher';
import { PageDispatcher, BindingCallDispatcher, WorkerDispatcher } from './pageDispatcher';
import { playwrightFetch } from '../server/fetch';
import { FrameDispatcher } from './frameDispatcher';
import * as channels from '../protocol/channels';
import { RouteDispatcher, RequestDispatcher, ResponseDispatcher } from './networkDispatchers';
import { RouteDispatcher, RequestDispatcher, ResponseDispatcher, FetchRequestDispatcher } from './networkDispatchers';
import { CRBrowserContext } from '../server/chromium/crBrowser';
import { CDPSessionDispatcher } from './cdpSessionDispatcher';
import { RecorderSupplement } from '../server/supplements/recorderSupplement';
@ -28,13 +27,15 @@ import { CallMetadata } from '../server/instrumentation';
import { ArtifactDispatcher } from './artifactDispatcher';
import { Artifact } from '../server/artifact';
import { Request, Response } from '../server/network';
import { arrayToObject, headersArrayToObject } from '../utils/utils';
export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channels.BrowserContextInitializer, channels.BrowserContextEvents> implements channels.BrowserContextChannel {
private _context: BrowserContext;
constructor(scope: DispatcherScope, context: BrowserContext) {
super(scope, context, 'BrowserContext', { isChromium: context._browser.options.isChromium }, true);
super(scope, context, 'BrowserContext', {
isChromium: context._browser.options.isChromium,
fetchRequest: FetchRequestDispatcher.from(scope, context.fetchRequest),
}, true);
this._context = context;
// Note: when launching persistent context, dispatcher is created very late,
// so we can already have pages, videos and everything else.
@ -57,6 +58,10 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
context.on(BrowserContext.Events.Close, () => {
this._dispatchEvent('close');
this._dispose();
const fetch = existingDispatcher<FetchRequestDispatcher>(this._context.fetchRequest);
// FetchRequestDispatcher is created in the browser rather then context scope but its
// lifetime is bound to the context dispatcher, so we manually dispose it here.
fetch._disposeDispatcher();
});
if (context._browser.options.name === 'chromium') {
@ -107,38 +112,6 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
});
}
async fetch(params: channels.BrowserContextFetchParams): Promise<channels.BrowserContextFetchResult> {
const { fetchResponse, error } = await playwrightFetch(this._context, {
url: params.url,
params: arrayToObject(params.params),
method: params.method,
headers: params.headers ? headersArrayToObject(params.headers, false) : undefined,
postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined,
timeout: params.timeout,
failOnStatusCode: params.failOnStatusCode,
});
let response;
if (fetchResponse) {
response = {
url: fetchResponse.url,
status: fetchResponse.status,
statusText: fetchResponse.statusText,
headers: fetchResponse.headers,
fetchUid: fetchResponse.fetchUid
};
}
return { response, error };
}
async fetchResponseBody(params: channels.BrowserContextFetchResponseBodyParams): Promise<channels.BrowserContextFetchResponseBodyResult> {
const buffer = this._context.fetchResponses.get(params.fetchUid);
return { binary: buffer ? buffer.toString('base64') : undefined };
}
async disposeFetchResponse(params: channels.BrowserContextDisposeFetchResponseParams): Promise<channels.BrowserContextDisposeFetchResponseResult> {
this._context.fetchResponses.delete(params.fetchUid);
}
async newPage(params: channels.BrowserContextNewPageParams, metadata: CallMetadata): Promise<channels.BrowserContextNewPageResult> {
return { page: lookupDispatcher<PageDispatcher>(await this._context.newPage(metadata)) };
}

55
src/dispatchers/networkDispatchers.ts
View File

@ -19,6 +19,8 @@ import * as channels from '../protocol/channels';
import { Dispatcher, DispatcherScope, lookupNullableDispatcher, existingDispatcher } from './dispatcher';
import { FrameDispatcher } from './frameDispatcher';
import { CallMetadata } from '../server/instrumentation';
import { FetchRequest } from '../server/fetch';
import { arrayToObject, headersArrayToObject } from '../utils/utils';
export class RequestDispatcher extends Dispatcher<Request, channels.RequestInitializer, channels.RequestEvents> implements channels.RequestChannel {
@ -156,3 +158,56 @@ export class WebSocketDispatcher extends Dispatcher<WebSocket, channels.WebSocke
webSocket.on(WebSocket.Events.Close, () => this._dispatchEvent('close', {}));
}
}
export class FetchRequestDispatcher extends Dispatcher<FetchRequest, channels.FetchRequestInitializer, channels.FetchRequestEvents> implements channels.FetchRequestChannel {
static from(scope: DispatcherScope, request: FetchRequest): FetchRequestDispatcher {
const result = existingDispatcher<FetchRequestDispatcher>(request);
return result || new FetchRequestDispatcher(scope, request);
}
static fromNullable(scope: DispatcherScope, request: FetchRequest | null): FetchRequestDispatcher | undefined {
return request ? FetchRequestDispatcher.from(scope, request) : undefined;
}
private constructor(scope: DispatcherScope, request: FetchRequest) {
super(scope, request, 'FetchRequest', {}, true);
}
async fetch(params: channels.FetchRequestFetchParams, metadata?: channels.Metadata): Promise<channels.FetchRequestFetchResult> {
const { fetchResponse, error } = await this._object.fetch({
url: params.url,
params: arrayToObject(params.params),
method: params.method,
headers: params.headers ? headersArrayToObject(params.headers, false) : undefined,
postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined,
timeout: params.timeout,
failOnStatusCode: params.failOnStatusCode,
});
let response;
if (fetchResponse) {
response = {
url: fetchResponse.url,
status: fetchResponse.status,
statusText: fetchResponse.statusText,
headers: fetchResponse.headers,
fetchUid: fetchResponse.fetchUid
};
}
return { response, error };
}
async fetchResponseBody(params: channels.FetchRequestFetchResponseBodyParams, metadata?: channels.Metadata): Promise<channels.FetchRequestFetchResponseBodyResult> {
const buffer = this._object.fetchResponses.get(params.fetchUid);
return { binary: buffer ? buffer.toString('base64') : undefined };
}
async disposeFetchResponse(params: channels.FetchRequestDisposeFetchResponseParams, metadata?: channels.Metadata): Promise<void> {
this._object.fetchResponses.delete(params.fetchUid);
}
_disposeDispatcher() {
if (!this._disposed)
super._dispose();
}
}

7
src/dispatchers/playwrightDispatcher.ts
View File

@ -26,6 +26,8 @@ import * as types from '../server/types';
import { SocksConnection, SocksConnectionClient } from '../utils/socksProxy';
import { createGuid } from '../utils/utils';
import { debugLogger } from '../utils/debugLogger';
import { GlobalFetchRequest } from '../server/fetch';
import { FetchRequestDispatcher } from './networkDispatchers';
export class PlaywrightDispatcher extends Dispatcher<Playwright, channels.PlaywrightInitializer, channels.PlaywrightEvents> implements channels.PlaywrightChannel {
private _socksProxy: SocksProxy | undefined;
@ -71,6 +73,11 @@ export class PlaywrightDispatcher extends Dispatcher<Playwright, channels.Playwr
async socksEnd(params: channels.PlaywrightSocksEndParams): Promise<void> {
this._socksProxy?.sendSocketEnd(params);
}
async newRequest(params: channels.PlaywrightNewRequestParams, metadata?: channels.Metadata): Promise<channels.PlaywrightNewRequestResult> {
const request = new GlobalFetchRequest(this._object);
return { request: FetchRequestDispatcher.from(this._scope, request) };
}
}
class SocksProxy implements SocksConnectionClient {

99
src/protocol/channels.ts
View File

@ -150,6 +150,54 @@ export type InterceptedResponse = {
headers: NameValue[],
};
// ----------- FetchRequest -----------
export type FetchRequestInitializer = {};
export interface FetchRequestChannel extends Channel {
fetch(params: FetchRequestFetchParams, metadata?: Metadata): Promise<FetchRequestFetchResult>;
fetchResponseBody(params: FetchRequestFetchResponseBodyParams, metadata?: Metadata): Promise<FetchRequestFetchResponseBodyResult>;
disposeFetchResponse(params: FetchRequestDisposeFetchResponseParams, metadata?: Metadata): Promise<FetchRequestDisposeFetchResponseResult>;
}
export type FetchRequestFetchParams = {
url: string,
params?: NameValue[],
method?: string,
headers?: NameValue[],
postData?: Binary,
timeout?: number,
failOnStatusCode?: boolean,
};
export type FetchRequestFetchOptions = {
params?: NameValue[],
method?: string,
headers?: NameValue[],
postData?: Binary,
timeout?: number,
failOnStatusCode?: boolean,
};
export type FetchRequestFetchResult = {
response?: FetchResponse,
error?: string,
};
export type FetchRequestFetchResponseBodyParams = {
fetchUid: string,
};
export type FetchRequestFetchResponseBodyOptions = {
};
export type FetchRequestFetchResponseBodyResult = {
binary?: Binary,
};
export type FetchRequestDisposeFetchResponseParams = {
fetchUid: string,
};
export type FetchRequestDisposeFetchResponseOptions = {
};
export type FetchRequestDisposeFetchResponseResult = void;
export interface FetchRequestEvents {
}
export type FetchResponse = {
fetchUid: string,
url: string,
@ -213,6 +261,7 @@ export interface PlaywrightChannel extends Channel {
socksData(params: PlaywrightSocksDataParams, metadata?: Metadata): Promise<PlaywrightSocksDataResult>;
socksError(params: PlaywrightSocksErrorParams, metadata?: Metadata): Promise<PlaywrightSocksErrorResult>;
socksEnd(params: PlaywrightSocksEndParams, metadata?: Metadata): Promise<PlaywrightSocksEndResult>;
newRequest(params: PlaywrightNewRequestParams, metadata?: Metadata): Promise<PlaywrightNewRequestResult>;
}
export type PlaywrightSocksRequestedEvent = {
uid: string,
@ -266,6 +315,15 @@ export type PlaywrightSocksEndOptions = {
};
export type PlaywrightSocksEndResult = void;
export type PlaywrightNewRequestParams = {
ignoreHTTPSErrors?: boolean,
};
export type PlaywrightNewRequestOptions = {
ignoreHTTPSErrors?: boolean,
};
export type PlaywrightNewRequestResult = {
request: FetchRequestChannel,
};
export interface PlaywrightEvents {
'socksRequested': PlaywrightSocksRequestedEvent;
@ -733,6 +791,7 @@ export interface EventTargetEvents {
// ----------- BrowserContext -----------
export type BrowserContextInitializer = {
isChromium: boolean,
fetchRequest: FetchRequestChannel,
};
export interface BrowserContextChannel extends EventTargetChannel {
on(event: 'bindingCall', callback: (params: BrowserContextBindingCallEvent) => void): this;
@ -753,9 +812,6 @@ export interface BrowserContextChannel extends EventTargetChannel {
close(params?: BrowserContextCloseParams, metadata?: Metadata): Promise<BrowserContextCloseResult>;
cookies(params: BrowserContextCookiesParams, metadata?: Metadata): Promise<BrowserContextCookiesResult>;
exposeBinding(params: BrowserContextExposeBindingParams, metadata?: Metadata): Promise<BrowserContextExposeBindingResult>;
fetch(params: BrowserContextFetchParams, metadata?: Metadata): Promise<BrowserContextFetchResult>;
fetchResponseBody(params: BrowserContextFetchResponseBodyParams, metadata?: Metadata): Promise<BrowserContextFetchResponseBodyResult>;
disposeFetchResponse(params: BrowserContextDisposeFetchResponseParams, metadata?: Metadata): Promise<BrowserContextDisposeFetchResponseResult>;
grantPermissions(params: BrowserContextGrantPermissionsParams, metadata?: Metadata): Promise<BrowserContextGrantPermissionsResult>;
newPage(params?: BrowserContextNewPageParams, metadata?: Metadata): Promise<BrowserContextNewPageResult>;
setDefaultNavigationTimeoutNoReply(params: BrowserContextSetDefaultNavigationTimeoutNoReplyParams, metadata?: Metadata): Promise<BrowserContextSetDefaultNavigationTimeoutNoReplyResult>;
@ -855,43 +911,6 @@ export type BrowserContextExposeBindingOptions = {
needsHandle?: boolean,
};
export type BrowserContextExposeBindingResult = void;
export type BrowserContextFetchParams = {
url: string,
params?: NameValue[],
method?: string,
headers?: NameValue[],
postData?: Binary,
timeout?: number,
failOnStatusCode?: boolean,
};
export type BrowserContextFetchOptions = {
params?: NameValue[],
method?: string,
headers?: NameValue[],
postData?: Binary,
timeout?: number,
failOnStatusCode?: boolean,
};
export type BrowserContextFetchResult = {
response?: FetchResponse,
error?: string,
};
export type BrowserContextFetchResponseBodyParams = {
fetchUid: string,
};
export type BrowserContextFetchResponseBodyOptions = {
};
export type BrowserContextFetchResponseBodyResult = {
binary?: Binary,
};
export type BrowserContextDisposeFetchResponseParams = {
fetchUid: string,
};
export type BrowserContextDisposeFetchResponseOptions = {
};
export type BrowserContextDisposeFetchResponseResult = void;
export type BrowserContextGrantPermissionsParams = {
permissions: string[],
origin?: string,

67
src/protocol/protocol.yml
View File

@ -217,6 +217,39 @@ InterceptedResponse:
items: NameValue
FetchRequest:
type: interface
commands:
fetch:
parameters:
url: string
params:
type: array?
items: NameValue
method: string?
headers:
type: array?
items: NameValue
postData: binary?
timeout: number?
failOnStatusCode: boolean?
returns:
response: FetchResponse?
error: string?
fetchResponseBody:
parameters:
fetchUid: string
returns:
binary?: binary
disposeFetchResponse:
parameters:
fetchUid: string
FetchResponse:
type: object
properties:
@ -417,6 +450,12 @@ Playwright:
parameters:
uid: string
newRequest:
parameters:
ignoreHTTPSErrors: boolean?
returns:
request: FetchRequest
events:
socksRequested:
parameters:
@ -579,6 +618,7 @@ BrowserContext:
initializer:
isChromium: boolean
fetchRequest: FetchRequest
commands:
@ -613,33 +653,6 @@ BrowserContext:
name: string
needsHandle: boolean?
fetch:
parameters:
url: string
params:
type: array?
items: NameValue
method: string?
headers:
type: array?
items: NameValue
postData: binary?
timeout: number?
failOnStatusCode: boolean?
returns:
response: FetchResponse?
error: string?
fetchResponseBody:
parameters:
fetchUid: string
returns:
binary?: binary
disposeFetchResponse:
parameters:
fetchUid: string
grantPermissions:
parameters:
permissions:

33
src/protocol/validator.ts
View File

@ -147,6 +147,21 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
statusText: tString,
headers: tArray(tType('NameValue')),
});
scheme.FetchRequestFetchParams = tObject({
url: tString,
params: tOptional(tArray(tType('NameValue'))),
method: tOptional(tString),
headers: tOptional(tArray(tType('NameValue'))),
postData: tOptional(tBinary),
timeout: tOptional(tNumber),
failOnStatusCode: tOptional(tBoolean),
});
scheme.FetchRequestFetchResponseBodyParams = tObject({
fetchUid: tString,
});
scheme.FetchRequestDisposeFetchResponseParams = tObject({
fetchUid: tString,
});
scheme.FetchResponse = tObject({
fetchUid: tString,
url: tString,
@ -177,6 +192,9 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
scheme.PlaywrightSocksEndParams = tObject({
uid: tString,
});
scheme.PlaywrightNewRequestParams = tObject({
ignoreHTTPSErrors: tOptional(tBoolean),
});
scheme.SelectorsRegisterParams = tObject({
name: tString,
source: tString,
@ -392,21 +410,6 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
name: tString,
needsHandle: tOptional(tBoolean),
});
scheme.BrowserContextFetchParams = tObject({
url: tString,
params: tOptional(tArray(tType('NameValue'))),
method: tOptional(tString),
headers: tOptional(tArray(tType('NameValue'))),
postData: tOptional(tBinary),
timeout: tOptional(tNumber),
failOnStatusCode: tOptional(tBoolean),
});
scheme.BrowserContextFetchResponseBodyParams = tObject({
fetchUid: tString,
});
scheme.BrowserContextDisposeFetchResponseParams = tObject({
fetchUid: tString,
});
scheme.BrowserContextGrantPermissionsParams = tObject({
permissions: tArray(tString),
origin: tOptional(tString),

11
src/server/browserContext.ts
View File

@ -34,6 +34,7 @@ import { Tracing } from './trace/recorder/tracing';
import { HarRecorder } from './supplements/har/harRecorder';
import { RecorderSupplement } from './supplements/recorderSupplement';
import * as consoleApiSource from '../generated/consoleApiSource';
import { BrowserContextFetchRequest } from './fetch';
export abstract class BrowserContext extends SdkObject {
static Events = {
@ -63,7 +64,7 @@ export abstract class BrowserContext extends SdkObject {
private _origins = new Set<string>();
readonly _harRecorder: HarRecorder | undefined;
readonly tracing: Tracing;
readonly fetchResponses: Map<string, Buffer> = new Map();
readonly fetchRequest = new BrowserContextFetchRequest(this);
constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) {
super(browser, 'browser-context');
@ -133,7 +134,7 @@ export abstract class BrowserContext extends SdkObject {
this._closedStatus = 'closed';
this._deleteAllDownloads();
this._downloads.clear();
this.fetchResponses.clear();
this.fetchRequest.dispose();
if (this._isPersistentContext)
this._onClosePersistent();
this._closePromiseFulfill!(new Error('Context closed'));
@ -382,12 +383,6 @@ export abstract class BrowserContext extends SdkObject {
this.on(BrowserContext.Events.Page, installInPage);
return Promise.all(this.pages().map(installInPage));
}
storeFetchResponseBody(body: Buffer): string {
const uid = createGuid();
this.fetchResponses.set(uid, body);
return uid;
}
}
export function assertBrowserContextIsNotOwned(context: BrowserContext) {

470
src/server/fetch.ts
View File

@ -22,209 +22,303 @@ import * as https from 'https';
import { BrowserContext } from './browserContext';
import * as types from './types';
import { pipeline, Readable, Transform } from 'stream';
import { monotonicTime } from '../utils/utils';
import { createGuid, monotonicTime } from '../utils/utils';
import { SdkObject } from './instrumentation';
import { Playwright } from './playwright';
import { HeadersArray, ProxySettings } from './types';
import { HTTPCredentials } from '../../types/types';
import { TimeoutSettings } from '../utils/timeoutSettings';
export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: Omit<types.FetchResponse, 'body'> & { fetchUid: string }, error?: string}> {
try {
const headers: { [name: string]: string } = {};
if (params.headers) {
for (const [name, value] of Object.entries(params.headers))
headers[name.toLowerCase()] = value;
}
headers['user-agent'] ??= context._options.userAgent || context._browser.userAgent();
headers['accept'] ??= '*/*';
headers['accept-encoding'] ??= 'gzip,deflate,br';
if (context._options.extraHTTPHeaders) {
for (const {name, value} of context._options.extraHTTPHeaders)
headers[name.toLowerCase()] = value;
}
type FetchRequestOptions = {
userAgent: string;
extraHTTPHeaders?: HeadersArray;
httpCredentials?: HTTPCredentials;
proxy?: ProxySettings;
timeoutSettings: TimeoutSettings;
ignoreHTTPSErrors?: boolean;
baseURL?: string;
};
const method = params.method?.toUpperCase() || 'GET';
const proxy = context._options.proxy || context._browser.options.proxy;
let agent;
if (proxy) {
// TODO: support bypass proxy
const proxyOpts = url.parse(proxy.server);
if (proxy.username)
proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`;
agent = new HttpsProxyAgent(proxyOpts);
}
export abstract class FetchRequest extends SdkObject {
readonly fetchResponses: Map<string, Buffer> = new Map();
const timeout = context._timeoutSettings.timeout(params);
const deadline = monotonicTime() + timeout;
const options: https.RequestOptions & { maxRedirects: number, deadline: number } = {
method,
headers,
agent,
maxRedirects: 20,
timeout,
deadline
};
// rejectUnauthorized = undefined is treated as true in node 12.
if (context._options.ignoreHTTPSErrors)
options.rejectUnauthorized = false;
const requestUrl = new URL(params.url, context._options.baseURL);
if (params.params) {
for (const [name, value] of Object.entries(params.params))
requestUrl.searchParams.set(name, value);
}
const fetchResponse = await sendRequest(context, requestUrl, options, params.postData);
const fetchUid = context.storeFetchResponseBody(fetchResponse.body);
if (params.failOnStatusCode && (fetchResponse.status < 200 || fetchResponse.status >= 400))
return { error: `${fetchResponse.status} ${fetchResponse.statusText}` };
return { fetchResponse: { ...fetchResponse, fetchUid } };
} catch (e) {
return { error: String(e) };
constructor(parent: SdkObject) {
super(parent, 'fetchRequest');
}
}
async function updateCookiesFromHeader(context: BrowserContext, responseUrl: string, setCookie: string[]) {
const url = new URL(responseUrl);
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
const cookies: types.SetNetworkCookieParam[] = [];
for (const header of setCookie) {
// Decode cookie value?
const cookie: types.SetNetworkCookieParam | null = parseCookie(header);
if (!cookie)
continue;
if (!cookie.domain)
cookie.domain = url.hostname;
if (!canSetCookie(cookie.domain!, url.hostname))
continue;
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4
if (!cookie.path || !cookie.path.startsWith('/'))
cookie.path = defaultPath;
cookies.push(cookie);
dispose() {
this.fetchResponses.clear();
}
if (cookies.length)
await context.addCookies(cookies);
}
async function updateRequestCookieHeader(context: BrowserContext, url: URL, options: http.RequestOptions) {
if (options.headers!['cookie'] !== undefined)
return;
const cookies = await context.cookies(url.toString());
if (cookies.length) {
const valueArray = cookies.map(c => `${c.name}=${c.value}`);
options.headers!['cookie'] = valueArray.join('; ');
abstract _defaultOptions(): FetchRequestOptions;
abstract _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void>;
abstract _cookies(url: string): Promise<types.NetworkCookie[]>;
private _storeResponseBody(body: Buffer): string {
const uid = createGuid();
this.fetchResponses.set(uid, body);
return uid;
}
}
async function sendRequest(context: BrowserContext, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.FetchResponse>{
await updateRequestCookieHeader(context, url, options);
return new Promise<types.FetchResponse>((fulfill, reject) => {
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
= (url.protocol === 'https:' ? https : http).request;
const request = requestConstructor(url, options, async response => {
if (response.headers['set-cookie'])
await updateCookiesFromHeader(context, response.url || url.toString(), response.headers['set-cookie']);
if (redirectStatus.includes(response.statusCode!)) {
if (!options.maxRedirects) {
reject(new Error('Max redirect count exceeded'));
request.abort();
return;
}
const headers = { ...options.headers };
delete headers[`cookie`];
async fetch(params: types.FetchOptions): Promise<{fetchResponse?: Omit<types.FetchResponse, 'body'> & { fetchUid: string }, error?: string}> {
try {
const headers: { [name: string]: string } = {};
const defaults = this._defaultOptions();
headers['user-agent'] = defaults.userAgent;
headers['accept'] = '*/*';
headers['accept-encoding'] = 'gzip,deflate,br';
// HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch)
const status = response.statusCode!;
let method = options.method!;
if ((status === 301 || status === 302) && method === 'POST' ||
status === 303 && !['GET', 'HEAD'].includes(method)) {
method = 'GET';
postData = undefined;
delete headers[`content-encoding`];
delete headers[`content-language`];
delete headers[`content-location`];
delete headers[`content-type`];
}
const redirectOptions: http.RequestOptions & { maxRedirects: number, deadline: number } = {
method,
headers,
agent: options.agent,
maxRedirects: options.maxRedirects - 1,
timeout: options.timeout,
deadline: options.deadline
};
// HTTP-redirect fetch step 4: If locationURL is null, then return response.
if (response.headers.location) {
const locationURL = new URL(response.headers.location, url);
fulfill(sendRequest(context, locationURL, redirectOptions, postData));
request.abort();
return;
}
}
if (response.statusCode === 401 && !options.headers!['authorization']) {
const auth = response.headers['www-authenticate'];
const credentials = context._options.httpCredentials;
if (auth?.trim().startsWith('Basic ') && credentials) {
const {username, password} = credentials;
const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
options.headers!['authorization'] = `Basic ${encoded}`;
fulfill(sendRequest(context, url, options, postData));
request.abort();
return;
}
}
response.on('aborted', () => reject(new Error('aborted')));
let body: Readable = response;
let transform: Transform | undefined;
const encoding = response.headers['content-encoding'];
if (encoding === 'gzip' || encoding === 'x-gzip') {
transform = zlib.createGunzip({
flush: zlib.constants.Z_SYNC_FLUSH,
finishFlush: zlib.constants.Z_SYNC_FLUSH
});
} else if (encoding === 'br') {
transform = zlib.createBrotliDecompress();
} else if (encoding === 'deflate') {
transform = zlib.createInflate();
}
if (transform) {
body = pipeline(response, transform, e => {
if (e)
reject(new Error(`failed to decompress '${encoding}' encoding: ${e}`));
});
if (defaults.extraHTTPHeaders) {
for (const {name, value} of defaults.extraHTTPHeaders)
headers[name.toLowerCase()] = value;
}
const chunks: Buffer[] = [];
body.on('data', chunk => chunks.push(chunk));
body.on('end', () => {
const body = Buffer.concat(chunks);
fulfill({
url: response.url || url.toString(),
status: response.statusCode || 0,
statusText: response.statusMessage || '',
headers: toHeadersArray(response.rawHeaders),
body
});
});
body.on('error',reject);
});
request.on('error', reject);
const rejectOnTimeout = () => {
reject(new Error(`Request timed out after ${options.timeout}ms`));
request.abort();
};
const remaining = options.deadline - monotonicTime();
if (remaining <= 0) {
rejectOnTimeout();
if (params.headers) {
for (const [name, value] of Object.entries(params.headers))
headers[name.toLowerCase()] = value;
}
const method = params.method?.toUpperCase() || 'GET';
const proxy = defaults.proxy;
let agent;
if (proxy) {
// TODO: support bypass proxy
const proxyOpts = url.parse(proxy.server);
if (proxy.username)
proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`;
agent = new HttpsProxyAgent(proxyOpts);
}
const timeout = defaults.timeoutSettings.timeout(params);
const deadline = monotonicTime() + timeout;
const options: https.RequestOptions & { maxRedirects: number, deadline: number } = {
method,
headers,
agent,
maxRedirects: 20,
timeout,
deadline
};
// rejectUnauthorized = undefined is treated as true in node 12.
if (defaults.ignoreHTTPSErrors)
options.rejectUnauthorized = false;
const requestUrl = new URL(params.url, defaults.baseURL);
if (params.params) {
for (const [name, value] of Object.entries(params.params))
requestUrl.searchParams.set(name, value);
}
const fetchResponse = await this._sendRequest(requestUrl, options, params.postData);
const fetchUid = this._storeResponseBody(fetchResponse.body);
if (params.failOnStatusCode && (fetchResponse.status < 200 || fetchResponse.status >= 400))
return { error: `${fetchResponse.status} ${fetchResponse.statusText}` };
return { fetchResponse: { ...fetchResponse, fetchUid } };
} catch (e) {
return { error: String(e) };
}
}
private async _updateCookiesFromHeader(responseUrl: string, setCookie: string[]) {
const url = new URL(responseUrl);
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
const cookies: types.SetNetworkCookieParam[] = [];
for (const header of setCookie) {
// Decode cookie value?
const cookie: types.SetNetworkCookieParam | null = parseCookie(header);
if (!cookie)
continue;
if (!cookie.domain)
cookie.domain = url.hostname;
if (!canSetCookie(cookie.domain!, url.hostname))
continue;
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4
if (!cookie.path || !cookie.path.startsWith('/'))
cookie.path = defaultPath;
cookies.push(cookie);
}
if (cookies.length)
await this._addCookies(cookies);
}
private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) {
if (options.headers!['cookie'] !== undefined)
return;
const cookies = await this._cookies(url.toString());
if (cookies.length) {
const valueArray = cookies.map(c => `${c.name}=${c.value}`);
options.headers!['cookie'] = valueArray.join('; ');
}
request.setTimeout(remaining, rejectOnTimeout);
if (postData)
request.write(postData);
request.end();
});
}
private async _sendRequest(url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.FetchResponse>{
await this._updateRequestCookieHeader(url, options);
return new Promise<types.FetchResponse>((fulfill, reject) => {
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
= (url.protocol === 'https:' ? https : http).request;
const request = requestConstructor(url, options, async response => {
if (response.headers['set-cookie'])
await this._updateCookiesFromHeader(response.url || url.toString(), response.headers['set-cookie']);
if (redirectStatus.includes(response.statusCode!)) {
if (!options.maxRedirects) {
reject(new Error('Max redirect count exceeded'));
request.abort();
return;
}
const headers = { ...options.headers };
delete headers[`cookie`];
// HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch)
const status = response.statusCode!;
let method = options.method!;
if ((status === 301 || status === 302) && method === 'POST' ||
status === 303 && !['GET', 'HEAD'].includes(method)) {
method = 'GET';
postData = undefined;
delete headers[`content-encoding`];
delete headers[`content-language`];
delete headers[`content-location`];
delete headers[`content-type`];
}
const redirectOptions: http.RequestOptions & { maxRedirects: number, deadline: number } = {
method,
headers,
agent: options.agent,
maxRedirects: options.maxRedirects - 1,
timeout: options.timeout,
deadline: options.deadline
};
// HTTP-redirect fetch step 4: If locationURL is null, then return response.
if (response.headers.location) {
const locationURL = new URL(response.headers.location, url);
fulfill(this._sendRequest(locationURL, redirectOptions, postData));
request.abort();
return;
}
}
if (response.statusCode === 401 && !options.headers!['authorization']) {
const auth = response.headers['www-authenticate'];
const credentials = this._defaultOptions().httpCredentials;
if (auth?.trim().startsWith('Basic ') && credentials) {
const {username, password} = credentials;
const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
options.headers!['authorization'] = `Basic ${encoded}`;
fulfill(this._sendRequest(url, options, postData));
request.abort();
return;
}
}
response.on('aborted', () => reject(new Error('aborted')));
let body: Readable = response;
let transform: Transform | undefined;
const encoding = response.headers['content-encoding'];
if (encoding === 'gzip' || encoding === 'x-gzip') {
transform = zlib.createGunzip({
flush: zlib.constants.Z_SYNC_FLUSH,
finishFlush: zlib.constants.Z_SYNC_FLUSH
});
} else if (encoding === 'br') {
transform = zlib.createBrotliDecompress();
} else if (encoding === 'deflate') {
transform = zlib.createInflate();
}
if (transform) {
body = pipeline(response, transform, e => {
if (e)
reject(new Error(`failed to decompress '${encoding}' encoding: ${e}`));
});
}
const chunks: Buffer[] = [];
body.on('data', chunk => chunks.push(chunk));
body.on('end', () => {
const body = Buffer.concat(chunks);
fulfill({
url: response.url || url.toString(),
status: response.statusCode || 0,
statusText: response.statusMessage || '',
headers: toHeadersArray(response.rawHeaders),
body
});
});
body.on('error',reject);
});
request.on('error', reject);
const rejectOnTimeout = () => {
reject(new Error(`Request timed out after ${options.timeout}ms`));
request.abort();
};
const remaining = options.deadline - monotonicTime();
if (remaining <= 0) {
rejectOnTimeout();
return;
}
request.setTimeout(remaining, rejectOnTimeout);
if (postData)
request.write(postData);
request.end();
});
}
}
export class BrowserContextFetchRequest extends FetchRequest {
private readonly _context: BrowserContext;
constructor(context: BrowserContext) {
super(context);
this._context = context;
}
_defaultOptions(): FetchRequestOptions {
return {
userAgent: this._context._options.userAgent || this._context._browser.userAgent(),
extraHTTPHeaders: this._context._options.extraHTTPHeaders,
httpCredentials: this._context._options.httpCredentials,
proxy: this._context._options.proxy || this._context._browser.options.proxy,
timeoutSettings: this._context._timeoutSettings,
ignoreHTTPSErrors: this._context._options.ignoreHTTPSErrors,
baseURL: this._context._options.baseURL,
};
}
async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
await this._context.addCookies(cookies);
}
async _cookies(url: string): Promise<types.NetworkCookie[]> {
return await this._context.cookies(url);
}
}
export class GlobalFetchRequest extends FetchRequest {
constructor(playwright: Playwright) {
super(playwright);
}
_defaultOptions(): FetchRequestOptions {
return {
userAgent: '',
extraHTTPHeaders: undefined,
proxy: undefined,
timeoutSettings: new TimeoutSettings(),
ignoreHTTPSErrors: false,
baseURL: undefined,
};
}
async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
}
async _cookies(url: string): Promise<types.NetworkCookie[]> {
return [];
}
}
function toHeadersArray(rawHeaders: string[]): types.HeadersArray {

2
src/server/network.ts
View File

@ -228,7 +228,7 @@ export class Route extends SdkObject {
if (body === undefined) {
if (overrides.fetchResponseUid) {
const context = this._request.frame()._page._browserContext;
const buffer = context.fetchResponses.get(overrides.fetchResponseUid);
const buffer = context.fetchRequest.fetchResponses.get(overrides.fetchResponseUid);
assert(buffer, 'Fetch response has been disposed');
body = buffer.toString('utf8');
isBase64 = false;

13
tests/browsercontext-fetch.spec.ts
View File

@ -40,6 +40,19 @@ it.afterAll(() => {
http.globalAgent = prevAgent;
});
it('global get should work', async ({playwright, context, server}) => {
const request = await playwright._newRequest();
const response = await request.get(server.PREFIX + '/simple.json');
expect(response.url()).toBe(server.PREFIX + '/simple.json');
expect(response.status()).toBe(200);
expect(response.statusText()).toBe('OK');
expect(response.ok()).toBeTruthy();
expect(response.url()).toBe(server.PREFIX + '/simple.json');
expect(response.headers()['content-type']).toBe('application/json; charset=utf-8');
expect(response.headersArray()).toContainEqual({ name: 'Content-Type', value: 'application/json; charset=utf-8' });
expect(await response.text()).toBe('{"foo": "bar"}\n');
});
it('get should work', async ({context, server}) => {
const response = await context._request.get(server.PREFIX + '/simple.json');
expect(response.url()).toBe(server.PREFIX + '/simple.json');

4
tests/channels.spec.ts
View File

@ -57,6 +57,7 @@ it('should scope context handles', async ({browserType, browserOptions, server})
{ _guid: 'request', objects: [] },
{ _guid: 'response', objects: [] },
]},
{ _guid: 'fetchRequest', objects: [] }
] },
] },
{ _guid: 'electron', objects: [] },
@ -140,7 +141,8 @@ it('should scope browser handles', async ({browserType, browserOptions}) => {
{ _guid: 'browser-type', objects: [
{
_guid: 'browser', objects: [
{ _guid: 'browser-context', objects: [] }
{ _guid: 'browser-context', objects: [] },
{ _guid: 'fetchRequest', objects: [] }
]
},
]

1
types/types.d.ts vendored
View File

@ -10668,6 +10668,7 @@ export const firefox: BrowserType;
export const webkit: BrowserType;
export const _electron: Electron;
export const _android: Android;
export const _newRequest: () => Promise<FetchRequest>;
// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459
export {};

1
utils/generate_types/overrides.d.ts vendored
View File

@ -348,6 +348,7 @@ export const firefox: BrowserType;
export const webkit: BrowserType;
export const _electron: Electron;
export const _android: Android;
export const _newRequest: () => Promise<FetchRequest>;
// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459
export {};