mirror of
https://github.com/microsoft/playwright.git
synced 2025-06-26 21:40:17 +00:00
feat(fetch): introduce global fetch request (#8927)
This commit is contained in:
Yury Semikhatsky
GitHub
parent
5253a7eb54
commit
c58f34fb2e
@ -69,7 +69,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
|
||||
this._browser = parent;
|
||||
this._isChromium = this._browser?._name === 'chromium';
|
||||
this.tracing = new Tracing(this);
|
||||
this._request = new FetchRequest(this);
|
||||
this._request = FetchRequest.from(initializer.fetchRequest);
|
||||
|
||||
this._channel.on('bindingCall', ({binding}) => this._onBinding(BindingCall.from(binding)));
|
||||
this._channel.on('close', () => this._onClose());
|
||||
|
||||
@ -39,6 +39,7 @@ import { ParsedStackTrace } from '../utils/stackTrace';
|
||||
import { Artifact } from './artifact';
|
||||
import { EventEmitter } from 'events';
|
||||
import { JsonPipe } from './jsonPipe';
|
||||
import { FetchRequest } from './fetch';
|
||||
|
||||
class Root extends ChannelOwner<channels.RootChannel, {}> {
|
||||
constructor(connection: Connection) {
|
||||
@ -216,6 +217,9 @@ export class Connection extends EventEmitter {
|
||||
case 'ElementHandle':
|
||||
result = new ElementHandle(parent, type, guid, initializer);
|
||||
break;
|
||||
case 'FetchRequest':
|
||||
result = new FetchRequest(parent, type, guid, initializer);
|
||||
break;
|
||||
case 'Frame':
|
||||
result = new Frame(parent, type, guid, initializer);
|
||||
break;
|
||||
|
||||
@ -18,7 +18,7 @@ import * as api from '../../types/types';
|
||||
import { HeadersArray } from '../common/types';
|
||||
import * as channels from '../protocol/channels';
|
||||
import { assert, headersObjectToArray, isString, objectToArray } from '../utils/utils';
|
||||
import { BrowserContext } from './browserContext';
|
||||
import { ChannelOwner } from './channelOwner';
|
||||
import * as network from './network';
|
||||
import { RawHeaders } from './network';
|
||||
import { Headers } from './types';
|
||||
@ -32,11 +32,13 @@ export type FetchOptions = {
|
||||
failOnStatusCode?: boolean,
|
||||
};
|
||||
|
||||
export class FetchRequest implements api.FetchRequest {
|
||||
private _context: BrowserContext;
|
||||
export class FetchRequest extends ChannelOwner<channels.FetchRequestChannel, channels.FetchRequestInitializer> implements api.FetchRequest {
|
||||
static from(channel: channels.FetchRequestChannel): FetchRequest {
|
||||
return (channel as any)._object;
|
||||
}
|
||||
|
||||
constructor(context: BrowserContext) {
|
||||
this._context = context;
|
||||
constructor(parent: ChannelOwner, type: string, guid: string, initializer: channels.FetchRequestInitializer) {
|
||||
super(parent, type, guid, initializer);
|
||||
}
|
||||
|
||||
async get(
|
||||
@ -69,7 +71,7 @@ export class FetchRequest implements api.FetchRequest {
|
||||
}
|
||||
|
||||
async fetch(urlOrRequest: string | api.Request, options: FetchOptions = {}): Promise<FetchResponse> {
|
||||
return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
|
||||
return this._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
|
||||
const request: network.Request | undefined = (urlOrRequest instanceof network.Request) ? urlOrRequest as network.Request : undefined;
|
||||
assert(request || typeof urlOrRequest === 'string', 'First argument must be either URL string or Request');
|
||||
const url = request ? request.url() : urlOrRequest as string;
|
||||
@ -93,7 +95,7 @@ export class FetchRequest implements api.FetchRequest {
|
||||
});
|
||||
if (result.error)
|
||||
throw new Error(`Request failed: ${result.error}`);
|
||||
return new FetchResponse(this._context, result.response!);
|
||||
return new FetchResponse(this, result.response!);
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -101,10 +103,10 @@ export class FetchRequest implements api.FetchRequest {
|
||||
export class FetchResponse implements api.FetchResponse {
|
||||
private readonly _initializer: channels.FetchResponse;
|
||||
private readonly _headers: RawHeaders;
|
||||
private readonly _context: BrowserContext;
|
||||
private readonly _request: FetchRequest;
|
||||
|
||||
constructor(context: BrowserContext, initializer: channels.FetchResponse) {
|
||||
this._context = context;
|
||||
constructor(context: FetchRequest, initializer: channels.FetchResponse) {
|
||||
this._request = context;
|
||||
this._initializer = initializer;
|
||||
this._headers = new RawHeaders(this._initializer.headers);
|
||||
}
|
||||
@ -134,7 +136,7 @@ export class FetchResponse implements api.FetchResponse {
|
||||
}
|
||||
|
||||
async body(): Promise<Buffer> {
|
||||
return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
|
||||
return this._request._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
|
||||
const result = await channel.fetchResponseBody({ fetchUid: this._fetchUid() });
|
||||
if (!result.binary)
|
||||
throw new Error('Response has been disposed');
|
||||
@ -153,7 +155,7 @@ export class FetchResponse implements api.FetchResponse {
|
||||
}
|
||||
|
||||
async dispose(): Promise<void> {
|
||||
return this._context._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
|
||||
return this._request._wrapApiCall(async (channel: channels.FetchRequestChannel) => {
|
||||
await channel.disposeFetchResponse({ fetchUid: this._fetchUid() });
|
||||
});
|
||||
}
|
||||
|
||||
@ -102,7 +102,7 @@ export class Page extends ChannelOwner<channels.PageChannel, channels.PageInitia
|
||||
this.accessibility = new Accessibility(this._channel);
|
||||
this.keyboard = new Keyboard(this);
|
||||
this.mouse = new Mouse(this);
|
||||
this._request = new FetchRequest(this._browserContext);
|
||||
this._request = this._browserContext._request;
|
||||
this.touchscreen = new Touchscreen(this);
|
||||
|
||||
this._mainFrame = Frame.from(initializer.mainFrame);
|
||||
|
||||
@ -24,6 +24,7 @@ import { Android } from './android';
|
||||
import { BrowserType } from './browserType';
|
||||
import { ChannelOwner } from './channelOwner';
|
||||
import { Electron } from './electron';
|
||||
import { FetchRequest } from './fetch';
|
||||
import { Selectors, SelectorsOwner, sharedSelectors } from './selectors';
|
||||
import { Size } from './types';
|
||||
const dnsLookupAsync = util.promisify(dns.lookup);
|
||||
@ -68,6 +69,12 @@ export class Playwright extends ChannelOwner<channels.PlaywrightChannel, channel
|
||||
this.selectors._addChannel(this._selectorsOwner);
|
||||
}
|
||||
|
||||
async _newRequest(options?: {}): Promise<FetchRequest> {
|
||||
return await this._wrapApiCall(async (channel: channels.PlaywrightChannel) => {
|
||||
return FetchRequest.from((await channel.newRequest({})).request);
|
||||
});
|
||||
}
|
||||
|
||||
_enablePortForwarding(redirectPortForTest?: number) {
|
||||
this._redirectPortForTest = redirectPortForTest;
|
||||
this._channel.on('socksRequested', ({ uid, host, port }) => this._onSocksRequested(uid, host, port));
|
||||
|
||||
@ -15,12 +15,11 @@
|
||||
*/
|
||||
|
||||
import { BrowserContext } from '../server/browserContext';
|
||||
import { Dispatcher, DispatcherScope, lookupDispatcher } from './dispatcher';
|
||||
import { Dispatcher, DispatcherScope, existingDispatcher, lookupDispatcher } from './dispatcher';
|
||||
import { PageDispatcher, BindingCallDispatcher, WorkerDispatcher } from './pageDispatcher';
|
||||
import { playwrightFetch } from '../server/fetch';
|
||||
import { FrameDispatcher } from './frameDispatcher';
|
||||
import * as channels from '../protocol/channels';
|
||||
import { RouteDispatcher, RequestDispatcher, ResponseDispatcher } from './networkDispatchers';
|
||||
import { RouteDispatcher, RequestDispatcher, ResponseDispatcher, FetchRequestDispatcher } from './networkDispatchers';
|
||||
import { CRBrowserContext } from '../server/chromium/crBrowser';
|
||||
import { CDPSessionDispatcher } from './cdpSessionDispatcher';
|
||||
import { RecorderSupplement } from '../server/supplements/recorderSupplement';
|
||||
@ -28,13 +27,15 @@ import { CallMetadata } from '../server/instrumentation';
|
||||
import { ArtifactDispatcher } from './artifactDispatcher';
|
||||
import { Artifact } from '../server/artifact';
|
||||
import { Request, Response } from '../server/network';
|
||||
import { arrayToObject, headersArrayToObject } from '../utils/utils';
|
||||
|
||||
export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channels.BrowserContextInitializer, channels.BrowserContextEvents> implements channels.BrowserContextChannel {
|
||||
private _context: BrowserContext;
|
||||
|
||||
constructor(scope: DispatcherScope, context: BrowserContext) {
|
||||
super(scope, context, 'BrowserContext', { isChromium: context._browser.options.isChromium }, true);
|
||||
super(scope, context, 'BrowserContext', {
|
||||
isChromium: context._browser.options.isChromium,
|
||||
fetchRequest: FetchRequestDispatcher.from(scope, context.fetchRequest),
|
||||
}, true);
|
||||
this._context = context;
|
||||
// Note: when launching persistent context, dispatcher is created very late,
|
||||
// so we can already have pages, videos and everything else.
|
||||
@ -57,6 +58,10 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
|
||||
context.on(BrowserContext.Events.Close, () => {
|
||||
this._dispatchEvent('close');
|
||||
this._dispose();
|
||||
const fetch = existingDispatcher<FetchRequestDispatcher>(this._context.fetchRequest);
|
||||
// FetchRequestDispatcher is created in the browser rather then context scope but its
|
||||
// lifetime is bound to the context dispatcher, so we manually dispose it here.
|
||||
fetch._disposeDispatcher();
|
||||
});
|
||||
|
||||
if (context._browser.options.name === 'chromium') {
|
||||
@ -107,38 +112,6 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
|
||||
});
|
||||
}
|
||||
|
||||
async fetch(params: channels.BrowserContextFetchParams): Promise<channels.BrowserContextFetchResult> {
|
||||
const { fetchResponse, error } = await playwrightFetch(this._context, {
|
||||
url: params.url,
|
||||
params: arrayToObject(params.params),
|
||||
method: params.method,
|
||||
headers: params.headers ? headersArrayToObject(params.headers, false) : undefined,
|
||||
postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined,
|
||||
timeout: params.timeout,
|
||||
failOnStatusCode: params.failOnStatusCode,
|
||||
});
|
||||
let response;
|
||||
if (fetchResponse) {
|
||||
response = {
|
||||
url: fetchResponse.url,
|
||||
status: fetchResponse.status,
|
||||
statusText: fetchResponse.statusText,
|
||||
headers: fetchResponse.headers,
|
||||
fetchUid: fetchResponse.fetchUid
|
||||
};
|
||||
}
|
||||
return { response, error };
|
||||
}
|
||||
|
||||
async fetchResponseBody(params: channels.BrowserContextFetchResponseBodyParams): Promise<channels.BrowserContextFetchResponseBodyResult> {
|
||||
const buffer = this._context.fetchResponses.get(params.fetchUid);
|
||||
return { binary: buffer ? buffer.toString('base64') : undefined };
|
||||
}
|
||||
|
||||
async disposeFetchResponse(params: channels.BrowserContextDisposeFetchResponseParams): Promise<channels.BrowserContextDisposeFetchResponseResult> {
|
||||
this._context.fetchResponses.delete(params.fetchUid);
|
||||
}
|
||||
|
||||
async newPage(params: channels.BrowserContextNewPageParams, metadata: CallMetadata): Promise<channels.BrowserContextNewPageResult> {
|
||||
return { page: lookupDispatcher<PageDispatcher>(await this._context.newPage(metadata)) };
|
||||
}
|
||||
|
||||
@ -19,6 +19,8 @@ import * as channels from '../protocol/channels';
|
||||
import { Dispatcher, DispatcherScope, lookupNullableDispatcher, existingDispatcher } from './dispatcher';
|
||||
import { FrameDispatcher } from './frameDispatcher';
|
||||
import { CallMetadata } from '../server/instrumentation';
|
||||
import { FetchRequest } from '../server/fetch';
|
||||
import { arrayToObject, headersArrayToObject } from '../utils/utils';
|
||||
|
||||
export class RequestDispatcher extends Dispatcher<Request, channels.RequestInitializer, channels.RequestEvents> implements channels.RequestChannel {
|
||||
|
||||
@ -156,3 +158,56 @@ export class WebSocketDispatcher extends Dispatcher<WebSocket, channels.WebSocke
|
||||
webSocket.on(WebSocket.Events.Close, () => this._dispatchEvent('close', {}));
|
||||
}
|
||||
}
|
||||
|
||||
export class FetchRequestDispatcher extends Dispatcher<FetchRequest, channels.FetchRequestInitializer, channels.FetchRequestEvents> implements channels.FetchRequestChannel {
|
||||
static from(scope: DispatcherScope, request: FetchRequest): FetchRequestDispatcher {
|
||||
const result = existingDispatcher<FetchRequestDispatcher>(request);
|
||||
return result || new FetchRequestDispatcher(scope, request);
|
||||
}
|
||||
|
||||
static fromNullable(scope: DispatcherScope, request: FetchRequest | null): FetchRequestDispatcher | undefined {
|
||||
return request ? FetchRequestDispatcher.from(scope, request) : undefined;
|
||||
}
|
||||
|
||||
private constructor(scope: DispatcherScope, request: FetchRequest) {
|
||||
super(scope, request, 'FetchRequest', {}, true);
|
||||
}
|
||||
|
||||
async fetch(params: channels.FetchRequestFetchParams, metadata?: channels.Metadata): Promise<channels.FetchRequestFetchResult> {
|
||||
const { fetchResponse, error } = await this._object.fetch({
|
||||
url: params.url,
|
||||
params: arrayToObject(params.params),
|
||||
method: params.method,
|
||||
headers: params.headers ? headersArrayToObject(params.headers, false) : undefined,
|
||||
postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined,
|
||||
timeout: params.timeout,
|
||||
failOnStatusCode: params.failOnStatusCode,
|
||||
});
|
||||
let response;
|
||||
if (fetchResponse) {
|
||||
response = {
|
||||
url: fetchResponse.url,
|
||||
status: fetchResponse.status,
|
||||
statusText: fetchResponse.statusText,
|
||||
headers: fetchResponse.headers,
|
||||
fetchUid: fetchResponse.fetchUid
|
||||
};
|
||||
}
|
||||
return { response, error };
|
||||
}
|
||||
|
||||
async fetchResponseBody(params: channels.FetchRequestFetchResponseBodyParams, metadata?: channels.Metadata): Promise<channels.FetchRequestFetchResponseBodyResult> {
|
||||
const buffer = this._object.fetchResponses.get(params.fetchUid);
|
||||
return { binary: buffer ? buffer.toString('base64') : undefined };
|
||||
}
|
||||
|
||||
async disposeFetchResponse(params: channels.FetchRequestDisposeFetchResponseParams, metadata?: channels.Metadata): Promise<void> {
|
||||
this._object.fetchResponses.delete(params.fetchUid);
|
||||
}
|
||||
|
||||
_disposeDispatcher() {
|
||||
if (!this._disposed)
|
||||
super._dispose();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -26,6 +26,8 @@ import * as types from '../server/types';
|
||||
import { SocksConnection, SocksConnectionClient } from '../utils/socksProxy';
|
||||
import { createGuid } from '../utils/utils';
|
||||
import { debugLogger } from '../utils/debugLogger';
|
||||
import { GlobalFetchRequest } from '../server/fetch';
|
||||
import { FetchRequestDispatcher } from './networkDispatchers';
|
||||
|
||||
export class PlaywrightDispatcher extends Dispatcher<Playwright, channels.PlaywrightInitializer, channels.PlaywrightEvents> implements channels.PlaywrightChannel {
|
||||
private _socksProxy: SocksProxy | undefined;
|
||||
@ -71,6 +73,11 @@ export class PlaywrightDispatcher extends Dispatcher<Playwright, channels.Playwr
|
||||
async socksEnd(params: channels.PlaywrightSocksEndParams): Promise<void> {
|
||||
this._socksProxy?.sendSocketEnd(params);
|
||||
}
|
||||
|
||||
async newRequest(params: channels.PlaywrightNewRequestParams, metadata?: channels.Metadata): Promise<channels.PlaywrightNewRequestResult> {
|
||||
const request = new GlobalFetchRequest(this._object);
|
||||
return { request: FetchRequestDispatcher.from(this._scope, request) };
|
||||
}
|
||||
}
|
||||
|
||||
class SocksProxy implements SocksConnectionClient {
|
||||
|
||||
@ -150,6 +150,54 @@ export type InterceptedResponse = {
|
||||
headers: NameValue[],
|
||||
};
|
||||
|
||||
// ----------- FetchRequest -----------
|
||||
export type FetchRequestInitializer = {};
|
||||
export interface FetchRequestChannel extends Channel {
|
||||
fetch(params: FetchRequestFetchParams, metadata?: Metadata): Promise<FetchRequestFetchResult>;
|
||||
fetchResponseBody(params: FetchRequestFetchResponseBodyParams, metadata?: Metadata): Promise<FetchRequestFetchResponseBodyResult>;
|
||||
disposeFetchResponse(params: FetchRequestDisposeFetchResponseParams, metadata?: Metadata): Promise<FetchRequestDisposeFetchResponseResult>;
|
||||
}
|
||||
export type FetchRequestFetchParams = {
|
||||
url: string,
|
||||
params?: NameValue[],
|
||||
method?: string,
|
||||
headers?: NameValue[],
|
||||
postData?: Binary,
|
||||
timeout?: number,
|
||||
failOnStatusCode?: boolean,
|
||||
};
|
||||
export type FetchRequestFetchOptions = {
|
||||
params?: NameValue[],
|
||||
method?: string,
|
||||
headers?: NameValue[],
|
||||
postData?: Binary,
|
||||
timeout?: number,
|
||||
failOnStatusCode?: boolean,
|
||||
};
|
||||
export type FetchRequestFetchResult = {
|
||||
response?: FetchResponse,
|
||||
error?: string,
|
||||
};
|
||||
export type FetchRequestFetchResponseBodyParams = {
|
||||
fetchUid: string,
|
||||
};
|
||||
export type FetchRequestFetchResponseBodyOptions = {
|
||||
|
||||
};
|
||||
export type FetchRequestFetchResponseBodyResult = {
|
||||
binary?: Binary,
|
||||
};
|
||||
export type FetchRequestDisposeFetchResponseParams = {
|
||||
fetchUid: string,
|
||||
};
|
||||
export type FetchRequestDisposeFetchResponseOptions = {
|
||||
|
||||
};
|
||||
export type FetchRequestDisposeFetchResponseResult = void;
|
||||
|
||||
export interface FetchRequestEvents {
|
||||
}
|
||||
|
||||
export type FetchResponse = {
|
||||
fetchUid: string,
|
||||
url: string,
|
||||
@ -213,6 +261,7 @@ export interface PlaywrightChannel extends Channel {
|
||||
socksData(params: PlaywrightSocksDataParams, metadata?: Metadata): Promise<PlaywrightSocksDataResult>;
|
||||
socksError(params: PlaywrightSocksErrorParams, metadata?: Metadata): Promise<PlaywrightSocksErrorResult>;
|
||||
socksEnd(params: PlaywrightSocksEndParams, metadata?: Metadata): Promise<PlaywrightSocksEndResult>;
|
||||
newRequest(params: PlaywrightNewRequestParams, metadata?: Metadata): Promise<PlaywrightNewRequestResult>;
|
||||
}
|
||||
export type PlaywrightSocksRequestedEvent = {
|
||||
uid: string,
|
||||
@ -266,6 +315,15 @@ export type PlaywrightSocksEndOptions = {
|
||||
|
||||
};
|
||||
export type PlaywrightSocksEndResult = void;
|
||||
export type PlaywrightNewRequestParams = {
|
||||
ignoreHTTPSErrors?: boolean,
|
||||
};
|
||||
export type PlaywrightNewRequestOptions = {
|
||||
ignoreHTTPSErrors?: boolean,
|
||||
};
|
||||
export type PlaywrightNewRequestResult = {
|
||||
request: FetchRequestChannel,
|
||||
};
|
||||
|
||||
export interface PlaywrightEvents {
|
||||
'socksRequested': PlaywrightSocksRequestedEvent;
|
||||
@ -733,6 +791,7 @@ export interface EventTargetEvents {
|
||||
// ----------- BrowserContext -----------
|
||||
export type BrowserContextInitializer = {
|
||||
isChromium: boolean,
|
||||
fetchRequest: FetchRequestChannel,
|
||||
};
|
||||
export interface BrowserContextChannel extends EventTargetChannel {
|
||||
on(event: 'bindingCall', callback: (params: BrowserContextBindingCallEvent) => void): this;
|
||||
@ -753,9 +812,6 @@ export interface BrowserContextChannel extends EventTargetChannel {
|
||||
close(params?: BrowserContextCloseParams, metadata?: Metadata): Promise<BrowserContextCloseResult>;
|
||||
cookies(params: BrowserContextCookiesParams, metadata?: Metadata): Promise<BrowserContextCookiesResult>;
|
||||
exposeBinding(params: BrowserContextExposeBindingParams, metadata?: Metadata): Promise<BrowserContextExposeBindingResult>;
|
||||
fetch(params: BrowserContextFetchParams, metadata?: Metadata): Promise<BrowserContextFetchResult>;
|
||||
fetchResponseBody(params: BrowserContextFetchResponseBodyParams, metadata?: Metadata): Promise<BrowserContextFetchResponseBodyResult>;
|
||||
disposeFetchResponse(params: BrowserContextDisposeFetchResponseParams, metadata?: Metadata): Promise<BrowserContextDisposeFetchResponseResult>;
|
||||
grantPermissions(params: BrowserContextGrantPermissionsParams, metadata?: Metadata): Promise<BrowserContextGrantPermissionsResult>;
|
||||
newPage(params?: BrowserContextNewPageParams, metadata?: Metadata): Promise<BrowserContextNewPageResult>;
|
||||
setDefaultNavigationTimeoutNoReply(params: BrowserContextSetDefaultNavigationTimeoutNoReplyParams, metadata?: Metadata): Promise<BrowserContextSetDefaultNavigationTimeoutNoReplyResult>;
|
||||
@ -855,43 +911,6 @@ export type BrowserContextExposeBindingOptions = {
|
||||
needsHandle?: boolean,
|
||||
};
|
||||
export type BrowserContextExposeBindingResult = void;
|
||||
export type BrowserContextFetchParams = {
|
||||
url: string,
|
||||
params?: NameValue[],
|
||||
method?: string,
|
||||
headers?: NameValue[],
|
||||
postData?: Binary,
|
||||
timeout?: number,
|
||||
failOnStatusCode?: boolean,
|
||||
};
|
||||
export type BrowserContextFetchOptions = {
|
||||
params?: NameValue[],
|
||||
method?: string,
|
||||
headers?: NameValue[],
|
||||
postData?: Binary,
|
||||
timeout?: number,
|
||||
failOnStatusCode?: boolean,
|
||||
};
|
||||
export type BrowserContextFetchResult = {
|
||||
response?: FetchResponse,
|
||||
error?: string,
|
||||
};
|
||||
export type BrowserContextFetchResponseBodyParams = {
|
||||
fetchUid: string,
|
||||
};
|
||||
export type BrowserContextFetchResponseBodyOptions = {
|
||||
|
||||
};
|
||||
export type BrowserContextFetchResponseBodyResult = {
|
||||
binary?: Binary,
|
||||
};
|
||||
export type BrowserContextDisposeFetchResponseParams = {
|
||||
fetchUid: string,
|
||||
};
|
||||
export type BrowserContextDisposeFetchResponseOptions = {
|
||||
|
||||
};
|
||||
export type BrowserContextDisposeFetchResponseResult = void;
|
||||
export type BrowserContextGrantPermissionsParams = {
|
||||
permissions: string[],
|
||||
origin?: string,
|
||||
|
||||
@ -217,6 +217,39 @@ InterceptedResponse:
|
||||
items: NameValue
|
||||
|
||||
|
||||
FetchRequest:
|
||||
type: interface
|
||||
|
||||
commands:
|
||||
|
||||
fetch:
|
||||
parameters:
|
||||
url: string
|
||||
params:
|
||||
type: array?
|
||||
items: NameValue
|
||||
method: string?
|
||||
headers:
|
||||
type: array?
|
||||
items: NameValue
|
||||
postData: binary?
|
||||
timeout: number?
|
||||
failOnStatusCode: boolean?
|
||||
returns:
|
||||
response: FetchResponse?
|
||||
error: string?
|
||||
|
||||
fetchResponseBody:
|
||||
parameters:
|
||||
fetchUid: string
|
||||
returns:
|
||||
binary?: binary
|
||||
|
||||
disposeFetchResponse:
|
||||
parameters:
|
||||
fetchUid: string
|
||||
|
||||
|
||||
FetchResponse:
|
||||
type: object
|
||||
properties:
|
||||
@ -417,6 +450,12 @@ Playwright:
|
||||
parameters:
|
||||
uid: string
|
||||
|
||||
newRequest:
|
||||
parameters:
|
||||
ignoreHTTPSErrors: boolean?
|
||||
returns:
|
||||
request: FetchRequest
|
||||
|
||||
events:
|
||||
socksRequested:
|
||||
parameters:
|
||||
@ -579,6 +618,7 @@ BrowserContext:
|
||||
|
||||
initializer:
|
||||
isChromium: boolean
|
||||
fetchRequest: FetchRequest
|
||||
|
||||
commands:
|
||||
|
||||
@ -613,33 +653,6 @@ BrowserContext:
|
||||
name: string
|
||||
needsHandle: boolean?
|
||||
|
||||
fetch:
|
||||
parameters:
|
||||
url: string
|
||||
params:
|
||||
type: array?
|
||||
items: NameValue
|
||||
method: string?
|
||||
headers:
|
||||
type: array?
|
||||
items: NameValue
|
||||
postData: binary?
|
||||
timeout: number?
|
||||
failOnStatusCode: boolean?
|
||||
returns:
|
||||
response: FetchResponse?
|
||||
error: string?
|
||||
|
||||
fetchResponseBody:
|
||||
parameters:
|
||||
fetchUid: string
|
||||
returns:
|
||||
binary?: binary
|
||||
|
||||
disposeFetchResponse:
|
||||
parameters:
|
||||
fetchUid: string
|
||||
|
||||
grantPermissions:
|
||||
parameters:
|
||||
permissions:
|
||||
|
||||
@ -147,6 +147,21 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
|
||||
statusText: tString,
|
||||
headers: tArray(tType('NameValue')),
|
||||
});
|
||||
scheme.FetchRequestFetchParams = tObject({
|
||||
url: tString,
|
||||
params: tOptional(tArray(tType('NameValue'))),
|
||||
method: tOptional(tString),
|
||||
headers: tOptional(tArray(tType('NameValue'))),
|
||||
postData: tOptional(tBinary),
|
||||
timeout: tOptional(tNumber),
|
||||
failOnStatusCode: tOptional(tBoolean),
|
||||
});
|
||||
scheme.FetchRequestFetchResponseBodyParams = tObject({
|
||||
fetchUid: tString,
|
||||
});
|
||||
scheme.FetchRequestDisposeFetchResponseParams = tObject({
|
||||
fetchUid: tString,
|
||||
});
|
||||
scheme.FetchResponse = tObject({
|
||||
fetchUid: tString,
|
||||
url: tString,
|
||||
@ -177,6 +192,9 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
|
||||
scheme.PlaywrightSocksEndParams = tObject({
|
||||
uid: tString,
|
||||
});
|
||||
scheme.PlaywrightNewRequestParams = tObject({
|
||||
ignoreHTTPSErrors: tOptional(tBoolean),
|
||||
});
|
||||
scheme.SelectorsRegisterParams = tObject({
|
||||
name: tString,
|
||||
source: tString,
|
||||
@ -392,21 +410,6 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
|
||||
name: tString,
|
||||
needsHandle: tOptional(tBoolean),
|
||||
});
|
||||
scheme.BrowserContextFetchParams = tObject({
|
||||
url: tString,
|
||||
params: tOptional(tArray(tType('NameValue'))),
|
||||
method: tOptional(tString),
|
||||
headers: tOptional(tArray(tType('NameValue'))),
|
||||
postData: tOptional(tBinary),
|
||||
timeout: tOptional(tNumber),
|
||||
failOnStatusCode: tOptional(tBoolean),
|
||||
});
|
||||
scheme.BrowserContextFetchResponseBodyParams = tObject({
|
||||
fetchUid: tString,
|
||||
});
|
||||
scheme.BrowserContextDisposeFetchResponseParams = tObject({
|
||||
fetchUid: tString,
|
||||
});
|
||||
scheme.BrowserContextGrantPermissionsParams = tObject({
|
||||
permissions: tArray(tString),
|
||||
origin: tOptional(tString),
|
||||
|
||||
@ -34,6 +34,7 @@ import { Tracing } from './trace/recorder/tracing';
|
||||
import { HarRecorder } from './supplements/har/harRecorder';
|
||||
import { RecorderSupplement } from './supplements/recorderSupplement';
|
||||
import * as consoleApiSource from '../generated/consoleApiSource';
|
||||
import { BrowserContextFetchRequest } from './fetch';
|
||||
|
||||
export abstract class BrowserContext extends SdkObject {
|
||||
static Events = {
|
||||
@ -63,7 +64,7 @@ export abstract class BrowserContext extends SdkObject {
|
||||
private _origins = new Set<string>();
|
||||
readonly _harRecorder: HarRecorder | undefined;
|
||||
readonly tracing: Tracing;
|
||||
readonly fetchResponses: Map<string, Buffer> = new Map();
|
||||
readonly fetchRequest = new BrowserContextFetchRequest(this);
|
||||
|
||||
constructor(browser: Browser, options: types.BrowserContextOptions, browserContextId: string | undefined) {
|
||||
super(browser, 'browser-context');
|
||||
@ -133,7 +134,7 @@ export abstract class BrowserContext extends SdkObject {
|
||||
this._closedStatus = 'closed';
|
||||
this._deleteAllDownloads();
|
||||
this._downloads.clear();
|
||||
this.fetchResponses.clear();
|
||||
this.fetchRequest.dispose();
|
||||
if (this._isPersistentContext)
|
||||
this._onClosePersistent();
|
||||
this._closePromiseFulfill!(new Error('Context closed'));
|
||||
@ -382,12 +383,6 @@ export abstract class BrowserContext extends SdkObject {
|
||||
this.on(BrowserContext.Events.Page, installInPage);
|
||||
return Promise.all(this.pages().map(installInPage));
|
||||
}
|
||||
|
||||
storeFetchResponseBody(body: Buffer): string {
|
||||
const uid = createGuid();
|
||||
this.fetchResponses.set(uid, body);
|
||||
return uid;
|
||||
}
|
||||
}
|
||||
|
||||
export function assertBrowserContextIsNotOwned(context: BrowserContext) {
|
||||
|
||||
@ -22,26 +22,65 @@ import * as https from 'https';
|
||||
import { BrowserContext } from './browserContext';
|
||||
import * as types from './types';
|
||||
import { pipeline, Readable, Transform } from 'stream';
|
||||
import { monotonicTime } from '../utils/utils';
|
||||
import { createGuid, monotonicTime } from '../utils/utils';
|
||||
import { SdkObject } from './instrumentation';
|
||||
import { Playwright } from './playwright';
|
||||
import { HeadersArray, ProxySettings } from './types';
|
||||
import { HTTPCredentials } from '../../types/types';
|
||||
import { TimeoutSettings } from '../utils/timeoutSettings';
|
||||
|
||||
export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: Omit<types.FetchResponse, 'body'> & { fetchUid: string }, error?: string}> {
|
||||
|
||||
type FetchRequestOptions = {
|
||||
userAgent: string;
|
||||
extraHTTPHeaders?: HeadersArray;
|
||||
httpCredentials?: HTTPCredentials;
|
||||
proxy?: ProxySettings;
|
||||
timeoutSettings: TimeoutSettings;
|
||||
ignoreHTTPSErrors?: boolean;
|
||||
baseURL?: string;
|
||||
};
|
||||
|
||||
export abstract class FetchRequest extends SdkObject {
|
||||
readonly fetchResponses: Map<string, Buffer> = new Map();
|
||||
|
||||
constructor(parent: SdkObject) {
|
||||
super(parent, 'fetchRequest');
|
||||
}
|
||||
|
||||
dispose() {
|
||||
this.fetchResponses.clear();
|
||||
}
|
||||
|
||||
abstract _defaultOptions(): FetchRequestOptions;
|
||||
abstract _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void>;
|
||||
abstract _cookies(url: string): Promise<types.NetworkCookie[]>;
|
||||
|
||||
private _storeResponseBody(body: Buffer): string {
|
||||
const uid = createGuid();
|
||||
this.fetchResponses.set(uid, body);
|
||||
return uid;
|
||||
}
|
||||
|
||||
async fetch(params: types.FetchOptions): Promise<{fetchResponse?: Omit<types.FetchResponse, 'body'> & { fetchUid: string }, error?: string}> {
|
||||
try {
|
||||
const headers: { [name: string]: string } = {};
|
||||
const defaults = this._defaultOptions();
|
||||
headers['user-agent'] = defaults.userAgent;
|
||||
headers['accept'] = '*/*';
|
||||
headers['accept-encoding'] = 'gzip,deflate,br';
|
||||
|
||||
if (defaults.extraHTTPHeaders) {
|
||||
for (const {name, value} of defaults.extraHTTPHeaders)
|
||||
headers[name.toLowerCase()] = value;
|
||||
}
|
||||
|
||||
if (params.headers) {
|
||||
for (const [name, value] of Object.entries(params.headers))
|
||||
headers[name.toLowerCase()] = value;
|
||||
}
|
||||
headers['user-agent'] ??= context._options.userAgent || context._browser.userAgent();
|
||||
headers['accept'] ??= '*/*';
|
||||
headers['accept-encoding'] ??= 'gzip,deflate,br';
|
||||
|
||||
if (context._options.extraHTTPHeaders) {
|
||||
for (const {name, value} of context._options.extraHTTPHeaders)
|
||||
headers[name.toLowerCase()] = value;
|
||||
}
|
||||
|
||||
const method = params.method?.toUpperCase() || 'GET';
|
||||
const proxy = context._options.proxy || context._browser.options.proxy;
|
||||
const proxy = defaults.proxy;
|
||||
let agent;
|
||||
if (proxy) {
|
||||
// TODO: support bypass proxy
|
||||
@ -51,7 +90,7 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet
|
||||
agent = new HttpsProxyAgent(proxyOpts);
|
||||
}
|
||||
|
||||
const timeout = context._timeoutSettings.timeout(params);
|
||||
const timeout = defaults.timeoutSettings.timeout(params);
|
||||
const deadline = monotonicTime() + timeout;
|
||||
|
||||
const options: https.RequestOptions & { maxRedirects: number, deadline: number } = {
|
||||
@ -63,26 +102,26 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet
|
||||
deadline
|
||||
};
|
||||
// rejectUnauthorized = undefined is treated as true in node 12.
|
||||
if (context._options.ignoreHTTPSErrors)
|
||||
if (defaults.ignoreHTTPSErrors)
|
||||
options.rejectUnauthorized = false;
|
||||
|
||||
const requestUrl = new URL(params.url, context._options.baseURL);
|
||||
const requestUrl = new URL(params.url, defaults.baseURL);
|
||||
if (params.params) {
|
||||
for (const [name, value] of Object.entries(params.params))
|
||||
requestUrl.searchParams.set(name, value);
|
||||
}
|
||||
|
||||
const fetchResponse = await sendRequest(context, requestUrl, options, params.postData);
|
||||
const fetchUid = context.storeFetchResponseBody(fetchResponse.body);
|
||||
const fetchResponse = await this._sendRequest(requestUrl, options, params.postData);
|
||||
const fetchUid = this._storeResponseBody(fetchResponse.body);
|
||||
if (params.failOnStatusCode && (fetchResponse.status < 200 || fetchResponse.status >= 400))
|
||||
return { error: `${fetchResponse.status} ${fetchResponse.statusText}` };
|
||||
return { fetchResponse: { ...fetchResponse, fetchUid } };
|
||||
} catch (e) {
|
||||
return { error: String(e) };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function updateCookiesFromHeader(context: BrowserContext, responseUrl: string, setCookie: string[]) {
|
||||
private async _updateCookiesFromHeader(responseUrl: string, setCookie: string[]) {
|
||||
const url = new URL(responseUrl);
|
||||
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
|
||||
const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
|
||||
@ -102,27 +141,27 @@ async function updateCookiesFromHeader(context: BrowserContext, responseUrl: str
|
||||
cookies.push(cookie);
|
||||
}
|
||||
if (cookies.length)
|
||||
await context.addCookies(cookies);
|
||||
}
|
||||
await this._addCookies(cookies);
|
||||
}
|
||||
|
||||
async function updateRequestCookieHeader(context: BrowserContext, url: URL, options: http.RequestOptions) {
|
||||
private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) {
|
||||
if (options.headers!['cookie'] !== undefined)
|
||||
return;
|
||||
const cookies = await context.cookies(url.toString());
|
||||
const cookies = await this._cookies(url.toString());
|
||||
if (cookies.length) {
|
||||
const valueArray = cookies.map(c => `${c.name}=${c.value}`);
|
||||
options.headers!['cookie'] = valueArray.join('; ');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function sendRequest(context: BrowserContext, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.FetchResponse>{
|
||||
await updateRequestCookieHeader(context, url, options);
|
||||
private async _sendRequest(url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.FetchResponse>{
|
||||
await this._updateRequestCookieHeader(url, options);
|
||||
return new Promise<types.FetchResponse>((fulfill, reject) => {
|
||||
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
|
||||
= (url.protocol === 'https:' ? https : http).request;
|
||||
const request = requestConstructor(url, options, async response => {
|
||||
if (response.headers['set-cookie'])
|
||||
await updateCookiesFromHeader(context, response.url || url.toString(), response.headers['set-cookie']);
|
||||
await this._updateCookiesFromHeader(response.url || url.toString(), response.headers['set-cookie']);
|
||||
if (redirectStatus.includes(response.statusCode!)) {
|
||||
if (!options.maxRedirects) {
|
||||
reject(new Error('Max redirect count exceeded'));
|
||||
@ -157,19 +196,19 @@ async function sendRequest(context: BrowserContext, url: URL, options: https.Req
|
||||
// HTTP-redirect fetch step 4: If locationURL is null, then return response.
|
||||
if (response.headers.location) {
|
||||
const locationURL = new URL(response.headers.location, url);
|
||||
fulfill(sendRequest(context, locationURL, redirectOptions, postData));
|
||||
fulfill(this._sendRequest(locationURL, redirectOptions, postData));
|
||||
request.abort();
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (response.statusCode === 401 && !options.headers!['authorization']) {
|
||||
const auth = response.headers['www-authenticate'];
|
||||
const credentials = context._options.httpCredentials;
|
||||
const credentials = this._defaultOptions().httpCredentials;
|
||||
if (auth?.trim().startsWith('Basic ') && credentials) {
|
||||
const {username, password} = credentials;
|
||||
const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
|
||||
options.headers!['authorization'] = `Basic ${encoded}`;
|
||||
fulfill(sendRequest(context, url, options, postData));
|
||||
fulfill(this._sendRequest(url, options, postData));
|
||||
request.abort();
|
||||
return;
|
||||
}
|
||||
@ -225,6 +264,61 @@ async function sendRequest(context: BrowserContext, url: URL, options: https.Req
|
||||
request.write(postData);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export class BrowserContextFetchRequest extends FetchRequest {
|
||||
private readonly _context: BrowserContext;
|
||||
|
||||
constructor(context: BrowserContext) {
|
||||
super(context);
|
||||
this._context = context;
|
||||
}
|
||||
|
||||
_defaultOptions(): FetchRequestOptions {
|
||||
return {
|
||||
userAgent: this._context._options.userAgent || this._context._browser.userAgent(),
|
||||
extraHTTPHeaders: this._context._options.extraHTTPHeaders,
|
||||
httpCredentials: this._context._options.httpCredentials,
|
||||
proxy: this._context._options.proxy || this._context._browser.options.proxy,
|
||||
timeoutSettings: this._context._timeoutSettings,
|
||||
ignoreHTTPSErrors: this._context._options.ignoreHTTPSErrors,
|
||||
baseURL: this._context._options.baseURL,
|
||||
};
|
||||
}
|
||||
|
||||
async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
|
||||
await this._context.addCookies(cookies);
|
||||
}
|
||||
|
||||
async _cookies(url: string): Promise<types.NetworkCookie[]> {
|
||||
return await this._context.cookies(url);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export class GlobalFetchRequest extends FetchRequest {
|
||||
constructor(playwright: Playwright) {
|
||||
super(playwright);
|
||||
}
|
||||
|
||||
_defaultOptions(): FetchRequestOptions {
|
||||
return {
|
||||
userAgent: '',
|
||||
extraHTTPHeaders: undefined,
|
||||
proxy: undefined,
|
||||
timeoutSettings: new TimeoutSettings(),
|
||||
ignoreHTTPSErrors: false,
|
||||
baseURL: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
|
||||
}
|
||||
|
||||
async _cookies(url: string): Promise<types.NetworkCookie[]> {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function toHeadersArray(rawHeaders: string[]): types.HeadersArray {
|
||||
|
||||
@ -228,7 +228,7 @@ export class Route extends SdkObject {
|
||||
if (body === undefined) {
|
||||
if (overrides.fetchResponseUid) {
|
||||
const context = this._request.frame()._page._browserContext;
|
||||
const buffer = context.fetchResponses.get(overrides.fetchResponseUid);
|
||||
const buffer = context.fetchRequest.fetchResponses.get(overrides.fetchResponseUid);
|
||||
assert(buffer, 'Fetch response has been disposed');
|
||||
body = buffer.toString('utf8');
|
||||
isBase64 = false;
|
||||
|
||||
@ -40,6 +40,19 @@ it.afterAll(() => {
|
||||
http.globalAgent = prevAgent;
|
||||
});
|
||||
|
||||
it('global get should work', async ({playwright, context, server}) => {
|
||||
const request = await playwright._newRequest();
|
||||
const response = await request.get(server.PREFIX + '/simple.json');
|
||||
expect(response.url()).toBe(server.PREFIX + '/simple.json');
|
||||
expect(response.status()).toBe(200);
|
||||
expect(response.statusText()).toBe('OK');
|
||||
expect(response.ok()).toBeTruthy();
|
||||
expect(response.url()).toBe(server.PREFIX + '/simple.json');
|
||||
expect(response.headers()['content-type']).toBe('application/json; charset=utf-8');
|
||||
expect(response.headersArray()).toContainEqual({ name: 'Content-Type', value: 'application/json; charset=utf-8' });
|
||||
expect(await response.text()).toBe('{"foo": "bar"}\n');
|
||||
});
|
||||
|
||||
it('get should work', async ({context, server}) => {
|
||||
const response = await context._request.get(server.PREFIX + '/simple.json');
|
||||
expect(response.url()).toBe(server.PREFIX + '/simple.json');
|
||||
|
||||
@ -57,6 +57,7 @@ it('should scope context handles', async ({browserType, browserOptions, server})
|
||||
{ _guid: 'request', objects: [] },
|
||||
{ _guid: 'response', objects: [] },
|
||||
]},
|
||||
{ _guid: 'fetchRequest', objects: [] }
|
||||
] },
|
||||
] },
|
||||
{ _guid: 'electron', objects: [] },
|
||||
@ -140,7 +141,8 @@ it('should scope browser handles', async ({browserType, browserOptions}) => {
|
||||
{ _guid: 'browser-type', objects: [
|
||||
{
|
||||
_guid: 'browser', objects: [
|
||||
{ _guid: 'browser-context', objects: [] }
|
||||
{ _guid: 'browser-context', objects: [] },
|
||||
{ _guid: 'fetchRequest', objects: [] }
|
||||
]
|
||||
},
|
||||
]
|
||||
|
||||
1
types/types.d.ts
vendored
1
types/types.d.ts
vendored
@ -10668,6 +10668,7 @@ export const firefox: BrowserType;
|
||||
export const webkit: BrowserType;
|
||||
export const _electron: Electron;
|
||||
export const _android: Android;
|
||||
export const _newRequest: () => Promise<FetchRequest>;
|
||||
|
||||
// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459
|
||||
export {};
|
||||
|
||||
1
utils/generate_types/overrides.d.ts
vendored
1
utils/generate_types/overrides.d.ts
vendored
@ -348,6 +348,7 @@ export const firefox: BrowserType;
|
||||
export const webkit: BrowserType;
|
||||
export const _electron: Electron;
|
||||
export const _android: Android;
|
||||
export const _newRequest: () => Promise<FetchRequest>;
|
||||
|
||||
// This is required to not export everything by default. See https://github.com/Microsoft/TypeScript/issues/19545#issuecomment-340490459
|
||||
export {};
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user