From c0010d16c687f3b2b7ba9ed8032efa14f0075cb0 Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Tue, 24 Aug 2021 14:29:04 -0700 Subject: [PATCH] feat: introduce BrowserContext._fetch (#8349) --- package-lock.json | 144 ++++++++++++++++++ package.json | 2 + src/client/browserContext.ts | 17 ++- src/client/network.ts | 46 ++++++ src/dispatchers/browserContextDispatcher.ts | 22 +++ src/protocol/channels.ts | 24 +++ src/protocol/protocol.yml | 23 +++ src/protocol/validator.ts | 13 ++ src/server/fetch.ts | 148 +++++++++++++++++++ src/server/types.ts | 26 +++- tests/browsercontext-fetch.spec.ts | 153 ++++++++++++++++++++ 11 files changed, 609 insertions(+), 9 deletions(-) create mode 100644 src/server/fetch.ts create mode 100644 tests/browsercontext-fetch.spec.ts diff --git a/package-lock.json b/package-lock.json index 6eb651b259..ad1ed40e6c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,7 @@ "mime": "^2.4.6", "minimatch": "^3.0.3", "ms": "^2.1.2", + "node-fetch": "^2.6.1", "pirates": "^4.0.1", "pixelmatch": "^5.2.1", "pngjs": "^5.0.0", @@ -62,6 +63,7 @@ "@types/mime": "^2.0.3", "@types/minimatch": "^3.0.3", "@types/node": "^10.17.28", + "@types/node-fetch": "^2.5.12", "@types/pixelmatch": "^5.2.1", "@types/pngjs": "^3.4.2", "@types/progress": "^2.0.3", @@ -1481,6 +1483,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" }, + "node_modules/@types/node-fetch": { + "version": "2.5.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz", + "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==", + "dev": true, + "dependencies": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "node_modules/@types/pixelmatch": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.3.tgz", @@ -2359,6 +2371,12 @@ "dev": true, "optional": true }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, "node_modules/atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", @@ -3249,6 +3267,18 @@ "node": ">=0.1.90" } }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/commander": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", @@ -3698,6 +3728,15 @@ "node": ">=0.10.0" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true, + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/des.js": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", @@ -4973,6 +5012,20 @@ "node": ">=0.10.0" } }, + "node_modules/form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/formidable": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.2.tgz", @@ -6674,6 +6727,27 @@ "node": ">=4.0.0" } }, + "node_modules/mime-db": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", + "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==", + "dev": true, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.32", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz", + "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==", + "dev": true, + "dependencies": { + "mime-db": "1.49.0" + }, + "engines": { + "node": ">= 0.6" + } + }, "node_modules/mimic-response": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", @@ -6893,6 +6967,14 @@ "integrity": "sha512-gS9GVHRU+RGn5KQM2rllAlR3dU6m7AcpJKdtH8gFvQiC4Otgk98XnmMU+nZenHt/+VhnBPWwgrJsyrdcw6i23w==", "dev": true }, + "node_modules/node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==", + "engines": { + "node": "4.x || >=6.0.0" + } + }, "node_modules/node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", @@ -11337,6 +11419,16 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.60.tgz", "integrity": "sha512-F0KIgDJfy2nA3zMLmWGKxcH2ZVEtCZXHHdOQs2gSaQ27+lNeEfGxzkIw90aXswATX7AZ33tahPbzy6KAfUreVw==" }, + "@types/node-fetch": { + "version": "2.5.12", + "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.5.12.tgz", + "integrity": "sha512-MKgC4dlq4kKNa/mYrwpKfzQMB5X3ee5U6fSprkKpToBqBmX4nFZL9cW5jl6sWn+xpRJ7ypWh2yyqqr8UUCstSw==", + "dev": true, + "requires": { + "@types/node": "*", + "form-data": "^3.0.0" + } + }, "@types/pixelmatch": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/pixelmatch/-/pixelmatch-5.2.3.tgz", @@ -12083,6 +12175,12 @@ "dev": true, "optional": true }, + "asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k=", + "dev": true + }, "atob": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz", @@ -12829,6 +12927,15 @@ "resolved": "https://registry.npmjs.org/colors/-/colors-1.4.0.tgz", "integrity": "sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA==" }, + "combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dev": true, + "requires": { + "delayed-stream": "~1.0.0" + } + }, "commander": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/commander/-/commander-6.2.1.tgz", @@ -13197,6 +13304,12 @@ } } }, + "delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk=", + "dev": true + }, "des.js": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/des.js/-/des.js-1.0.1.tgz", @@ -14265,6 +14378,17 @@ "integrity": "sha1-gQaNKVqBQuwKxybG4iAMMPttXoA=", "dev": true }, + "form-data": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-3.0.1.tgz", + "integrity": "sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==", + "dev": true, + "requires": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "mime-types": "^2.1.12" + } + }, "formidable": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/formidable/-/formidable-1.2.2.tgz", @@ -15631,6 +15755,21 @@ "resolved": "https://registry.npmjs.org/mime/-/mime-2.5.2.tgz", "integrity": "sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg==" }, + "mime-db": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.49.0.tgz", + "integrity": "sha512-CIc8j9URtOVApSFCQIF+VBkX1RwXp/oMMOrqdyXSBXq5RWNEsRfyj1kiRnQgmNXmHxPoFIxOroKA3zcU9P+nAA==", + "dev": true + }, + "mime-types": { + "version": "2.1.32", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.32.tgz", + "integrity": "sha512-hJGaVS4G4c9TSMYh2n6SQAGrC4RnfU+daP8G7cSCmaqNjiOoUY0VHCMS42pxnQmVF1GWwFhbHWn3RIxCqTmZ9A==", + "dev": true, + "requires": { + "mime-db": "1.49.0" + } + }, "mimic-response": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz", @@ -15820,6 +15959,11 @@ } } }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, "node-libs-browser": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/node-libs-browser/-/node-libs-browser-2.2.1.tgz", diff --git a/package.json b/package.json index 0db099bb72..c8cd015bb8 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,7 @@ "mime": "^2.4.6", "minimatch": "^3.0.3", "ms": "^2.1.2", + "node-fetch": "^2.6.1", "pirates": "^4.0.1", "pixelmatch": "^5.2.1", "pngjs": "^5.0.0", @@ -90,6 +91,7 @@ "@types/mime": "^2.0.3", "@types/minimatch": "^3.0.3", "@types/node": "^10.17.28", + "@types/node-fetch": "^2.5.12", "@types/pixelmatch": "^5.2.1", "@types/pngjs": "^3.4.2", "@types/progress": "^2.0.3", diff --git a/src/client/browserContext.ts b/src/client/browserContext.ts index a1da538548..30d3ee66f3 100644 --- a/src/client/browserContext.ts +++ b/src/client/browserContext.ts @@ -28,7 +28,7 @@ import { Events } from './events'; import { TimeoutSettings } from '../utils/timeoutSettings'; import { Waiter } from './waiter'; import { URLMatch, Headers, WaitForEventOptions, BrowserContextOptions, StorageState, LaunchOptions } from './types'; -import { isUnderTest, headersObjectToArray, mkdirIfNeeded } from '../utils/utils'; +import { isUnderTest, headersObjectToArray, mkdirIfNeeded, isString } from '../utils/utils'; import { isSafeCloseError } from '../utils/errors'; import * as api from '../../types/types'; import * as structs from '../../types/structs'; @@ -209,6 +209,21 @@ export class BrowserContext extends ChannelOwner { + return this._wrapApiCall(async (channel: channels.BrowserContextChannel) => { + const postDataBuffer = isString(options.postData) ? Buffer.from(options.postData, 'utf8') : options.postData; + const result = await channel.fetch({ + url, + method: options.method, + headers: options.headers ? headersObjectToArray(options.headers) : undefined, + postData: postDataBuffer ? postDataBuffer.toString('base64') : undefined, + }); + if (result.error) + throw new Error(`Request failed: ${result.error}`); + return new network.FetchResponse(result.response!); + }); + } + async setGeolocation(geolocation: { longitude: number, latitude: number, accuracy?: number } | null): Promise { return this._wrapApiCall(async (channel: channels.BrowserContextChannel) => { await channel.setGeolocation({ geolocation: geolocation || undefined }); diff --git a/src/client/network.ts b/src/client/network.ts index 3699627351..8e3c319f4f 100644 --- a/src/client/network.ts +++ b/src/client/network.ts @@ -454,6 +454,52 @@ export class Response extends ChannelOwner= 200 && this._initializer.status <= 299); + } + + url(): string { + return this._initializer.url; + } + + status(): number { + return this._initializer.status; + } + + statusText(): string { + return this._initializer.statusText; + } + + headers(): Headers { + return { ...this._headers }; + } + + async body(): Promise { + return this._body; + } + + async text(): Promise { + const content = await this.body(); + return content.toString('utf8'); + } + + async json(): Promise { + const content = await this.text(); + return JSON.parse(content); + } +} + export class WebSocket extends ChannelOwner implements api.WebSocket { private _page: Page; private _isClosed: boolean; diff --git a/src/dispatchers/browserContextDispatcher.ts b/src/dispatchers/browserContextDispatcher.ts index 650969c452..eb3d6c6042 100644 --- a/src/dispatchers/browserContextDispatcher.ts +++ b/src/dispatchers/browserContextDispatcher.ts @@ -17,6 +17,7 @@ import { BrowserContext } from '../server/browserContext'; import { Dispatcher, DispatcherScope, lookupDispatcher } from './dispatcher'; import { PageDispatcher, BindingCallDispatcher, WorkerDispatcher } from './pageDispatcher'; +import { playwrightFetch } from '../server/fetch'; import { FrameDispatcher } from './frameDispatcher'; import * as channels from '../protocol/channels'; import { RouteDispatcher, RequestDispatcher, ResponseDispatcher } from './networkDispatchers'; @@ -27,6 +28,7 @@ import { CallMetadata } from '../server/instrumentation'; import { ArtifactDispatcher } from './artifactDispatcher'; import { Artifact } from '../server/artifact'; import { Request, Response } from '../server/network'; +import { headersArrayToObject } from '../utils/utils'; export class BrowserContextDispatcher extends Dispatcher implements channels.BrowserContextChannel { private _context: BrowserContext; @@ -104,6 +106,26 @@ export class BrowserContextDispatcher extends Dispatcher { + const { fetchResponse, error } = await playwrightFetch(this._context, { + url: params.url, + method: params.method, + headers: params.headers ? headersArrayToObject(params.headers, false) : undefined, + postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined, + }); + let response; + if (fetchResponse) { + response = { + url: fetchResponse.url, + status: fetchResponse.status, + statusText: fetchResponse.statusText, + headers: fetchResponse.headers, + body: fetchResponse.body.toString('base64') + }; + } + return { response, error }; + } + async newPage(params: channels.BrowserContextNewPageParams, metadata: CallMetadata): Promise { return { page: lookupDispatcher(await this._context.newPage(metadata)) }; } diff --git a/src/protocol/channels.ts b/src/protocol/channels.ts index 74176ef724..c931502bfa 100644 --- a/src/protocol/channels.ts +++ b/src/protocol/channels.ts @@ -152,6 +152,14 @@ export type InterceptedResponse = { }[], }; +export type FetchResponse = { + url: string, + status: number, + statusText: string, + headers: NameValue[], + body: Binary, +}; + // ----------- Root ----------- export type RootInitializer = {}; export interface RootChannel extends Channel { @@ -706,6 +714,7 @@ export interface BrowserContextChannel extends EventTargetChannel { close(params?: BrowserContextCloseParams, metadata?: Metadata): Promise; cookies(params: BrowserContextCookiesParams, metadata?: Metadata): Promise; exposeBinding(params: BrowserContextExposeBindingParams, metadata?: Metadata): Promise; + fetch(params: BrowserContextFetchParams, metadata?: Metadata): Promise; grantPermissions(params: BrowserContextGrantPermissionsParams, metadata?: Metadata): Promise; newPage(params?: BrowserContextNewPageParams, metadata?: Metadata): Promise; setDefaultNavigationTimeoutNoReply(params: BrowserContextSetDefaultNavigationTimeoutNoReplyParams, metadata?: Metadata): Promise; @@ -802,6 +811,21 @@ export type BrowserContextExposeBindingOptions = { needsHandle?: boolean, }; export type BrowserContextExposeBindingResult = void; +export type BrowserContextFetchParams = { + url: string, + method?: string, + headers?: NameValue[], + postData?: Binary, +}; +export type BrowserContextFetchOptions = { + method?: string, + headers?: NameValue[], + postData?: Binary, +}; +export type BrowserContextFetchResult = { + response?: FetchResponse, + error?: string, +}; export type BrowserContextGrantPermissionsParams = { permissions: string[], origin?: string, diff --git a/src/protocol/protocol.yml b/src/protocol/protocol.yml index 97a7738963..15658455e3 100644 --- a/src/protocol/protocol.yml +++ b/src/protocol/protocol.yml @@ -220,6 +220,17 @@ InterceptedResponse: value: string +FetchResponse: + type: object + properties: + url: string + status: number + statusText: string + headers: + type: array + items: NameValue + body: binary + LaunchOptions: type: mixin properties: @@ -591,6 +602,18 @@ BrowserContext: name: string needsHandle: boolean? + fetch: + parameters: + url: string + method: string? + headers: + type: array? + items: NameValue + postData: binary? + returns: + response: FetchResponse? + error: string? + grantPermissions: parameters: permissions: diff --git a/src/protocol/validator.ts b/src/protocol/validator.ts index b115d6fa66..a055c7c041 100644 --- a/src/protocol/validator.ts +++ b/src/protocol/validator.ts @@ -149,6 +149,13 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { value: tString, })), }); + scheme.FetchResponse = tObject({ + url: tString, + status: tNumber, + statusText: tString, + headers: tArray(tType('NameValue')), + body: tBinary, + }); scheme.RootInitializeParams = tObject({ sdkLanguage: tString, }); @@ -379,6 +386,12 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { name: tString, needsHandle: tOptional(tBoolean), }); + scheme.BrowserContextFetchParams = tObject({ + url: tString, + method: tOptional(tString), + headers: tOptional(tArray(tType('NameValue'))), + postData: tOptional(tBinary), + }); scheme.BrowserContextGrantPermissionsParams = tObject({ permissions: tArray(tString), origin: tOptional(tString), diff --git a/src/server/fetch.ts b/src/server/fetch.ts new file mode 100644 index 0000000000..23ada7257c --- /dev/null +++ b/src/server/fetch.ts @@ -0,0 +1,148 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { HttpsProxyAgent } from 'https-proxy-agent'; +import nodeFetch from 'node-fetch'; +import * as url from 'url'; +import { BrowserContext } from './browserContext'; +import * as types from './types'; + +export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> { + try { + const cookies = await context.cookies(params.url); + const valueArray = cookies.map(c => `${c.name}=${c.value}`); + const clientCookie = params.headers?.['cookie']; + if (clientCookie) + valueArray.unshift(clientCookie); + const cookieHeader = valueArray.join('; '); + if (cookieHeader) { + if (!params.headers) + params.headers = {}; + params.headers['cookie'] = cookieHeader; + } + if (!params.method) + params.method = 'GET'; + let agent; + if (context._options.proxy) { + // TODO: support bypass proxy + const proxyOpts = url.parse(context._options.proxy.server); + if (context._options.proxy.username) + proxyOpts.auth = `${context._options.proxy.username}:${context._options.proxy.password || ''}`; + agent = new HttpsProxyAgent(proxyOpts); + } + + // TODO(https://github.com/microsoft/playwright/issues/8381): set user agent + const response = await nodeFetch(params.url, { + method: params.method, + headers: params.headers, + body: params.postData, + agent + }); + const body = await response.buffer(); + const setCookies = response.headers.raw()['set-cookie']; + if (setCookies) { + const url = new URL(response.url); + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4 + const defaultPath = '/' + url.pathname.split('/').slice(0, -1).join('/'); + const cookies: types.SetNetworkCookieParam[] = []; + for (const header of setCookies) { + // Decode cookie value? + const cookie: types.SetNetworkCookieParam | null = parseCookie(header); + if (!cookie) + continue; + if (!cookie.domain) + cookie.domain = url.hostname; + if (!canSetCookie(cookie.domain!, url.hostname)) + continue; + // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4 + if (!cookie.path || !cookie.path.startsWith('/')) + cookie.path = defaultPath; + cookies.push(cookie); + } + if (cookies.length) + await context.addCookies(cookies); + } + + const headers: types.HeadersArray = []; + for (const [name, value] of response.headers.entries()) + headers.push({ name, value }); + return { + fetchResponse: { + url: response.url, + status: response.status, + statusText: response.statusText, + headers, + body + } + }; + } catch (e) { + return { error: String(e) }; + } +} + +function canSetCookie(cookieDomain: string, hostname: string) { + // TODO: check public suffix list? + hostname = '.' + hostname; + if (!cookieDomain.startsWith('.')) + cookieDomain = '.' + cookieDomain; + return hostname.endsWith(cookieDomain); +} + + +function parseCookie(header: string) { + const pairs = header.split(';').filter(s => s.trim().length > 0).map(p => p.split('=').map(s => s.trim())); + if (!pairs.length) + return null; + const [name, value] = pairs[0]; + const cookie: types.NetworkCookie = { + name, + value, + domain: '', + path: '', + expires: -1, + httpOnly: false, + secure: false, + sameSite: 'Lax' // None for non-chromium + }; + for (let i = 1; i < pairs.length; i++) { + const [name, value] = pairs[i]; + switch (name.toLowerCase()) { + case 'expires': + const expiresMs = (+new Date(value)); + if (isFinite(expiresMs)) + cookie.expires = expiresMs / 1000; + break; + case 'max-age': + const maxAgeSec = parseInt(value, 10); + if (isFinite(maxAgeSec)) + cookie.expires = Date.now() / 1000 + maxAgeSec; + break; + case 'domain': + cookie.domain = value || ''; + break; + case 'path': + cookie.path = value || ''; + break; + case 'secure': + cookie.secure = true; + break; + case 'httponly': + cookie.httpOnly = true; + break; + } + } + return cookie; +} diff --git a/src/server/types.ts b/src/server/types.ts index a5a946b740..5a3f557d5d 100644 --- a/src/server/types.ts +++ b/src/server/types.ts @@ -211,14 +211,6 @@ export type NormalizedContinueOverrides = { interceptResponse?: boolean, }; -export type NormalizedResponseContinueOverrides = { - status?: number, - statusText?: string, - headers?: HeadersArray, - body?: string, - isBase64?: boolean, -}; - export type NetworkCookie = { name: string, value: string, @@ -375,3 +367,21 @@ export type SetStorageState = { cookies?: SetNetworkCookieParam[], origins?: OriginStorage[] }; + +export type FetchOptions = { + url: string, + method?: string, + headers?: { [name: string]: string }, + postData?: Buffer, +}; + +export type FetchResponse = { + url: string, + status: number, + statusText: string, + headers: { + name: string, + value: string, + }[], + body: Buffer, +}; diff --git a/tests/browsercontext-fetch.spec.ts b/tests/browsercontext-fetch.spec.ts new file mode 100644 index 0000000000..8a507d78a6 --- /dev/null +++ b/tests/browsercontext-fetch.spec.ts @@ -0,0 +1,153 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { contextTest as it, expect } from './config/browserTest'; + +it('should work', async ({context, server}) => { + // @ts-expect-error + const response = await context._fetch(server.PREFIX + '/simple.json'); + expect(response.url()).toBe(server.PREFIX + '/simple.json'); + expect(response.status()).toBe(200); + expect(response.statusText()).toBe('OK'); + expect(response.ok()).toBeTruthy(); + expect(response.url()).toBe(server.PREFIX + '/simple.json'); + expect(response.headers()['content-type']).toBe('application/json; charset=utf-8'); + expect(await response.text()).toBe('{"foo": "bar"}\n'); +}); + +it('should add session cookies to request', async ({context, server, isLinux}) => { + await context.addCookies([{ + name: 'username', + value: 'John Doe', + domain: isLinux ? '.my.localhost' : 'localhost', + path: '/', + expires: -1, + httpOnly: false, + secure: false, + sameSite: 'Lax', + }]); + const [req] = await Promise.all([ + server.waitForRequest('/simple.json'), + // @ts-expect-error + context._fetch(`http://${isLinux ? 'www.my.localhost' : 'localhost'}:${server.PORT}/simple.json`), + ]); + expect(req.headers.cookie).toEqual('username=John Doe'); +}); + +it('should follow redirects', async ({context, server, isLinux}) => { + server.setRedirect('/redirect1', '/redirect2'); + server.setRedirect('/redirect2', '/simple.json'); + await context.addCookies([{ + name: 'username', + value: 'John Doe', + domain: isLinux ? '.my.localhost' : 'localhost', + path: '/', + expires: -1, + httpOnly: false, + secure: false, + sameSite: 'Lax', + }]); + const [req, response] = await Promise.all([ + server.waitForRequest('/simple.json'), + // @ts-expect-error + context._fetch(`http://${isLinux ? 'www.my.localhost' : 'localhost'}:${server.PORT}/redirect1`), + ]); + expect(req.headers.cookie).toEqual('username=John Doe'); + expect(response.url()).toBe(`http://${isLinux ? 'www.my.localhost' : 'localhost'}:${server.PORT}/simple.json`); + expect(await response.json()).toEqual({foo: 'bar'}); +}); + +it('should add cookies from Set-Cookie header', async ({context, page, server}) => { + server.setRoute('/setcookie.html', (req, res) => { + res.setHeader('Set-Cookie', ['session=value', 'foo=bar; max-age=3600']); + res.end(); + }); + // @ts-expect-error + await context._fetch(server.PREFIX + '/setcookie.html'); + const cookies = await context.cookies(); + expect(new Set(cookies.map(c => ({ name: c.name, value: c.value })))).toEqual(new Set([ + { + name: 'session', + value: 'value' + }, + { + name: 'foo', + value: 'bar' + }, + ])); + await page.goto(server.EMPTY_PAGE); + expect((await page.evaluate(() => document.cookie)).split(';').map(s => s.trim()).sort()).toEqual(['foo=bar', 'session=value']); +}); + +it('should work with context level proxy', async ({browserOptions, browserType, contextOptions, server, proxyServer}) => { + server.setRoute('/target.html', async (req, res) => { + res.end('Served by the proxy'); + }); + + const browser = await browserType.launch({ + ...browserOptions, + proxy: { server: 'http://per-context' } + }); + + try { + proxyServer.forwardTo(server.PORT); + const context = await browser.newContext({ + ...contextOptions, + proxy: { server: `localhost:${proxyServer.PORT}` } + }); + + const [request, response] = await Promise.all([ + server.waitForRequest('/target.html'), + // @ts-expect-error + context._fetch(`http://non-existent.com/target.html`) + ]); + expect(response.status()).toBe(200); + expect(request.url).toBe('/target.html'); + } finally { + await browser.close(); + } +}); + +it('should work with http credentials', async ({context, server}) => { + server.setAuth('/empty.html', 'user', 'pass'); + + const [request, response] = await Promise.all([ + server.waitForRequest('/empty.html'), + // @ts-expect-error + context._fetch(server.EMPTY_PAGE, { + headers: { + 'authorization': 'Basic ' + Buffer.from('user:pass').toString('base64') + } + }) + ]); + expect(response.status()).toBe(200); + expect(request.url).toBe('/empty.html'); +}); + +it('should support post data', async ({context, server}) => { + const [request, response] = await Promise.all([ + server.waitForRequest('/simple.json'), + // @ts-expect-error + context._fetch(`${server.PREFIX}/simple.json`, { + method: 'POST', + postData: 'My request' + }) + ]); + expect(request.method).toBe('POST'); + expect((await request.postBody).toString()).toBe('My request'); + expect(response.status()).toBe(200); + expect(request.url).toBe('/simple.json'); +});