2021-08-24 14:29:04 -07:00
|
|
|
/**
|
|
|
|
* Copyright (c) Microsoft Corporation.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
|
|
|
import { HttpsProxyAgent } from 'https-proxy-agent';
|
2021-08-26 16:18:54 -07:00
|
|
|
import url from 'url';
|
2021-08-31 09:34:58 -07:00
|
|
|
import zlib from 'zlib';
|
2021-08-26 16:18:54 -07:00
|
|
|
import * as http from 'http';
|
|
|
|
import * as https from 'https';
|
2021-08-24 14:29:04 -07:00
|
|
|
import { BrowserContext } from './browserContext';
|
|
|
|
import * as types from './types';
|
2021-08-31 09:34:58 -07:00
|
|
|
import { pipeline, Readable, Transform } from 'stream';
|
2021-09-08 10:01:40 -07:00
|
|
|
import { monotonicTime } from '../utils/utils';
|
2021-08-24 14:29:04 -07:00
|
|
|
|
2021-09-08 13:40:07 -07:00
|
|
|
export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: Omit<types.FetchResponse, 'body'> & { fetchUid: string }, error?: string}> {
|
2021-08-24 14:29:04 -07:00
|
|
|
try {
|
2021-08-27 08:26:19 -07:00
|
|
|
const headers: { [name: string]: string } = {};
|
|
|
|
if (params.headers) {
|
|
|
|
for (const [name, value] of Object.entries(params.headers))
|
|
|
|
headers[name.toLowerCase()] = value;
|
|
|
|
}
|
2021-08-27 15:28:36 -07:00
|
|
|
headers['user-agent'] ??= context._options.userAgent || context._browser.userAgent();
|
|
|
|
headers['accept'] ??= '*/*';
|
2021-08-31 09:34:58 -07:00
|
|
|
headers['accept-encoding'] ??= 'gzip,deflate,br';
|
2021-08-27 08:26:19 -07:00
|
|
|
|
2021-08-27 23:47:21 -07:00
|
|
|
if (context._options.extraHTTPHeaders) {
|
|
|
|
for (const {name, value} of context._options.extraHTTPHeaders)
|
|
|
|
headers[name.toLowerCase()] = value;
|
|
|
|
}
|
|
|
|
|
2021-08-27 15:28:36 -07:00
|
|
|
const method = params.method?.toUpperCase() || 'GET';
|
2021-09-07 17:42:31 -07:00
|
|
|
const proxy = context._options.proxy || context._browser.options.proxy;
|
2021-08-24 14:29:04 -07:00
|
|
|
let agent;
|
2021-09-07 17:42:31 -07:00
|
|
|
if (proxy) {
|
2021-08-24 14:29:04 -07:00
|
|
|
// TODO: support bypass proxy
|
2021-09-07 17:42:31 -07:00
|
|
|
const proxyOpts = url.parse(proxy.server);
|
|
|
|
if (proxy.username)
|
|
|
|
proxyOpts.auth = `${proxy.username}:${proxy.password || ''}`;
|
2021-08-24 14:29:04 -07:00
|
|
|
agent = new HttpsProxyAgent(proxyOpts);
|
|
|
|
}
|
|
|
|
|
2021-09-08 10:01:40 -07:00
|
|
|
const timeout = context._timeoutSettings.timeout(params);
|
|
|
|
const deadline = monotonicTime() + timeout;
|
|
|
|
|
2021-08-30 13:41:25 -07:00
|
|
|
const fetchResponse = await sendRequest(context, new URL(params.url, context._options.baseURL), {
|
2021-08-27 15:28:36 -07:00
|
|
|
method,
|
|
|
|
headers,
|
2021-08-26 16:18:54 -07:00
|
|
|
agent,
|
2021-09-08 10:01:40 -07:00
|
|
|
maxRedirects: 20,
|
|
|
|
timeout,
|
|
|
|
deadline
|
2021-08-26 16:18:54 -07:00
|
|
|
}, params.postData);
|
2021-09-08 13:40:07 -07:00
|
|
|
const fetchUid = context.storeFetchResponseBody(fetchResponse.body);
|
|
|
|
return { fetchResponse: { ...fetchResponse, fetchUid } };
|
2021-08-24 14:29:04 -07:00
|
|
|
} catch (e) {
|
|
|
|
return { error: String(e) };
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-26 16:18:54 -07:00
|
|
|
async function updateCookiesFromHeader(context: BrowserContext, responseUrl: string, setCookie: string[]) {
|
|
|
|
const url = new URL(responseUrl);
|
|
|
|
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
|
2021-08-27 15:28:36 -07:00
|
|
|
const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
|
2021-08-26 16:18:54 -07:00
|
|
|
const cookies: types.SetNetworkCookieParam[] = [];
|
|
|
|
for (const header of setCookie) {
|
|
|
|
// Decode cookie value?
|
|
|
|
const cookie: types.SetNetworkCookieParam | null = parseCookie(header);
|
|
|
|
if (!cookie)
|
|
|
|
continue;
|
|
|
|
if (!cookie.domain)
|
|
|
|
cookie.domain = url.hostname;
|
|
|
|
if (!canSetCookie(cookie.domain!, url.hostname))
|
|
|
|
continue;
|
|
|
|
// https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4
|
|
|
|
if (!cookie.path || !cookie.path.startsWith('/'))
|
|
|
|
cookie.path = defaultPath;
|
|
|
|
cookies.push(cookie);
|
|
|
|
}
|
|
|
|
if (cookies.length)
|
|
|
|
await context.addCookies(cookies);
|
|
|
|
}
|
|
|
|
|
2021-08-27 15:28:36 -07:00
|
|
|
async function updateRequestCookieHeader(context: BrowserContext, url: URL, options: http.RequestOptions) {
|
|
|
|
if (options.headers!['cookie'] !== undefined)
|
|
|
|
return;
|
|
|
|
const cookies = await context.cookies(url.toString());
|
|
|
|
if (cookies.length) {
|
|
|
|
const valueArray = cookies.map(c => `${c.name}=${c.value}`);
|
|
|
|
options.headers!['cookie'] = valueArray.join('; ');
|
|
|
|
}
|
|
|
|
}
|
2021-08-26 16:18:54 -07:00
|
|
|
|
2021-09-08 10:01:40 -07:00
|
|
|
async function sendRequest(context: BrowserContext, url: URL, options: http.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.FetchResponse>{
|
2021-08-27 15:28:36 -07:00
|
|
|
await updateRequestCookieHeader(context, url, options);
|
|
|
|
return new Promise<types.FetchResponse>((fulfill, reject) => {
|
2021-08-26 16:18:54 -07:00
|
|
|
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
|
|
|
|
= (url.protocol === 'https:' ? https : http).request;
|
2021-08-27 15:28:36 -07:00
|
|
|
const request = requestConstructor(url, options, async response => {
|
|
|
|
if (response.headers['set-cookie'])
|
|
|
|
await updateCookiesFromHeader(context, response.url || url.toString(), response.headers['set-cookie']);
|
2021-08-26 16:18:54 -07:00
|
|
|
if (redirectStatus.includes(response.statusCode!)) {
|
|
|
|
if (!options.maxRedirects) {
|
|
|
|
reject(new Error('Max redirect count exceeded'));
|
|
|
|
request.abort();
|
|
|
|
return;
|
|
|
|
}
|
2021-08-27 15:28:36 -07:00
|
|
|
const headers = { ...options.headers };
|
|
|
|
delete headers[`cookie`];
|
2021-08-26 16:18:54 -07:00
|
|
|
|
|
|
|
// HTTP-redirect fetch step 13 (https://fetch.spec.whatwg.org/#http-redirect-fetch)
|
|
|
|
const status = response.statusCode!;
|
2021-08-27 15:28:36 -07:00
|
|
|
let method = options.method!;
|
2021-08-26 16:18:54 -07:00
|
|
|
if ((status === 301 || status === 302) && method === 'POST' ||
|
|
|
|
status === 303 && !['GET', 'HEAD'].includes(method)) {
|
2021-08-27 15:28:36 -07:00
|
|
|
method = 'GET';
|
2021-08-26 16:18:54 -07:00
|
|
|
postData = undefined;
|
2021-08-27 15:28:36 -07:00
|
|
|
delete headers[`content-encoding`];
|
|
|
|
delete headers[`content-language`];
|
|
|
|
delete headers[`content-location`];
|
|
|
|
delete headers[`content-type`];
|
2021-08-26 16:18:54 -07:00
|
|
|
}
|
|
|
|
|
2021-09-08 10:01:40 -07:00
|
|
|
const redirectOptions: http.RequestOptions & { maxRedirects: number, deadline: number } = {
|
2021-08-27 15:28:36 -07:00
|
|
|
method,
|
|
|
|
headers,
|
|
|
|
agent: options.agent,
|
|
|
|
maxRedirects: options.maxRedirects - 1,
|
2021-09-08 10:01:40 -07:00
|
|
|
timeout: options.timeout,
|
|
|
|
deadline: options.deadline
|
2021-08-27 15:28:36 -07:00
|
|
|
};
|
2021-08-26 16:18:54 -07:00
|
|
|
|
|
|
|
// HTTP-redirect fetch step 4: If locationURL is null, then return response.
|
|
|
|
if (response.headers.location) {
|
|
|
|
const locationURL = new URL(response.headers.location, url);
|
2021-08-27 15:28:36 -07:00
|
|
|
fulfill(sendRequest(context, locationURL, redirectOptions, postData));
|
2021-08-26 16:18:54 -07:00
|
|
|
request.abort();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2021-08-27 23:47:33 -07:00
|
|
|
if (response.statusCode === 401 && !options.headers!['authorization']) {
|
|
|
|
const auth = response.headers['www-authenticate'];
|
|
|
|
const credentials = context._options.httpCredentials;
|
|
|
|
if (auth?.trim().startsWith('Basic ') && credentials) {
|
|
|
|
const {username, password} = credentials;
|
|
|
|
const encoded = Buffer.from(`${username || ''}:${password || ''}`).toString('base64');
|
|
|
|
options.headers!['authorization'] = `Basic ${encoded}`;
|
|
|
|
fulfill(sendRequest(context, url, options, postData));
|
|
|
|
request.abort();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2021-08-31 09:34:58 -07:00
|
|
|
response.on('aborted', () => reject(new Error('aborted')));
|
|
|
|
|
|
|
|
let body: Readable = response;
|
|
|
|
let transform: Transform | undefined;
|
|
|
|
const encoding = response.headers['content-encoding'];
|
|
|
|
if (encoding === 'gzip' || encoding === 'x-gzip') {
|
|
|
|
transform = zlib.createGunzip({
|
|
|
|
flush: zlib.constants.Z_SYNC_FLUSH,
|
|
|
|
finishFlush: zlib.constants.Z_SYNC_FLUSH
|
|
|
|
});
|
|
|
|
} else if (encoding === 'br') {
|
|
|
|
transform = zlib.createBrotliDecompress();
|
|
|
|
} else if (encoding === 'deflate') {
|
|
|
|
transform = zlib.createInflate();
|
|
|
|
}
|
|
|
|
if (transform) {
|
|
|
|
body = pipeline(response, transform, e => {
|
|
|
|
if (e)
|
|
|
|
reject(new Error(`failed to decompress '${encoding}' encoding: ${e}`));
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2021-08-26 16:18:54 -07:00
|
|
|
const chunks: Buffer[] = [];
|
2021-08-31 09:34:58 -07:00
|
|
|
body.on('data', chunk => chunks.push(chunk));
|
|
|
|
body.on('end', () => {
|
2021-08-26 16:18:54 -07:00
|
|
|
const body = Buffer.concat(chunks);
|
|
|
|
fulfill({
|
2021-08-27 15:28:36 -07:00
|
|
|
url: response.url || url.toString(),
|
|
|
|
status: response.statusCode || 0,
|
|
|
|
statusText: response.statusMessage || '',
|
|
|
|
headers: flattenHeaders(response.headers),
|
|
|
|
body
|
2021-08-26 16:18:54 -07:00
|
|
|
});
|
|
|
|
});
|
2021-08-31 09:34:58 -07:00
|
|
|
body.on('error',reject);
|
2021-08-26 16:18:54 -07:00
|
|
|
});
|
|
|
|
request.on('error', reject);
|
2021-09-08 10:01:40 -07:00
|
|
|
const rejectOnTimeout = () => {
|
|
|
|
reject(new Error(`Request timed out after ${options.timeout}ms`));
|
|
|
|
request.abort();
|
|
|
|
};
|
|
|
|
const remaining = options.deadline - monotonicTime();
|
|
|
|
if (remaining <= 0) {
|
|
|
|
rejectOnTimeout();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
request.setTimeout(remaining, rejectOnTimeout);
|
2021-08-26 16:18:54 -07:00
|
|
|
if (postData)
|
|
|
|
request.write(postData);
|
|
|
|
request.end();
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
function flattenHeaders(headers: http.IncomingHttpHeaders): types.HeadersArray {
|
|
|
|
const result: types.HeadersArray = [];
|
|
|
|
for (const [name, values] of Object.entries(headers)) {
|
|
|
|
if (values === undefined)
|
|
|
|
continue;
|
|
|
|
if (typeof values === 'string') {
|
|
|
|
result.push({name, value: values as string});
|
|
|
|
} else {
|
|
|
|
for (const value of values)
|
|
|
|
result.push({name, value});
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
|
|
|
const redirectStatus = [301, 302, 303, 307, 308];
|
|
|
|
|
2021-08-24 14:29:04 -07:00
|
|
|
function canSetCookie(cookieDomain: string, hostname: string) {
|
|
|
|
// TODO: check public suffix list?
|
|
|
|
hostname = '.' + hostname;
|
|
|
|
if (!cookieDomain.startsWith('.'))
|
|
|
|
cookieDomain = '.' + cookieDomain;
|
|
|
|
return hostname.endsWith(cookieDomain);
|
|
|
|
}
|
|
|
|
|
|
|
|
function parseCookie(header: string) {
|
|
|
|
const pairs = header.split(';').filter(s => s.trim().length > 0).map(p => p.split('=').map(s => s.trim()));
|
|
|
|
if (!pairs.length)
|
|
|
|
return null;
|
|
|
|
const [name, value] = pairs[0];
|
|
|
|
const cookie: types.NetworkCookie = {
|
|
|
|
name,
|
|
|
|
value,
|
|
|
|
domain: '',
|
|
|
|
path: '',
|
|
|
|
expires: -1,
|
|
|
|
httpOnly: false,
|
|
|
|
secure: false,
|
|
|
|
sameSite: 'Lax' // None for non-chromium
|
|
|
|
};
|
|
|
|
for (let i = 1; i < pairs.length; i++) {
|
|
|
|
const [name, value] = pairs[i];
|
|
|
|
switch (name.toLowerCase()) {
|
|
|
|
case 'expires':
|
|
|
|
const expiresMs = (+new Date(value));
|
|
|
|
if (isFinite(expiresMs))
|
|
|
|
cookie.expires = expiresMs / 1000;
|
|
|
|
break;
|
|
|
|
case 'max-age':
|
|
|
|
const maxAgeSec = parseInt(value, 10);
|
|
|
|
if (isFinite(maxAgeSec))
|
|
|
|
cookie.expires = Date.now() / 1000 + maxAgeSec;
|
|
|
|
break;
|
|
|
|
case 'domain':
|
|
|
|
cookie.domain = value || '';
|
|
|
|
break;
|
|
|
|
case 'path':
|
|
|
|
cookie.path = value || '';
|
|
|
|
break;
|
|
|
|
case 'secure':
|
|
|
|
cookie.secure = true;
|
|
|
|
break;
|
|
|
|
case 'httponly':
|
|
|
|
cookie.httpOnly = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return cookie;
|
|
|
|
}
|