yujunjun/playwright
1
0
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-06-26 21:40:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(fetch): happy eyeballs (#19902)

Browse Source 
Fixes #18790
This commit is contained in:
Yury Semikhatsky 2023-01-05 14:39:49 -08:00 committed by GitHub
parent 4359111289
commit 10ccfa9517
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 253 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/playwright-core/src/client/fetch.ts
View File

@ -195,6 +195,9 @@ export class APIRequestContext extends ChannelOwner<channels.APIRequestContextCh
}
if (postDataBuffer === undefined && jsonData === undefined && formData === undefined && multipartData === undefined)
postDataBuffer = options.request?.postDataBuffer() || undefined;
const fixtures = {
__testHookLookup: (options as any).__testHookLookup
};
const result = await this._channel.fetch({
url,
params,
@ -208,6 +211,7 @@ export class APIRequestContext extends ChannelOwner<channels.APIRequestContextCh
failOnStatusCode: options.failOnStatusCode,
ignoreHTTPSErrors: options.ignoreHTTPSErrors,
maxRedirects: maxRedirects,
...fixtures
});
return new APIResponse(this, result.response);
});

29
packages/playwright-core/src/server/fetch.ts
View File

@ -14,6 +14,8 @@
* limitations under the License.
*/
import type * as channels from '@protocol/channels';
import type { LookupAddress } from 'dns';
import * as http from 'http';
import * as https from 'https';
import type { Readable, TransformCallback } from 'stream';
@ -21,13 +23,14 @@ import { pipeline, Transform } from 'stream';
import url from 'url';
import zlib from 'zlib';
import type { HTTPCredentials } from '../../types/types';
import type * as channels from '@protocol/channels';
import { TimeoutSettings } from '../common/timeoutSettings';
import { getUserAgent } from '../common/userAgent';
import { assert, createGuid, monotonicTime } from '../utils';
import { HttpsProxyAgent, SocksProxyAgent } from '../utilsBundle';
import { BrowserContext } from './browserContext';
import { CookieStore, domainMatches } from './cookieStore';
import { MultipartFormData } from './formData';
import { httpHappyEyeballsAgent, httpsHappyEyeballsAgent } from './happy-eyeballs';
import type { CallMetadata } from './instrumentation';
import { SdkObject } from './instrumentation';
import type { Playwright } from './playwright';
@ -36,7 +39,6 @@ import { ProgressController } from './progress';
import { Tracing } from './trace/recorder/tracing';
import type * as types from './types';
import type { HeadersArray, ProxySettings } from './types';
import { HttpsProxyAgent, SocksProxyAgent } from '../utilsBundle';
type FetchRequestOptions = {
userAgent: string;
@ -67,6 +69,12 @@ export type APIRequestFinishedEvent = {
body?: Buffer;
};
export type SendRequestOptions = https.RequestOptions & {
maxRedirects: number,
deadline: number,
__testHookLookup?: (hostname: string) => LookupAddress[]
};
export abstract class APIRequestContext extends SdkObject {
static Events = {
Dispose: 'dispose',
@ -159,13 +167,14 @@ export abstract class APIRequestContext extends SdkObject {
const timeout = defaults.timeoutSettings.timeout(params);
const deadline = timeout && (monotonicTime() + timeout);
const options: https.RequestOptions & { maxRedirects: number, deadline: number } = {
const options: SendRequestOptions = {
method,
headers,
agent,
maxRedirects: params.maxRedirects === 0 ? -1 : params.maxRedirects === undefined ? 20 : params.maxRedirects,
timeout,
deadline
deadline,
__testHookLookup: (params as any).__testHookLookup,
};
// rejectUnauthorized = undefined is treated as true in node 12.
if (params.ignoreHTTPSErrors || defaults.ignoreHTTPSErrors)
@ -228,7 +237,7 @@ export abstract class APIRequestContext extends SdkObject {
}
}
private async _sendRequest(progress: Progress, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<Omit<channels.APIResponse, 'fetchUid'> & { body: Buffer }>{
private async _sendRequest(progress: Progress, url: URL, options: SendRequestOptions, postData?: Buffer): Promise<Omit<channels.APIResponse, 'fetchUid'> & { body: Buffer }>{
await this._updateRequestCookieHeader(url, options);
const requestCookies = (options.headers!['cookie'] as (string | undefined))?.split(';').map(p => {
@ -247,7 +256,10 @@ export abstract class APIRequestContext extends SdkObject {
return new Promise((fulfill, reject) => {
const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
= (url.protocol === 'https:' ? https : http).request;
const request = requestConstructor(url, options, async response => {
// If we have a proxy agent already, do not override it.
const agent = options.agent || (url.protocol === 'https:' ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent);
const requestOptions = { ...options, agent };
const request = requestConstructor(url, requestOptions as any, async response => {
const notifyRequestFinished = (body?: Buffer) => {
const requestFinishedEvent: APIRequestFinishedEvent = {
requestEvent,
@ -292,13 +304,14 @@ export abstract class APIRequestContext extends SdkObject {
delete headers[`content-type`];
}
const redirectOptions: https.RequestOptions & { maxRedirects: number, deadline: number } = {
const redirectOptions: SendRequestOptions = {
method,
headers,
agent: options.agent,
maxRedirects: options.maxRedirects - 1,
timeout: options.timeout,
deadline: options.deadline
deadline: options.deadline,
__testHookLookup: options.__testHookLookup,
};
// rejectUnauthorized = undefined is treated as true in node 12.
if (options.rejectUnauthorized === false)

128
packages/playwright-core/src/server/happy-eyeballs.ts Normal file
View File

@ -0,0 +1,128 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import * as dns from 'dns';
import * as http from 'http';
import * as https from 'https';
import * as net from 'net';
import * as tls from 'tls';
import { ManualPromise } from '../utils/manualPromise';
import type { SendRequestOptions } from './fetch';
// Implementation(partial) of Happy Eyeballs 2 algorithm described in
// https://www.rfc-editor.org/rfc/rfc8305
// Same as in Chromium (https://source.chromium.org/chromium/chromium/src/+/5666ff4f5077a7e2f72902f3a95f5d553ea0d88d:net/socket/transport_connect_job.cc;l=102)
const connectionAttemptDelayMs = 300;
class HttpHappyEyeballsAgent extends http.Agent {
createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void): net.Socket | undefined {
// There is no ambiguity in case of IP address.
if (net.isIP(options.hostname!))
return net.createConnection(options as net.NetConnectOpts);
createConnectionAsync(options, oncreate).catch(err => oncreate?.(err));
}
}
class HttpsHappyEyeballsAgent extends https.Agent {
createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void): net.Socket | undefined {
// There is no ambiguity in case of IP address.
if (net.isIP(options.hostname!))
return tls.connect(options as tls.ConnectionOptions);
createConnectionAsync(options, oncreate).catch(err => oncreate?.(err));
}
}
export const httpsHappyEyeballsAgent = new HttpsHappyEyeballsAgent();
export const httpHappyEyeballsAgent = new HttpHappyEyeballsAgent();
async function createConnectionAsync(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void) {
const lookup = (options as SendRequestOptions).__testHookLookup || lookupAddresses;
const addresses = await lookup(options.hostname!);
const sockets = new Set<net.Socket>();
let firstError;
let errorCount = 0;
const handleError = (socket: net.Socket, err: Error) => {
if (!sockets.delete(socket))
return;
++errorCount;
firstError ??= err;
if (errorCount === addresses.length)
oncreate?.(firstError);
};
const connected = new ManualPromise();
for (const { address } of addresses) {
const socket = options.protocol === 'https:' ?
tls.connect({
...(options as tls.ConnectionOptions),
port: options.port as number,
host: address,
servername: options.hostname || undefined }) :
net.createConnection({
...options,
port: options.port as number,
host: address });
// Each socket may fire only one of 'connect', 'timeout' or 'error' events.
// None of these events are fired after socket.destroy() is called.
socket.on('connect', () => {
connected.resolve();
oncreate?.(null, socket);
// TODO: Cache the result?
// Close other outstanding sockets.
sockets.delete(socket);
for (const s of sockets)
s.destroy();
sockets.clear();
});
socket.on('timeout', () => {
// Timeout is not an error, so we have to manually close the socket.
socket.destroy();
handleError(socket, new Error('Connection timeout'));
});
socket.on('error', e => handleError(socket, e));
sockets.add(socket);
await Promise.race([
connected,
new Promise(f => setTimeout(f, connectionAttemptDelayMs))
]);
if (connected.isDone())
break;
}
}
async function lookupAddresses(hostname: string): Promise<dns.LookupAddress[]> {
const addresses = await dns.promises.lookup(hostname, { all: true, family: 0, verbatim: true });
let firstFamily = addresses.filter(({ family }) => family === 6);
let secondFamily = addresses.filter(({ family }) => family === 4);
// Make sure first address in the list is the same as in the original order.
if (firstFamily.length && firstFamily[0] !== addresses[0]) {
const tmp = firstFamily;
firstFamily = secondFamily;
secondFamily = tmp;
}
const result = [];
// Alternate ipv6 and ipv4 addreses.
for (let i = 0; i < Math.max(firstFamily.length, secondFamily.length); i++) {
if (firstFamily[i])
result.push(firstFamily[i]);
if (secondFamily[i])
result.push(secondFamily[i]);
}
return result;
}

62
tests/library/browsercontext-fetch-happy-eyeballs.spec.ts Normal file
View File

@ -0,0 +1,62 @@
/**
* Copyright (c) Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { LookupAddress } from 'dns';
import { contextTest as it, expect } from '../config/browserTest';
it.skip(({ mode }) => mode !== 'default');
const __testHookLookup = (hostname: string): LookupAddress[] => {
interceptedHostnameLookup = hostname;
if (hostname === 'localhost') {
return [
// First two do are not served (at least on macOS).
{ address: '::2', family: 6 },
{ address: '127.0.0.2', family: 4 },
{ address: '::1', family: 6 },
{ address: '127.0.0.1', family: 4 }];
} else {
throw new Error(`Failed to resolve hostname: ${hostname}`);
}
};
let interceptedHostnameLookup: string | undefined;
it.beforeEach(() => {
interceptedHostnameLookup = undefined;
});
it('get should work', async ({ context, server }) => {
const response = await context.request.get(server.PREFIX + '/simple.json', { __testHookLookup } as any);
expect(response.url()).toBe(server.PREFIX + '/simple.json');
expect(response).toBeOK();
expect(interceptedHostnameLookup).toBe('localhost');
});
it('get should work on request fixture', async ({ request, server }) => {
const response = await request.get(server.PREFIX + '/simple.json', { __testHookLookup } as any);
expect(response.url()).toBe(server.PREFIX + '/simple.json');
expect(response).toBeOK();
expect(interceptedHostnameLookup).toBe('localhost');
});
it('https post should work with ignoreHTTPSErrors option', async ({ context, httpsServer }) => {
const response = await context.request.post(httpsServer.EMPTY_PAGE,
{ ignoreHTTPSErrors: true, __testHookLookup } as any);
expect(response.status()).toBe(200);
expect(interceptedHostnameLookup).toBe('localhost');
});

45
tests/library/browsercontext-fetch.spec.ts
View File

@ -14,35 +14,25 @@
* limitations under the License.
*/
import type { LookupAddress } from 'dns';
import formidable from 'formidable';
import http from 'http';
import zlib from 'zlib';
import fs from 'fs';
import type { IncomingMessage } from 'http';
import { pipeline } from 'stream';
import zlib from 'zlib';
import { contextTest as it, expect } from '../config/browserTest';
import { suppressCertificateWarning } from '../config/utils';
it.skip(({ mode }) => mode !== 'default');
let prevAgent: http.Agent;
it.beforeAll(() => {
prevAgent = http.globalAgent;
http.globalAgent = new http.Agent({
// @ts-expect-error
lookup: (hostname, options, callback) => {
if (hostname === 'localhost' || hostname.endsWith('playwright.dev'))
callback(null, '127.0.0.1', 4);
else
throw new Error(`Failed to resolve hostname: ${hostname}`);
}
});
});
const __testHookLookup = (hostname: string): LookupAddress[] => {
if (hostname === 'localhost' || hostname.endsWith('playwright.dev'))
return [{ address: '127.0.0.1', family: 4 }];
else
throw new Error(`Failed to resolve hostname: ${hostname}`);
};
it.afterAll(() => {
http.globalAgent = prevAgent;
});
it('get should work @smoke', async ({ context, server }) => {
it('get should work @smoke', async ({ context, server, mode }) => {
const response = await context.request.get(server.PREFIX + '/simple.json');
expect(response.url()).toBe(server.PREFIX + '/simple.json');
expect(response.status()).toBe(200);
@ -123,7 +113,9 @@ it('should add session cookies to request', async ({ context, server }) => {
}]);
const [req] = await Promise.all([
server.waitForRequest('/simple.json'),
context.request.get(`http://www.my.playwright.dev:${server.PORT}/simple.json`),
context.request.get(`http://www.my.playwright.dev:${server.PORT}/simple.json`, {
__testHookLookup
} as any),
]);
expect(req.headers.cookie).toEqual('username=John Doe');
});
@ -176,8 +168,9 @@ it('should not add context cookie if cookie header passed as a parameter', async
context.request.get(`http://www.my.playwright.dev:${server.PORT}/empty.html`, {
headers: {
'Cookie': 'foo=bar'
}
}),
},
__testHookLookup
} as any),
]);
expect(req.headers.cookie).toEqual('foo=bar');
});
@ -197,7 +190,7 @@ it('should follow redirects', async ({ context, server }) => {
}]);
const [req, response] = await Promise.all([
server.waitForRequest('/simple.json'),
context.request.get(`http://www.my.playwright.dev:${server.PORT}/redirect1`),
context.request.get(`http://www.my.playwright.dev:${server.PORT}/redirect1`, { __testHookLookup } as any),
]);
expect(req.headers.cookie).toEqual('username=John Doe');
expect(response.url()).toBe(`http://www.my.playwright.dev:${server.PORT}/simple.json`);
@ -858,7 +851,7 @@ it('should encode to application/json by default', async function({ context, pag
});
it('should support multipart/form-data', async function({ context, server }) {
const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record<string, formidable.File>, serverRequest: http.IncomingMessage}>(resolve => {
const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record<string, formidable.File>, serverRequest: IncomingMessage}>(resolve => {
server.setRoute('/empty.html', async (serverRequest, res) => {
const form = new formidable.IncomingForm();
form.parse(serverRequest, (error, fields, files) => {
@ -895,7 +888,7 @@ it('should support multipart/form-data', async function({ context, server }) {
});
it('should support multipart/form-data with ReadSream values', async function({ context, page, asset, server }) {
const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record<string, formidable.File>, serverRequest: http.IncomingMessage}>(resolve => {
const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record<string, formidable.File>, serverRequest: IncomingMessage}>(resolve => {
server.setRoute('/empty.html', async (serverRequest, res) => {
const form = new formidable.IncomingForm();
form.parse(serverRequest, (error, fields, files) => {

49
tests/library/global-fetch-cookie.spec.ts
View File

@ -14,8 +14,8 @@
* limitations under the License.
*/
import type { LookupAddress } from 'dns';
import fs from 'fs';
import http from 'http';
import type { APIRequestContext } from 'playwright-core';
import { expect, playwrightTest } from '../config/browserTest';
@ -36,33 +36,22 @@ type StorageStateType = PromiseArg<ReturnType<APIRequestContext['storageState']>
it.skip(({ mode }) => mode !== 'default');
let prevAgent: http.Agent;
it.beforeAll(() => {
prevAgent = http.globalAgent;
http.globalAgent = new http.Agent({
// @ts-expect-error
lookup: (hostname, options, callback) => {
if (hostname === 'localhost' || hostname.endsWith('one.com') || hostname.endsWith('two.com'))
callback(null, '127.0.0.1', 4);
else
throw new Error(`Failed to resolve hostname: ${hostname}`);
}
});
});
it.afterAll(() => {
http.globalAgent = prevAgent;
});
const __testHookLookup = (hostname: string): LookupAddress[] => {
if (hostname === 'localhost' || hostname.endsWith('one.com') || hostname.endsWith('two.com'))
return [{ address: '127.0.0.1', family: 4 }];
else
throw new Error(`Failed to resolve hostname: ${hostname}`);
};
it('should store cookie from Set-Cookie header', async ({ request, server }) => {
server.setRoute('/setcookie.html', (req, res) => {
res.setHeader('Set-Cookie', ['a=b', 'c=d; max-age=3600; domain=b.one.com; path=/input', 'e=f; domain=b.one.com; path=/input/subfolder']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any);
const [serverRequest] = await Promise.all([
server.waitForRequest('/input/button.html'),
request.get(`http://b.one.com:${server.PORT}/input/button.html`)
request.get(`http://b.one.com:${server.PORT}/input/button.html`, { __testHookLookup } as any)
]);
expect(serverRequest.headers.cookie).toBe('c=d');
});
@ -85,16 +74,16 @@ it('should filter outgoing cookies by domain', async ({ request, server }) => {
res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com', 'c=v; domain=other.com']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://www.b.one.com:${server.PORT}/empty.html`)
request.get(`http://www.b.one.com:${server.PORT}/empty.html`, { __testHookLookup } as any)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
const [serverRequest2] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://two.com:${server.PORT}/empty.html`)
request.get(`http://two.com:${server.PORT}/empty.html`, { __testHookLookup } as any)
]);
expect(serverRequest2.headers.cookie).toBeFalsy();
});
@ -104,10 +93,10 @@ it('should do case-insensitive match of cookie domain', async ({ request, server
res.setHeader('Set-Cookie', ['a=v; domain=One.com', 'b=v; domain=.B.oNe.com']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://www.b.one.com:${server.PORT}/empty.html`)
request.get(`http://www.b.one.com:${server.PORT}/empty.html`, { __testHookLookup } as any)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
});
@ -117,10 +106,10 @@ it('should do case-insensitive match of request domain', async ({ request, serve
res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any);
const [serverRequest] = await Promise.all([
server.waitForRequest('/empty.html'),
request.get(`http://WWW.B.ONE.COM:${server.PORT}/empty.html`)
request.get(`http://WWW.B.ONE.COM:${server.PORT}/empty.html`, { __testHookLookup } as any)
]);
expect(serverRequest.headers.cookie).toBe('a=v; b=v');
});
@ -187,7 +176,7 @@ it('should store cookie from Set-Cookie header even if it contains equal signs',
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any);
const state = await request.storageState();
expect(state).toEqual({
'cookies': [
@ -266,7 +255,7 @@ it('should export cookies to storage state', async ({ request, server }) => {
res.setHeader('Set-Cookie', ['a=b', `c=d; expires=${expires.toUTCString()}; domain=b.one.com; path=/input`, 'e=f; domain=b.one.com; path=/input/subfolder']);
res.end();
});
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`);
await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any);
const state = await request.storageState();
expect(state).toEqual({
'cookies': [
@ -376,7 +365,7 @@ it('should send cookies from storage state', async ({ playwright, server }) => {
const request = await playwright.request.newContext({ storageState });
const [serverRequest] = await Promise.all([
server.waitForRequest('/first/second/third/not_found.html'),
request.get(`http://www.a.b.one.com:${server.PORT}/first/second/third/not_found.html`)
request.get(`http://www.a.b.one.com:${server.PORT}/first/second/third/not_found.html`, { __testHookLookup } as any)
]);
expect(serverRequest.headers.cookie).toBe('c=d; e=f');
});

19
tests/library/global-fetch.spec.ts
View File

@ -14,7 +14,6 @@
* limitations under the License.
*/
import http from 'http';
import os from 'os';
import * as util from 'util';
import { getPlaywrightVersion } from '../../packages/playwright-core/lib/common/userAgent';
@ -22,24 +21,6 @@ import { expect, playwrightTest as it } from '../config/browserTest';
it.skip(({ mode }) => mode !== 'default');
let prevAgent: http.Agent;
it.beforeAll(() => {
prevAgent = http.globalAgent;
http.globalAgent = new http.Agent({
// @ts-expect-error
lookup: (hostname, options, callback) => {
if (hostname === 'localhost' || hostname.endsWith('playwright.dev'))
callback(null, '127.0.0.1', 4);
else
throw new Error(`Failed to resolve hostname: ${hostname}`);
}
});
});
it.afterAll(() => {
http.globalAgent = prevAgent;
});
for (const method of ['fetch', 'delete', 'get', 'head', 'patch', 'post', 'put'] as const) {
it(`${method} should work @smoke`, async ({ playwright, server }) => {
const request = await playwright.request.newContext();