From 10ccfa95173f80574ce4dfd8570204e88169e30e Mon Sep 17 00:00:00 2001 From: Yury Semikhatsky Date: Thu, 5 Jan 2023 14:39:49 -0800 Subject: [PATCH] feat(fetch): happy eyeballs (#19902) Fixes #18790 --- packages/playwright-core/src/client/fetch.ts | 4 + packages/playwright-core/src/server/fetch.ts | 29 ++-- .../src/server/happy-eyeballs.ts | 128 ++++++++++++++++++ ...rowsercontext-fetch-happy-eyeballs.spec.ts | 62 +++++++++ tests/library/browsercontext-fetch.spec.ts | 45 +++--- tests/library/global-fetch-cookie.spec.ts | 49 +++---- tests/library/global-fetch.spec.ts | 19 --- 7 files changed, 253 insertions(+), 83 deletions(-) create mode 100644 packages/playwright-core/src/server/happy-eyeballs.ts create mode 100644 tests/library/browsercontext-fetch-happy-eyeballs.spec.ts diff --git a/packages/playwright-core/src/client/fetch.ts b/packages/playwright-core/src/client/fetch.ts index 7a48c6ede0..ae42727320 100644 --- a/packages/playwright-core/src/client/fetch.ts +++ b/packages/playwright-core/src/client/fetch.ts @@ -195,6 +195,9 @@ export class APIRequestContext extends ChannelOwner LookupAddress[] +}; + export abstract class APIRequestContext extends SdkObject { static Events = { Dispose: 'dispose', @@ -159,13 +167,14 @@ export abstract class APIRequestContext extends SdkObject { const timeout = defaults.timeoutSettings.timeout(params); const deadline = timeout && (monotonicTime() + timeout); - const options: https.RequestOptions & { maxRedirects: number, deadline: number } = { + const options: SendRequestOptions = { method, headers, agent, maxRedirects: params.maxRedirects === 0 ? -1 : params.maxRedirects === undefined ? 20 : params.maxRedirects, timeout, - deadline + deadline, + __testHookLookup: (params as any).__testHookLookup, }; // rejectUnauthorized = undefined is treated as true in node 12. if (params.ignoreHTTPSErrors || defaults.ignoreHTTPSErrors) @@ -228,7 +237,7 @@ export abstract class APIRequestContext extends SdkObject { } } - private async _sendRequest(progress: Progress, url: URL, options: https.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise & { body: Buffer }>{ + private async _sendRequest(progress: Progress, url: URL, options: SendRequestOptions, postData?: Buffer): Promise & { body: Buffer }>{ await this._updateRequestCookieHeader(url, options); const requestCookies = (options.headers!['cookie'] as (string | undefined))?.split(';').map(p => { @@ -247,7 +256,10 @@ export abstract class APIRequestContext extends SdkObject { return new Promise((fulfill, reject) => { const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest) = (url.protocol === 'https:' ? https : http).request; - const request = requestConstructor(url, options, async response => { + // If we have a proxy agent already, do not override it. + const agent = options.agent || (url.protocol === 'https:' ? httpsHappyEyeballsAgent : httpHappyEyeballsAgent); + const requestOptions = { ...options, agent }; + const request = requestConstructor(url, requestOptions as any, async response => { const notifyRequestFinished = (body?: Buffer) => { const requestFinishedEvent: APIRequestFinishedEvent = { requestEvent, @@ -292,13 +304,14 @@ export abstract class APIRequestContext extends SdkObject { delete headers[`content-type`]; } - const redirectOptions: https.RequestOptions & { maxRedirects: number, deadline: number } = { + const redirectOptions: SendRequestOptions = { method, headers, agent: options.agent, maxRedirects: options.maxRedirects - 1, timeout: options.timeout, - deadline: options.deadline + deadline: options.deadline, + __testHookLookup: options.__testHookLookup, }; // rejectUnauthorized = undefined is treated as true in node 12. if (options.rejectUnauthorized === false) diff --git a/packages/playwright-core/src/server/happy-eyeballs.ts b/packages/playwright-core/src/server/happy-eyeballs.ts new file mode 100644 index 0000000000..e346b6411c --- /dev/null +++ b/packages/playwright-core/src/server/happy-eyeballs.ts @@ -0,0 +1,128 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as dns from 'dns'; +import * as http from 'http'; +import * as https from 'https'; +import * as net from 'net'; +import * as tls from 'tls'; +import { ManualPromise } from '../utils/manualPromise'; +import type { SendRequestOptions } from './fetch'; + +// Implementation(partial) of Happy Eyeballs 2 algorithm described in +// https://www.rfc-editor.org/rfc/rfc8305 + +// Same as in Chromium (https://source.chromium.org/chromium/chromium/src/+/5666ff4f5077a7e2f72902f3a95f5d553ea0d88d:net/socket/transport_connect_job.cc;l=102) +const connectionAttemptDelayMs = 300; + +class HttpHappyEyeballsAgent extends http.Agent { + createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void): net.Socket | undefined { + // There is no ambiguity in case of IP address. + if (net.isIP(options.hostname!)) + return net.createConnection(options as net.NetConnectOpts); + createConnectionAsync(options, oncreate).catch(err => oncreate?.(err)); + } +} + +class HttpsHappyEyeballsAgent extends https.Agent { + createConnection(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void): net.Socket | undefined { + // There is no ambiguity in case of IP address. + if (net.isIP(options.hostname!)) + return tls.connect(options as tls.ConnectionOptions); + createConnectionAsync(options, oncreate).catch(err => oncreate?.(err)); + } +} + +export const httpsHappyEyeballsAgent = new HttpsHappyEyeballsAgent(); +export const httpHappyEyeballsAgent = new HttpHappyEyeballsAgent(); + +async function createConnectionAsync(options: http.ClientRequestArgs, oncreate?: (err: Error | null, socket?: net.Socket) => void) { + const lookup = (options as SendRequestOptions).__testHookLookup || lookupAddresses; + const addresses = await lookup(options.hostname!); + const sockets = new Set(); + let firstError; + let errorCount = 0; + const handleError = (socket: net.Socket, err: Error) => { + if (!sockets.delete(socket)) + return; + ++errorCount; + firstError ??= err; + if (errorCount === addresses.length) + oncreate?.(firstError); + }; + + const connected = new ManualPromise(); + for (const { address } of addresses) { + const socket = options.protocol === 'https:' ? + tls.connect({ + ...(options as tls.ConnectionOptions), + port: options.port as number, + host: address, + servername: options.hostname || undefined }) : + net.createConnection({ + ...options, + port: options.port as number, + host: address }); + + // Each socket may fire only one of 'connect', 'timeout' or 'error' events. + // None of these events are fired after socket.destroy() is called. + socket.on('connect', () => { + connected.resolve(); + oncreate?.(null, socket); + // TODO: Cache the result? + // Close other outstanding sockets. + sockets.delete(socket); + for (const s of sockets) + s.destroy(); + sockets.clear(); + }); + socket.on('timeout', () => { + // Timeout is not an error, so we have to manually close the socket. + socket.destroy(); + handleError(socket, new Error('Connection timeout')); + }); + socket.on('error', e => handleError(socket, e)); + sockets.add(socket); + await Promise.race([ + connected, + new Promise(f => setTimeout(f, connectionAttemptDelayMs)) + ]); + if (connected.isDone()) + break; + } +} + +async function lookupAddresses(hostname: string): Promise { + const addresses = await dns.promises.lookup(hostname, { all: true, family: 0, verbatim: true }); + let firstFamily = addresses.filter(({ family }) => family === 6); + let secondFamily = addresses.filter(({ family }) => family === 4); + // Make sure first address in the list is the same as in the original order. + if (firstFamily.length && firstFamily[0] !== addresses[0]) { + const tmp = firstFamily; + firstFamily = secondFamily; + secondFamily = tmp; + } + const result = []; + // Alternate ipv6 and ipv4 addreses. + for (let i = 0; i < Math.max(firstFamily.length, secondFamily.length); i++) { + if (firstFamily[i]) + result.push(firstFamily[i]); + if (secondFamily[i]) + result.push(secondFamily[i]); + } + return result; +} + diff --git a/tests/library/browsercontext-fetch-happy-eyeballs.spec.ts b/tests/library/browsercontext-fetch-happy-eyeballs.spec.ts new file mode 100644 index 0000000000..9de9cd058a --- /dev/null +++ b/tests/library/browsercontext-fetch-happy-eyeballs.spec.ts @@ -0,0 +1,62 @@ +/** + * Copyright (c) Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { LookupAddress } from 'dns'; +import { contextTest as it, expect } from '../config/browserTest'; + +it.skip(({ mode }) => mode !== 'default'); + +const __testHookLookup = (hostname: string): LookupAddress[] => { + interceptedHostnameLookup = hostname; + if (hostname === 'localhost') { + return [ + // First two do are not served (at least on macOS). + { address: '::2', family: 6 }, + { address: '127.0.0.2', family: 4 }, + { address: '::1', family: 6 }, + { address: '127.0.0.1', family: 4 }]; + } else { + throw new Error(`Failed to resolve hostname: ${hostname}`); + } +}; + +let interceptedHostnameLookup: string | undefined; + +it.beforeEach(() => { + interceptedHostnameLookup = undefined; +}); + +it('get should work', async ({ context, server }) => { + const response = await context.request.get(server.PREFIX + '/simple.json', { __testHookLookup } as any); + expect(response.url()).toBe(server.PREFIX + '/simple.json'); + expect(response).toBeOK(); + expect(interceptedHostnameLookup).toBe('localhost'); +}); + +it('get should work on request fixture', async ({ request, server }) => { + const response = await request.get(server.PREFIX + '/simple.json', { __testHookLookup } as any); + expect(response.url()).toBe(server.PREFIX + '/simple.json'); + expect(response).toBeOK(); + expect(interceptedHostnameLookup).toBe('localhost'); +}); + +it('https post should work with ignoreHTTPSErrors option', async ({ context, httpsServer }) => { + const response = await context.request.post(httpsServer.EMPTY_PAGE, + { ignoreHTTPSErrors: true, __testHookLookup } as any); + expect(response.status()).toBe(200); + expect(interceptedHostnameLookup).toBe('localhost'); +}); + diff --git a/tests/library/browsercontext-fetch.spec.ts b/tests/library/browsercontext-fetch.spec.ts index eaf8833e56..0c2bdb25c3 100644 --- a/tests/library/browsercontext-fetch.spec.ts +++ b/tests/library/browsercontext-fetch.spec.ts @@ -14,35 +14,25 @@ * limitations under the License. */ +import type { LookupAddress } from 'dns'; import formidable from 'formidable'; -import http from 'http'; -import zlib from 'zlib'; import fs from 'fs'; +import type { IncomingMessage } from 'http'; import { pipeline } from 'stream'; +import zlib from 'zlib'; import { contextTest as it, expect } from '../config/browserTest'; import { suppressCertificateWarning } from '../config/utils'; it.skip(({ mode }) => mode !== 'default'); -let prevAgent: http.Agent; -it.beforeAll(() => { - prevAgent = http.globalAgent; - http.globalAgent = new http.Agent({ - // @ts-expect-error - lookup: (hostname, options, callback) => { - if (hostname === 'localhost' || hostname.endsWith('playwright.dev')) - callback(null, '127.0.0.1', 4); - else - throw new Error(`Failed to resolve hostname: ${hostname}`); - } - }); -}); +const __testHookLookup = (hostname: string): LookupAddress[] => { + if (hostname === 'localhost' || hostname.endsWith('playwright.dev')) + return [{ address: '127.0.0.1', family: 4 }]; + else + throw new Error(`Failed to resolve hostname: ${hostname}`); +}; -it.afterAll(() => { - http.globalAgent = prevAgent; -}); - -it('get should work @smoke', async ({ context, server }) => { +it('get should work @smoke', async ({ context, server, mode }) => { const response = await context.request.get(server.PREFIX + '/simple.json'); expect(response.url()).toBe(server.PREFIX + '/simple.json'); expect(response.status()).toBe(200); @@ -123,7 +113,9 @@ it('should add session cookies to request', async ({ context, server }) => { }]); const [req] = await Promise.all([ server.waitForRequest('/simple.json'), - context.request.get(`http://www.my.playwright.dev:${server.PORT}/simple.json`), + context.request.get(`http://www.my.playwright.dev:${server.PORT}/simple.json`, { + __testHookLookup + } as any), ]); expect(req.headers.cookie).toEqual('username=John Doe'); }); @@ -176,8 +168,9 @@ it('should not add context cookie if cookie header passed as a parameter', async context.request.get(`http://www.my.playwright.dev:${server.PORT}/empty.html`, { headers: { 'Cookie': 'foo=bar' - } - }), + }, + __testHookLookup + } as any), ]); expect(req.headers.cookie).toEqual('foo=bar'); }); @@ -197,7 +190,7 @@ it('should follow redirects', async ({ context, server }) => { }]); const [req, response] = await Promise.all([ server.waitForRequest('/simple.json'), - context.request.get(`http://www.my.playwright.dev:${server.PORT}/redirect1`), + context.request.get(`http://www.my.playwright.dev:${server.PORT}/redirect1`, { __testHookLookup } as any), ]); expect(req.headers.cookie).toEqual('username=John Doe'); expect(response.url()).toBe(`http://www.my.playwright.dev:${server.PORT}/simple.json`); @@ -858,7 +851,7 @@ it('should encode to application/json by default', async function({ context, pag }); it('should support multipart/form-data', async function({ context, server }) { - const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record, serverRequest: http.IncomingMessage}>(resolve => { + const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record, serverRequest: IncomingMessage}>(resolve => { server.setRoute('/empty.html', async (serverRequest, res) => { const form = new formidable.IncomingForm(); form.parse(serverRequest, (error, fields, files) => { @@ -895,7 +888,7 @@ it('should support multipart/form-data', async function({ context, server }) { }); it('should support multipart/form-data with ReadSream values', async function({ context, page, asset, server }) { - const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record, serverRequest: http.IncomingMessage}>(resolve => { + const formReceived = new Promise<{error: any, fields: formidable.Fields, files: Record, serverRequest: IncomingMessage}>(resolve => { server.setRoute('/empty.html', async (serverRequest, res) => { const form = new formidable.IncomingForm(); form.parse(serverRequest, (error, fields, files) => { diff --git a/tests/library/global-fetch-cookie.spec.ts b/tests/library/global-fetch-cookie.spec.ts index d089db18a7..25532d2f91 100644 --- a/tests/library/global-fetch-cookie.spec.ts +++ b/tests/library/global-fetch-cookie.spec.ts @@ -14,8 +14,8 @@ * limitations under the License. */ +import type { LookupAddress } from 'dns'; import fs from 'fs'; -import http from 'http'; import type { APIRequestContext } from 'playwright-core'; import { expect, playwrightTest } from '../config/browserTest'; @@ -36,33 +36,22 @@ type StorageStateType = PromiseArg it.skip(({ mode }) => mode !== 'default'); -let prevAgent: http.Agent; -it.beforeAll(() => { - prevAgent = http.globalAgent; - http.globalAgent = new http.Agent({ - // @ts-expect-error - lookup: (hostname, options, callback) => { - if (hostname === 'localhost' || hostname.endsWith('one.com') || hostname.endsWith('two.com')) - callback(null, '127.0.0.1', 4); - else - throw new Error(`Failed to resolve hostname: ${hostname}`); - } - }); -}); - -it.afterAll(() => { - http.globalAgent = prevAgent; -}); +const __testHookLookup = (hostname: string): LookupAddress[] => { + if (hostname === 'localhost' || hostname.endsWith('one.com') || hostname.endsWith('two.com')) + return [{ address: '127.0.0.1', family: 4 }]; + else + throw new Error(`Failed to resolve hostname: ${hostname}`); +}; it('should store cookie from Set-Cookie header', async ({ request, server }) => { server.setRoute('/setcookie.html', (req, res) => { res.setHeader('Set-Cookie', ['a=b', 'c=d; max-age=3600; domain=b.one.com; path=/input', 'e=f; domain=b.one.com; path=/input/subfolder']); res.end(); }); - await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any); const [serverRequest] = await Promise.all([ server.waitForRequest('/input/button.html'), - request.get(`http://b.one.com:${server.PORT}/input/button.html`) + request.get(`http://b.one.com:${server.PORT}/input/button.html`, { __testHookLookup } as any) ]); expect(serverRequest.headers.cookie).toBe('c=d'); }); @@ -85,16 +74,16 @@ it('should filter outgoing cookies by domain', async ({ request, server }) => { res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com', 'c=v; domain=other.com']); res.end(); }); - await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any); const [serverRequest] = await Promise.all([ server.waitForRequest('/empty.html'), - request.get(`http://www.b.one.com:${server.PORT}/empty.html`) + request.get(`http://www.b.one.com:${server.PORT}/empty.html`, { __testHookLookup } as any) ]); expect(serverRequest.headers.cookie).toBe('a=v; b=v'); const [serverRequest2] = await Promise.all([ server.waitForRequest('/empty.html'), - request.get(`http://two.com:${server.PORT}/empty.html`) + request.get(`http://two.com:${server.PORT}/empty.html`, { __testHookLookup } as any) ]); expect(serverRequest2.headers.cookie).toBeFalsy(); }); @@ -104,10 +93,10 @@ it('should do case-insensitive match of cookie domain', async ({ request, server res.setHeader('Set-Cookie', ['a=v; domain=One.com', 'b=v; domain=.B.oNe.com']); res.end(); }); - await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any); const [serverRequest] = await Promise.all([ server.waitForRequest('/empty.html'), - request.get(`http://www.b.one.com:${server.PORT}/empty.html`) + request.get(`http://www.b.one.com:${server.PORT}/empty.html`, { __testHookLookup } as any) ]); expect(serverRequest.headers.cookie).toBe('a=v; b=v'); }); @@ -117,10 +106,10 @@ it('should do case-insensitive match of request domain', async ({ request, serve res.setHeader('Set-Cookie', ['a=v; domain=one.com', 'b=v; domain=.b.one.com']); res.end(); }); - await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any); const [serverRequest] = await Promise.all([ server.waitForRequest('/empty.html'), - request.get(`http://WWW.B.ONE.COM:${server.PORT}/empty.html`) + request.get(`http://WWW.B.ONE.COM:${server.PORT}/empty.html`, { __testHookLookup } as any) ]); expect(serverRequest.headers.cookie).toBe('a=v; b=v'); }); @@ -187,7 +176,7 @@ it('should store cookie from Set-Cookie header even if it contains equal signs', res.end(); }); - await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any); const state = await request.storageState(); expect(state).toEqual({ 'cookies': [ @@ -266,7 +255,7 @@ it('should export cookies to storage state', async ({ request, server }) => { res.setHeader('Set-Cookie', ['a=b', `c=d; expires=${expires.toUTCString()}; domain=b.one.com; path=/input`, 'e=f; domain=b.one.com; path=/input/subfolder']); res.end(); }); - await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`); + await request.get(`http://a.b.one.com:${server.PORT}/setcookie.html`, { __testHookLookup } as any); const state = await request.storageState(); expect(state).toEqual({ 'cookies': [ @@ -376,7 +365,7 @@ it('should send cookies from storage state', async ({ playwright, server }) => { const request = await playwright.request.newContext({ storageState }); const [serverRequest] = await Promise.all([ server.waitForRequest('/first/second/third/not_found.html'), - request.get(`http://www.a.b.one.com:${server.PORT}/first/second/third/not_found.html`) + request.get(`http://www.a.b.one.com:${server.PORT}/first/second/third/not_found.html`, { __testHookLookup } as any) ]); expect(serverRequest.headers.cookie).toBe('c=d; e=f'); }); diff --git a/tests/library/global-fetch.spec.ts b/tests/library/global-fetch.spec.ts index 6e20f171ae..cc92df4e8a 100644 --- a/tests/library/global-fetch.spec.ts +++ b/tests/library/global-fetch.spec.ts @@ -14,7 +14,6 @@ * limitations under the License. */ -import http from 'http'; import os from 'os'; import * as util from 'util'; import { getPlaywrightVersion } from '../../packages/playwright-core/lib/common/userAgent'; @@ -22,24 +21,6 @@ import { expect, playwrightTest as it } from '../config/browserTest'; it.skip(({ mode }) => mode !== 'default'); -let prevAgent: http.Agent; -it.beforeAll(() => { - prevAgent = http.globalAgent; - http.globalAgent = new http.Agent({ - // @ts-expect-error - lookup: (hostname, options, callback) => { - if (hostname === 'localhost' || hostname.endsWith('playwright.dev')) - callback(null, '127.0.0.1', 4); - else - throw new Error(`Failed to resolve hostname: ${hostname}`); - } - }); -}); - -it.afterAll(() => { - http.globalAgent = prevAgent; -}); - for (const method of ['fetch', 'delete', 'get', 'head', 'patch', 'post', 'put'] as const) { it(`${method} should work @smoke`, async ({ playwright, server }) => { const request = await playwright.request.newContext();