feat(fetch): timeout option and default timeout (#8762)

src/client/browserContext.ts

@@ -216,7 +216,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
     });
   }
 
-  async _fetch(url: string, options: { url?: string, method?: string, headers?: Headers, postData?: string | Buffer } = {}): Promise<network.FetchResponse> {
+  async _fetch(url: string, options: { url?: string, method?: string, headers?: Headers, postData?: string | Buffer, timeout?: number } = {}): Promise<network.FetchResponse> {
     return this._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
       const postDataBuffer = isString(options.postData) ? Buffer.from(options.postData, 'utf8') : options.postData;
       const result = await channel.fetch({
@@ -224,6 +224,7 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
         method: options.method,
         headers: options.headers ? headersObjectToArray(options.headers) : undefined,
         postData: postDataBuffer ? postDataBuffer.toString('base64') : undefined,
+        timeout: options.timeout,
       });
       if (result.error)
         throw new Error(`Request failed: ${result.error}`);

src/dispatchers/browserContextDispatcher.ts

@@ -113,6 +113,7 @@ export class BrowserContextDispatcher extends Dispatcher<BrowserContext, channel
       method: params.method,
       headers: params.headers ? headersArrayToObject(params.headers, false) : undefined,
       postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined,
+      timeout: params.timeout,
     });
     let response;
     if (fetchResponse) {

src/protocol/channels.ts

@@ -858,11 +858,13 @@ export type BrowserContextFetchParams = {
   method?: string,
   headers?: NameValue[],
   postData?: Binary,
+  timeout?: number,
 };
 export type BrowserContextFetchOptions = {
   method?: string,
   headers?: NameValue[],
   postData?: Binary,
+  timeout?: number,
 };
 export type BrowserContextFetchResult = {
   response?: FetchResponse,

src/protocol/protocol.yml

@@ -621,6 +621,7 @@ BrowserContext:
           type: array?
           items: NameValue
         postData: binary?
+        timeout: number?
       returns:
         response: FetchResponse?
         error: string?

src/protocol/validator.ts

@@ -397,6 +397,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     method: tOptional(tString),
     headers: tOptional(tArray(tType('NameValue'))),
     postData: tOptional(tBinary),
+    timeout: tOptional(tNumber),
   });
   scheme.BrowserContextGrantPermissionsParams = tObject({
     permissions: tArray(tString),

src/server/fetch.ts

@@ -22,6 +22,7 @@ import * as https from 'https';
 import { BrowserContext } from './browserContext';
 import * as types from './types';
 import { pipeline, Readable, Transform } from 'stream';
+import { monotonicTime } from '../utils/utils';
 
 export async function playwrightFetch(context: BrowserContext, params: types.FetchOptions): Promise<{fetchResponse?: types.FetchResponse, error?: string}> {
   try {
@@ -50,11 +51,16 @@ export async function playwrightFetch(context: BrowserContext, params: types.Fet
       agent = new HttpsProxyAgent(proxyOpts);
     }
 
+    const timeout = context._timeoutSettings.timeout(params);
+    const deadline = monotonicTime() + timeout;
+
     const fetchResponse = await sendRequest(context, new URL(params.url, context._options.baseURL), {
       method,
       headers,
       agent,
-      maxRedirects: 20
+      maxRedirects: 20,
+      timeout,
+      deadline
     }, params.postData);
     return { fetchResponse };
   } catch (e) {
@@ -95,7 +101,7 @@ async function updateRequestCookieHeader(context: BrowserContext, url: URL, opti
   }
 }
 
-async function sendRequest(context: BrowserContext, url: URL, options: http.RequestOptions & { maxRedirects: number }, postData?: Buffer): Promise<types.FetchResponse>{
+async function sendRequest(context: BrowserContext, url: URL, options: http.RequestOptions & { maxRedirects: number, deadline: number }, postData?: Buffer): Promise<types.FetchResponse>{
   await updateRequestCookieHeader(context, url, options);
   return new Promise<types.FetchResponse>((fulfill, reject) => {
     const requestConstructor: ((url: URL, options: http.RequestOptions, callback?: (res: http.IncomingMessage) => void) => http.ClientRequest)
@@ -125,11 +131,13 @@ async function sendRequest(context: BrowserContext, url: URL, options: http.Requ
           delete headers[`content-type`];
         }
 
-        const redirectOptions: http.RequestOptions & { maxRedirects: number } = {
+        const redirectOptions: http.RequestOptions & { maxRedirects: number, deadline: number } = {
           method,
           headers,
           agent: options.agent,
           maxRedirects: options.maxRedirects - 1,
+          timeout: options.timeout,
+          deadline: options.deadline
         };
 
         // HTTP-redirect fetch step 4: If locationURL is null, then return response.
@@ -189,6 +197,16 @@ async function sendRequest(context: BrowserContext, url: URL, options: http.Requ
       body.on('error',reject);
     });
     request.on('error', reject);
+    const rejectOnTimeout = () =>  {
+      reject(new Error(`Request timed out after ${options.timeout}ms`));
+      request.abort();
+    };
+    const remaining = options.deadline - monotonicTime();
+    if (remaining <= 0) {
+      rejectOnTimeout();
+      return;
+    }
+    request.setTimeout(remaining, rejectOnTimeout);
     if (postData)
       request.write(postData);
     request.end();

src/server/types.ts

@@ -377,6 +377,7 @@ export type FetchOptions = {
   method?: string,
   headers?: { [name: string]: string },
   postData?: Buffer,
+  timeout?: number,
 };
 
 export type FetchResponse = {

tests/browsercontext-fetch.spec.ts

@@ -193,6 +193,16 @@ it('should add cookies from Set-Cookie header', async ({context, page, server})
   expect((await page.evaluate(() => document.cookie)).split(';').map(s => s.trim()).sort()).toEqual(['foo=bar', 'session=value']);
 });
 
+it('should not lose body while handling Set-Cookie header', async ({context, page, server}) => {
+  server.setRoute('/setcookie.html', (req, res) => {
+    res.setHeader('Set-Cookie', ['session=value', 'foo=bar; max-age=3600']);
+    res.end('text content');
+  });
+  // @ts-expect-error
+  const response = await context._fetch(server.PREFIX + '/setcookie.html');
+  expect(await response.text()).toBe('text content');
+});
+
 it('should handle cookies on redirects', async ({context, server, browserName, isWindows}) => {
   server.setRoute('/redirect1', (req, res) => {
     res.setHeader('Set-Cookie', 'r1=v1;SameSite=Lax');
@@ -576,3 +586,30 @@ it('should throw informatibe error on corrupted deflate body', async function({c
   expect(error.message).toContain(`failed to decompress 'deflate' encoding`);
 });
 
+it('should support timeout option', async function({context, server}) {
+  server.setRoute('/slow', (req, res) => {
+    res.writeHead(200, {
+      'content-length': 4096,
+      'content-type': 'text/html',
+    });
+  });
+
+  // @ts-expect-error
+  const error = await context._fetch(server.PREFIX + '/slow', { timeout: 10 }).catch(e => e);
+  expect(error.message).toContain(`Request timed out after 10ms`);
+});
+
+it('should respect timeout after redirects', async function({context, server}) {
+  server.setRedirect('/redirect', '/slow');
+  server.setRoute('/slow', (req, res) => {
+    res.writeHead(200, {
+      'content-length': 4096,
+      'content-type': 'text/html',
+    });
+  });
+
+  context.setDefaultTimeout(100);
+  // @ts-expect-error
+  const error = await context._fetch(server.PREFIX + '/redirect').catch(e => e);
+  expect(error.message).toContain(`Request timed out after 100ms`);
+});