feat(fetch): support form data and json encodings (#8975)

docs/src/api/class-fetchrequest.md

@@ -39,9 +39,12 @@ If set changes the fetch method (e.g. PUT or POST). If not specified, GET method
 Allows to set HTTP headers.
 
 ### option: FetchRequest.fetch.data
-- `data` <[string]|[Buffer]>
+- `data` <[string]|[Buffer]|[Serializable]>
 
-Allows to set post data of the fetch.
+Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way:
+* If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form using `application/x-www-form-urlencoded` encoding.
+* If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using `multipart/form-data` encoding.
+* Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`.
 
 ### option: FetchRequest.fetch.timeout
 - `timeout` <[float]>
@@ -108,9 +111,12 @@ Query parameters to be send with the URL.
 Allows to set HTTP headers.
 
 ### option: FetchRequest.post.data
-- `data` <[string]|[Buffer]>
+- `data` <[string]|[Buffer]|[Serializable]>
 
-Allows to set post data of the fetch.
+Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way:
+* If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form using `application/x-www-form-urlencoded` encoding.
+* If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using `multipart/form-data` encoding.
+* Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`.
 
 ### option: FetchRequest.post.timeout
 - `timeout` <[float]>

src/client/fetch.ts

@@ -14,21 +14,25 @@
  * limitations under the License.
  */
 
+import { ReadStream } from 'fs';
+import path from 'path';
+import * as mime from 'mime';
+import { Serializable } from '../../types/structs';
 import * as api from '../../types/types';
 import { HeadersArray } from '../common/types';
 import * as channels from '../protocol/channels';
 import { kBrowserOrContextClosedError } from '../utils/errors';
-import { assert, headersObjectToArray, isString, objectToArray } from '../utils/utils';
+import { assert, headersObjectToArray, isFilePayload, isString, objectToArray } from '../utils/utils';
 import { ChannelOwner } from './channelOwner';
 import * as network from './network';
 import { RawHeaders } from './network';
-import { Headers } from './types';
+import { FilePayload, Headers } from './types';
 
 export type FetchOptions = {
   params?: { [key: string]: string; },
   method?: string,
   headers?: Headers,
-  data?: string | Buffer,
+  data?: string | Buffer | Serializable,
   timeout?: number,
   failOnStatusCode?: boolean,
 };
@@ -67,7 +71,7 @@ export class FetchRequest extends ChannelOwner<channels.FetchRequestChannel, cha
     options?: {
       params?: { [key: string]: string; };
       headers?: { [key: string]: string; };
-      data?: string | Buffer;
+      data?: string | Buffer | Serializable;
       timeout?: number;
       failOnStatusCode?: boolean;
     }): Promise<FetchResponse> {
@@ -87,9 +91,34 @@ export class FetchRequest extends ChannelOwner<channels.FetchRequestChannel, cha
       // Cannot call allHeaders() here as the request may be paused inside route handler.
       const headersObj = options.headers || request?.headers() ;
       const headers = headersObj ? headersObjectToArray(headersObj) : undefined;
-      let postDataBuffer = isString(options.data) ? Buffer.from(options.data, 'utf8') : options.data;
+      let formData: any;
-      if (postDataBuffer === undefined)
+      let postDataBuffer: Buffer | undefined;
-        postDataBuffer = request?.postDataBuffer() || undefined;
+      if (options.data) {
+        if (isString(options.data)) {
+          postDataBuffer = Buffer.from(options.data, 'utf8');
+        } else if (Buffer.isBuffer(options.data)) {
+          postDataBuffer = options.data;
+        } else if (typeof options.data === 'object') {
+          formData = {};
+          // Convert file-like values to ServerFilePayload structs.
+          for (const [name, value] of Object.entries(options.data)) {
+            if (isFilePayload(value)) {
+              const payload = value as FilePayload;
+              if (!Buffer.isBuffer(payload.buffer))
+                throw new Error(`Unexpected buffer type of 'data.${name}'`);
+              formData[name] = filePayloadToJson(payload);
+            } else if (value instanceof ReadStream) {
+              formData[name] = await readStreamToJson(value as ReadStream);
+            } else {
+              formData[name] = value;
+            }
+          }
+        } else {
+          throw new Error(`Unexpected 'data' type`);
+        }
+        if (postDataBuffer === undefined && formData === undefined)
+          postDataBuffer = request?.postDataBuffer() || undefined;
+      }
       const postData = (postDataBuffer ? postDataBuffer.toString('base64') : undefined);
       const result = await channel.fetch({
         url,
@@ -97,11 +126,12 @@ export class FetchRequest extends ChannelOwner<channels.FetchRequestChannel, cha
         method,
         headers,
         postData,
+        formData,
         timeout: options.timeout,
         failOnStatusCode: options.failOnStatusCode,
       });
       if (result.error)
-        throw new Error(`Request failed: ${result.error}`);
+        throw new Error(result.error);
       return new FetchResponse(this, result.response!);
     });
   }
@@ -177,3 +207,32 @@ export class FetchResponse implements api.FetchResponse {
     return this._initializer.fetchUid;
   }
 }
+
+type ServerFilePayload = {
+  name: string,
+  mimeType: string,
+  buffer: string,
+};
+
+function filePayloadToJson(payload: FilePayload): ServerFilePayload {
+  return {
+    name: payload.name,
+    mimeType: payload.mimeType,
+    buffer: payload.buffer.toString('base64'),
+  };
+}
+
+async function readStreamToJson(stream: ReadStream): Promise<ServerFilePayload> {
+  const buffer = await new Promise<Buffer>((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    stream.on('data', chunk => chunks.push(chunk));
+    stream.on('end', () => resolve(Buffer.concat(chunks)));
+    stream.on('error', err => reject(err));
+  });
+  const streamPath: string = Buffer.isBuffer(stream.path) ? stream.path.toString('utf8') : stream.path;
+  return {
+    name: path.basename(streamPath),
+    mimeType: mime.getType(streamPath) || 'application/octet-stream',
+    buffer: buffer.toString('base64'),
+  };
+}

src/dispatchers/networkDispatchers.ts

@@ -188,6 +188,7 @@ export class FetchRequestDispatcher extends Dispatcher<FetchRequest, channels.Fe
       method: params.method,
       headers: params.headers ? headersArrayToObject(params.headers, false) : undefined,
       postData: params.postData ? Buffer.from(params.postData, 'base64') : undefined,
+      formData: params.formData,
       timeout: params.timeout,
       failOnStatusCode: params.failOnStatusCode,
     });
@@ -213,4 +214,3 @@ export class FetchRequestDispatcher extends Dispatcher<FetchRequest, channels.Fe
     this._object.fetchResponses.delete(params.fetchUid);
   }
 }
-

src/protocol/channels.ts

@@ -164,6 +164,7 @@ export type FetchRequestFetchParams = {
   method?: string,
   headers?: NameValue[],
   postData?: Binary,
+  formData?: any,
   timeout?: number,
   failOnStatusCode?: boolean,
 };
@@ -172,6 +173,7 @@ export type FetchRequestFetchOptions = {
   method?: string,
   headers?: NameValue[],
   postData?: Binary,
+  formData?: any,
   timeout?: number,
   failOnStatusCode?: boolean,
 };

src/protocol/protocol.yml

@@ -233,6 +233,7 @@ FetchRequest:
           type: array?
           items: NameValue
         postData: binary?
+        formData: json?
         timeout: number?
         failOnStatusCode: boolean?
       returns:

src/protocol/validator.ts

@@ -153,6 +153,7 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     method: tOptional(tString),
     headers: tOptional(tArray(tType('NameValue'))),
     postData: tOptional(tBinary),
+    formData: tOptional(tAny),
     timeout: tOptional(tNumber),
     failOnStatusCode: tOptional(tBoolean),
   });

src/server/fetch.ts

@@ -22,12 +22,13 @@ import * as https from 'https';
 import { BrowserContext } from './browserContext';
 import * as types from './types';
 import { pipeline, Readable, Transform } from 'stream';
-import { createGuid, monotonicTime } from '../utils/utils';
+import { createGuid, isFilePayload, monotonicTime } from '../utils/utils';
 import { SdkObject } from './instrumentation';
 import { Playwright } from './playwright';
 import { HeadersArray, ProxySettings } from './types';
 import { HTTPCredentials } from '../../types/types';
 import { TimeoutSettings } from '../utils/timeoutSettings';
+import { MultipartFormData } from './formData';
 
 
 type FetchRequestOptions = {
@@ -130,7 +131,13 @@ export abstract class FetchRequest extends SdkObject {
           requestUrl.searchParams.set(name, value);
       }
 
-      const fetchResponse = await this._sendRequest(requestUrl, options, params.postData);
+      let postData;
+      if (['POST', 'PUSH', 'PATCH'].includes(method))
+        postData = params.formData ? serilizeFormData(params.formData, headers) : params.postData;
+      else if (params.postData || params.formData)
+        throw new Error(`Method ${method} does not accept post data`);
+
+      const fetchResponse = await this._sendRequest(requestUrl, options, postData);
       const fetchUid = this._storeResponseBody(fetchResponse.body);
       if (params.failOnStatusCode && (fetchResponse.status < 200 || fetchResponse.status >= 400))
         return { error: `${fetchResponse.status} ${fetchResponse.statusText}` };
@@ -410,3 +417,31 @@ function parseCookie(header: string) {
   }
   return cookie;
 }
+
+function serilizeFormData(data: any, headers: { [name: string]: string }): Buffer {
+  const contentType = headers['content-type'] || 'application/json';
+  if (contentType === 'application/json') {
+    const json = JSON.stringify(data);
+    headers['content-type'] ??= contentType;
+    return Buffer.from(json, 'utf8');
+  } else if (contentType === 'application/x-www-form-urlencoded') {
+    const searchParams = new URLSearchParams();
+    for (const [name, value] of Object.entries(data))
+      searchParams.append(name, String(value));
+    return Buffer.from(searchParams.toString(), 'utf8');
+  } else if (contentType === 'multipart/form-data') {
+    const formData = new MultipartFormData();
+    for (const [name, value] of Object.entries(data)) {
+      if (isFilePayload(value)) {
+        const payload = value as types.FilePayload;
+        formData.addFileField(name, payload);
+      } else if (value !== undefined) {
+        formData.addField(name, String(value));
+      }
+    }
+    headers['content-type'] = formData.contentTypeHeader();
+    return formData.finish();
+  } else {
+    throw new Error(`Cannot serialize data using content type: ${contentType}`);
+  }
+}

src/server/formData.ts

@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as types from './types';
+
+export class MultipartFormData {
+  private readonly _boundary: string;
+  private readonly _chunks: Buffer[] = [];
+
+  constructor() {
+    this._boundary = generateUniqueBoundaryString();
+  }
+
+  contentTypeHeader() {
+    return `multipart/form-data; boundary=${this._boundary}`;
+  }
+
+  addField(name: string, value: string) {
+    this._beginMultiPartHeader(name);
+    this._finishMultiPartHeader();
+    this._chunks.push(Buffer.from(value));
+    this._finishMultiPartField();
+  }
+
+  addFileField(name: string, value: types.FilePayload) {
+    this._beginMultiPartHeader(name);
+    this._chunks.push(Buffer.from(`; filename="${value.name}"`));
+    this._chunks.push(Buffer.from(`\r\ncontent-type: ${value.mimeType || 'application/octet-stream'}`));
+    this._finishMultiPartHeader();
+    this._chunks.push(Buffer.from(value.buffer, 'base64'));
+    this._finishMultiPartField();
+  }
+
+  finish(): Buffer {
+    this._addBoundary(true);
+    return Buffer.concat(this._chunks);
+  }
+
+  private _beginMultiPartHeader(name: string) {
+    this._addBoundary();
+    this._chunks.push(Buffer.from(`content-disposition: form-data; name="${name}"`));
+  }
+
+  private _finishMultiPartHeader() {
+    this._chunks.push(Buffer.from(`\r\n\r\n`));
+  }
+
+  private _finishMultiPartField() {
+    this._chunks.push(Buffer.from(`\r\n`));
+  }
+
+  private _addBoundary(isLastBoundary?: boolean) {
+    this._chunks.push(Buffer.from('--' + this._boundary));
+    if (isLastBoundary)
+      this._chunks.push(Buffer.from('--'));
+    this._chunks.push(Buffer.from('\r\n'));
+  }
+}
+
+const alphaNumericEncodingMap = [
+  0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48,
+  0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50,
+  0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58,
+  0x59, 0x5A, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66,
+  0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E,
+  0x6F, 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76,
+  0x77, 0x78, 0x79, 0x7A, 0x30, 0x31, 0x32, 0x33,
+  0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x41, 0x42
+];
+
+// See generateUniqueBoundaryString() in WebKit
+function generateUniqueBoundaryString(): string {
+  const charCodes = [];
+  for (let i = 0; i < 16; i++)
+    charCodes.push(alphaNumericEncodingMap[Math.floor(Math.random() * alphaNumericEncodingMap.length)]);
+  return '----WebKitFormBoundary' + String.fromCharCode(...charCodes);
+}

src/server/types.ts

@@ -372,12 +372,25 @@ export type SetStorageState = {
   origins?: OriginStorage[]
 };
 
+export type FileInfo = {
+  name: string,
+  mimeType?: string,
+  buffer: Buffer,
+};
+
+export type FormField = {
+  name: string,
+  value?: string,
+  file?: FileInfo,
+};
+
 export type FetchOptions = {
   url: string,
   params?: { [name: string]: string },
   method?: string,
   headers?: { [name: string]: string },
   postData?: Buffer,
+  formData?: FormField[],
   timeout?: number,
   failOnStatusCode?: boolean,
 };

src/utils/utils.ts

@@ -411,3 +411,7 @@ export function wrapInASCIIBox(text: string, padding = 0): string {
     '╚' + '═'.repeat(maxLength + padding * 2) + '╝',
   ].join('\n');
 }
+
+export function isFilePayload(value: any): boolean {
+  return typeof value === 'object' && value['name'] && value['mimeType'] && value['buffer'];
+}

tests/browsercontext-fetch.spec.ts

@@ -14,8 +14,10 @@
  * limitations under the License.
  */
 
+import formidable from 'formidable';
 import http from 'http';
 import zlib from 'zlib';
+import fs from 'fs';
 import { pipeline } from 'stream';
 import { contextTest as it, expect } from './config/browserTest';
 import { suppressCertificateWarning } from './config/utils';
@@ -166,7 +168,7 @@ for (const method of ['get', 'post', 'fetch']) {
     const error = await context._request[method](server.PREFIX + '/does-not-exist.html', {
       failOnStatusCode: true
     }).catch(e => e);
-    expect(error.message).toContain('Request failed: 404 Not Found');
+    expect(error.message).toContain('404 Not Found');
   });
 }
 
@@ -705,3 +707,156 @@ it('should override request parameters', async function({context, page, server})
   expect(req.headers.foo).toBe('bar');
   expect((await req.postBody).toString('utf8')).toBe('data');
 });
+
+it('should support application/x-www-form-urlencoded', async function({context, page, server}) {
+  const [req] = await Promise.all([
+    server.waitForRequest('/empty.html'),
+    context._request.post(server.EMPTY_PAGE, {
+      headers: {
+        'content-type': 'application/x-www-form-urlencoded'
+      },
+      data: {
+        firstName: 'John',
+        lastName: 'Doe',
+        file: 'f.js',
+      }
+    })
+  ]);
+  expect(req.method).toBe('POST');
+  expect(req.headers['content-type']).toBe('application/x-www-form-urlencoded');
+  const body = (await req.postBody).toString('utf8');
+  const params = new URLSearchParams(body);
+  expect(params.get('firstName')).toBe('John');
+  expect(params.get('lastName')).toBe('Doe');
+  expect(params.get('file')).toBe('f.js');
+});
+
+it('should encode to application/json by default', async function({context, page, server}) {
+  const data = {
+    firstName: 'John',
+    lastName: 'Doe',
+    file: {
+      name: 'f.js'
+    },
+  };
+  const [req] = await Promise.all([
+    server.waitForRequest('/empty.html'),
+    context._request.post(server.EMPTY_PAGE, { data })
+  ]);
+  expect(req.method).toBe('POST');
+  expect(req.headers['content-type']).toBe('application/json');
+  const body = (await req.postBody).toString('utf8');
+  const json = JSON.parse(body);
+  expect(json).toEqual(data);
+});
+
+it('should support multipart/form-data', async function({context, page, server}) {
+  const formReceived = new Promise<any>(resolve => {
+    server.setRoute('/empty.html', async (serverRequest, res) => {
+      const form = new formidable.IncomingForm();
+      form.parse(serverRequest, (error, fields, files) => {
+        server.serveFile(serverRequest, res);
+        resolve({error, fields, files, serverRequest });
+      });
+    });
+  });
+
+  const file = {
+    name: 'f.js',
+    mimeType: 'text/javascript',
+    buffer: Buffer.from('var x = 10;\r\n;console.log(x);')
+  };
+  const [{error, fields, files, serverRequest}, response] = await Promise.all([
+    formReceived,
+    context._request.post(server.EMPTY_PAGE, {
+      headers: {
+        'content-type': 'multipart/form-data'
+      },
+      data: {
+        firstName: 'John',
+        lastName: 'Doe',
+        file
+      }
+    })
+  ]);
+  expect(error).toBeFalsy();
+  expect(serverRequest.method).toBe('POST');
+  expect(serverRequest.headers['content-type']).toContain('multipart/form-data');
+  expect(fields['firstName']).toBe('John');
+  expect(fields['lastName']).toBe('Doe');
+  expect(files['file'].name).toBe(file.name);
+  expect(files['file'].type).toBe(file.mimeType);
+  expect(fs.readFileSync(files['file'].path).toString()).toBe(file.buffer.toString('utf8'));
+  expect(response.status()).toBe(200);
+});
+
+it('should support multipart/form-data with ReadSream values', async function({context, page, asset, server}) {
+  const formReceived = new Promise<any>(resolve => {
+    server.setRoute('/empty.html', async (serverRequest, res) => {
+      const form = new formidable.IncomingForm();
+      form.parse(serverRequest, (error, fields, files) => {
+        server.serveFile(serverRequest, res);
+        resolve({error, fields, files, serverRequest });
+      });
+    });
+  });
+  const readStream = fs.createReadStream(asset('simplezip.json'));
+  const [{error, fields, files, serverRequest}, response] = await Promise.all([
+    formReceived,
+    context._request.post(server.EMPTY_PAGE, {
+      headers: {
+        'content-type': 'multipart/form-data'
+      },
+      data: {
+        firstName: 'John',
+        lastName: 'Doe',
+        readStream
+      }
+    })
+  ]);
+  expect(error).toBeFalsy();
+  expect(serverRequest.method).toBe('POST');
+  expect(serverRequest.headers['content-type']).toContain('multipart/form-data');
+  expect(fields['firstName']).toBe('John');
+  expect(fields['lastName']).toBe('Doe');
+  expect(files['readStream'].name).toBe('simplezip.json');
+  expect(files['readStream'].type).toBe('application/json');
+  expect(fs.readFileSync(files['readStream'].path).toString()).toBe(fs.readFileSync(asset('simplezip.json')).toString());
+  expect(response.status()).toBe(200);
+});
+
+it('should throw nice error on unsupported encoding', async function({context, server}) {
+  const error = await context._request.post(server.EMPTY_PAGE, {
+    headers: {
+      'content-type': 'unknown'
+    },
+    data: {
+      firstName: 'John',
+      lastName: 'Doe',
+    }
+  }).catch(e => e);
+  expect(error.message).toContain('Cannot serialize data using content type: unknown');
+});
+
+it('should throw nice error on unsupported data type', async function({context, server}) {
+  const error = await context._request.post(server.EMPTY_PAGE, {
+    headers: {
+      'content-type': 'application/json'
+    },
+    data: () => true
+  }).catch(e => e);
+  expect(error.message).toContain(`Unexpected 'data' type`);
+});
+
+it('should throw when data passed for unsupported request', async function({context, server}) {
+  const error = await context._request.fetch(server.EMPTY_PAGE, {
+    method: 'GET',
+    headers: {
+      'content-type': 'application/json'
+    },
+    data: {
+      foo: 'bar'
+    }
+  }).catch(e => e);
+  expect(error.message).toContain(`Method GET does not accept post data`);
+});

types/types.d.ts

@@ -12646,9 +12646,14 @@ export interface FetchRequest {
    */
   fetch(urlOrRequest: string|Request, options?: {
     /**
-     * Allows to set post data of the fetch.
+     * Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way:
+     * - If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form
+     *   using `application/x-www-form-urlencoded` encoding.
+     * - If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using
+     *   `multipart/form-data` encoding.
+     * - Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`.
      */
-    data?: string|Buffer;
+    data?: string|Buffer|Serializable;
 
     /**
      * Whether to throw on response codes other than 2xx and 3xx. By default response object is returned for all status codes.
@@ -12712,9 +12717,14 @@ export interface FetchRequest {
    */
   post(urlOrRequest: string|Request, options?: {
     /**
-     * Allows to set post data of the fetch.
+     * Allows to set post data of the fetch. If the data parameter is an object, it will be serialized the following way:
+     * - If `content-type` header is set to `application/x-www-form-urlencoded` the object will be serialized as html form
+     *   using `application/x-www-form-urlencoded` encoding.
+     * - If `content-type` header is set to `multipart/form-data` the object will be serialized as html form using
+     *   `multipart/form-data` encoding.
+     * - Otherwise the object will be serialized to json string and `content-type` header will be set to `application/json`.
      */
-    data?: string|Buffer;
+    data?: string|Buffer|Serializable;
 
     /**
      * Whether to throw on response codes other than 2xx and 3xx. By default response object is returned for all status codes.