chore: use utils via index export (2) (#13399)

packages/playwright-core/src/server/chromium/chromium.ts

@@ -30,8 +30,9 @@ import { CRDevTools } from './crDevTools';
 import type { BrowserOptions, BrowserProcess, PlaywrightOptions } from '../browser';
 import { Browser } from '../browser';
 import type * as types from '../types';
-import type { HTTPRequestParams } from '../../utils/utils';
+import type { HTTPRequestParams } from '../../utils';
-import { debugMode, fetchData, getUserAgent, headersArrayToObject, removeFolders, streamToString, wrapInASCIIBox } from '../../utils/utils';
+import { fetchData } from '../../utils';
+import { debugMode, getUserAgent, headersArrayToObject, removeFolders, streamToString, wrapInASCIIBox } from '../../utils/utils';
 import { RecentLogsCollector } from '../../utils/debugLogger';
 import type { Progress } from '../progress';
 import { ProgressController } from '../progress';

packages/playwright-core/src/server/registry/browserFetcher.ts

@@ -19,8 +19,9 @@ import extract from 'extract-zip';
 import fs from 'fs';
 import os from 'os';
 import path from 'path';
-import { existsAsync, download, getUserAgent } from '../../utils/utils';
+import { existsAsync, getUserAgent } from '../../utils/utils';
 import { debugLogger } from '../../utils/debugLogger';
+import { download } from './download';
 
 export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise<boolean> {
   const progressBarName = `Playwright build of ${title}`;

packages/playwright-core/src/server/registry/download.ts

@@ -0,0 +1,146 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import ProgressBar from 'progress';
+import { httpRequest } from '../../utils';
+
+type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
+type DownloadFileLogger = (message: string) => void;
+type DownloadFileOptions = {
+  progressCallback?: OnProgressCallback,
+  log?: DownloadFileLogger,
+  userAgent?: string
+};
+
+function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> {
+  const {
+    progressCallback,
+    log = () => {},
+  } = options;
+  log(`running download:`);
+  log(`-- from url: ${url}`);
+  log(`-- to location: ${destinationPath}`);
+  let fulfill: ({ error }: {error: any}) => void = ({ error }) => {};
+  let downloadedBytes = 0;
+  let totalBytes = 0;
+
+  const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; });
+
+  httpRequest({
+    url,
+    headers: options.userAgent ? {
+      'User-Agent': options.userAgent,
+    } : undefined,
+  }, response => {
+    log(`-- response status code: ${response.statusCode}`);
+    if (response.statusCode !== 200) {
+      const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
+      // consume response data to free up memory
+      response.resume();
+      fulfill({ error });
+      return;
+    }
+    const file = fs.createWriteStream(destinationPath);
+    file.on('finish', () => fulfill({ error: null }));
+    file.on('error', error => fulfill({ error }));
+    response.pipe(file);
+    totalBytes = parseInt(response.headers['content-length'] || '0', 10);
+    log(`-- total bytes: ${totalBytes}`);
+    if (progressCallback)
+      response.on('data', onData);
+  }, (error: any) => fulfill({ error }));
+  return promise;
+
+  function onData(chunk: string) {
+    downloadedBytes += chunk.length;
+    progressCallback!(downloadedBytes, totalBytes);
+  }
+}
+
+type DownloadOptions = {
+  progressBarName?: string,
+  retryCount?: number
+  log?: DownloadFileLogger
+  userAgent?: string
+};
+
+export async function download(
+  url: string,
+  destination: string,
+  options: DownloadOptions = {}
+) {
+  const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options;
+  for (let attempt = 1; attempt <= retryCount; ++attempt) {
+    log(
+        `downloading ${progressBarName} - attempt #${attempt}`
+    );
+    const { error } = await downloadFile(url, destination, {
+      progressCallback: getDownloadProgress(progressBarName),
+      log,
+      userAgent,
+    });
+    if (!error) {
+      log(`SUCCESS downloading ${progressBarName}`);
+      break;
+    }
+    const errorMessage = error?.message || '';
+    log(`attempt #${attempt} - ERROR: ${errorMessage}`);
+    if (
+      attempt < retryCount &&
+      (errorMessage.includes('ECONNRESET') ||
+        errorMessage.includes('ETIMEDOUT'))
+    ) {
+      // Maximum default delay is 3rd retry: 1337.5ms
+      const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt);
+      log(`sleeping ${millis}ms before retry...`);
+      await new Promise(c => setTimeout(c, millis));
+    } else {
+      throw error;
+    }
+  }
+}
+
+function getDownloadProgress(progressBarName: string): OnProgressCallback {
+  let progressBar: ProgressBar;
+  let lastDownloadedBytes = 0;
+
+  return (downloadedBytes: number, totalBytes: number) => {
+    if (!process.stderr.isTTY)
+      return;
+    if (!progressBar) {
+      progressBar = new ProgressBar(
+          `Downloading ${progressBarName} - ${toMegabytes(
+              totalBytes
+          )} [:bar] :percent :etas `,
+          {
+            complete: '=',
+            incomplete: ' ',
+            width: 20,
+            total: totalBytes,
+          }
+      );
+    }
+    const delta = downloadedBytes - lastDownloadedBytes;
+    lastDownloadedBytes = downloadedBytes;
+    progressBar.tick(delta);
+  };
+}
+
+function toMegabytes(bytes: number) {
+  const mb = bytes / 1024 / 1024;
+  return `${Math.round(mb * 10) / 10} Mb`;
+}

packages/playwright-core/src/server/registry/index.ts

@@ -21,7 +21,8 @@ import * as util from 'util';
 import * as fs from 'fs';
 import lockfile from 'proper-lockfile';
 import { getUbuntuVersion } from '../../utils/ubuntuVersion';
-import { getFromENV, getAsBooleanFromENV, getClientLanguage, calculateSha1, removeFolders, existsAsync, hostPlatform, canAccessFile, spawnAsync, fetchData, wrapInASCIIBox, transformCommandsForRoot } from '../../utils/utils';
+import { fetchData } from '../../utils';
+import { getFromENV, getAsBooleanFromENV, getClientLanguage, calculateSha1, removeFolders, existsAsync, hostPlatform, canAccessFile, spawnAsync, wrapInASCIIBox, transformCommandsForRoot } from '../../utils/utils';
 import type { DependencyGroup } from './dependencies';
 import { installDependenciesLinux, installDependenciesWindows, validateDependenciesLinux, validateDependenciesWindows } from './dependencies';
 import { downloadBrowserWithProgressBar, logPolitely } from './browserFetcher';

packages/playwright-core/src/utils/index.ts

@@ -17,3 +17,5 @@
 export { ManualPromise } from './manualPromise';
 export { MultiMap } from './multimap';
 export { raceAgainstTimeout, TimeoutRunner, TimeoutRunnerError } from './timeoutRunner';
+export type { HTTPRequestParams } from './netUtils';
+export { httpRequest, fetchData } from './netUtils';

packages/playwright-core/src/utils/netUtils.ts

@@ -14,7 +14,11 @@
  * limitations under the License.
  */
 
+import http from 'http';
+import https from 'https';
 import net from 'net';
+import { getProxyForUrl } from 'proxy-from-env';
+import * as URL from 'url';
 
 export async function createSocket(host: string, port: number): Promise<net.Socket> {
   return new Promise((resolve, reject) => {
@@ -23,3 +27,85 @@ export async function createSocket(host: string, port: number): Promise<net.Sock
     socket.on('error', error => reject(error));
   });
 }
+
+// `https-proxy-agent` v5 is written in TypeScript and exposes generated types.
+// However, as of June 2020, its types are generated with tsconfig that enables
+// `esModuleInterop` option.
+//
+// As a result, we can't depend on the package unless we enable the option
+// for our codebase. Instead of doing this, we abuse "require" to import module
+// without types.
+const ProxyAgent = require('https-proxy-agent');
+
+export type HTTPRequestParams = {
+  url: string,
+  method?: string,
+  headers?: http.OutgoingHttpHeaders,
+  data?: string | Buffer,
+  timeout?: number,
+};
+
+export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) {
+  const parsedUrl = URL.parse(params.url);
+  let options: https.RequestOptions = { ...parsedUrl };
+  options.method = params.method || 'GET';
+  options.headers = params.headers;
+
+  const proxyURL = getProxyForUrl(params.url);
+  if (proxyURL) {
+    if (params.url.startsWith('http:')) {
+      const proxy = URL.parse(proxyURL);
+      options = {
+        path: parsedUrl.href,
+        host: proxy.hostname,
+        port: proxy.port,
+      };
+    } else {
+      const parsedProxyURL = URL.parse(proxyURL);
+      (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:';
+
+      options.agent = new ProxyAgent(parsedProxyURL);
+      options.rejectUnauthorized = false;
+    }
+  }
+
+  const requestCallback = (res: http.IncomingMessage) => {
+    const statusCode = res.statusCode || 0;
+    if (statusCode >= 300 && statusCode < 400 && res.headers.location)
+      httpRequest({ ...params, url: res.headers.location }, onResponse, onError);
+    else
+      onResponse(res);
+  };
+  const request = options.protocol === 'https:' ?
+    https.request(options, requestCallback) :
+    http.request(options, requestCallback);
+  request.on('error', onError);
+  if (params.timeout !== undefined) {
+    const rejectOnTimeout = () =>  {
+      onError(new Error(`Request to ${params.url} timed out after ${params.timeout}ms`));
+      request.abort();
+    };
+    if (params.timeout <= 0) {
+      rejectOnTimeout();
+      return;
+    }
+    request.setTimeout(params.timeout, rejectOnTimeout);
+  }
+  request.end(params.data);
+}
+
+export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequestParams, response: http.IncomingMessage) => Promise<Error>): Promise<string> {
+  return new Promise((resolve, reject) => {
+    httpRequest(params, async response => {
+      if (response.statusCode !== 200) {
+        const error = onError ? await onError(params, response) : new Error(`fetch failed: server returned code ${response.statusCode}. URL: ${params.url}`);
+        reject(error);
+        return;
+      }
+      let body = '';
+      response.on('data', (chunk: string) => body += chunk);
+      response.on('error', (error: any) => reject(error));
+      response.on('end', () => resolve(body));
+    }, reject);
+  });
+}

packages/playwright-core/src/utils/utils.ts

@@ -14,233 +14,20 @@
  * limitations under the License.
  */
 
-import path from 'path';
-import fs from 'fs';
-import type stream from 'stream';
-import removeFolder from 'rimraf';
-import * as crypto from 'crypto';
-import os from 'os';
-import http from 'http';
-import https from 'https';
 import type { SpawnOptions } from 'child_process';
-import { spawn, execSync } from 'child_process';
+import { execSync, spawn } from 'child_process';
-import { getProxyForUrl } from 'proxy-from-env';
+import * as crypto from 'crypto';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import removeFolder from 'rimraf';
+import type stream from 'stream';
 import * as URL from 'url';
-import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion';
 import type { NameValue } from '../protocol/channels';
-import ProgressBar from 'progress';
+import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion';
-
-// `https-proxy-agent` v5 is written in TypeScript and exposes generated types.
-// However, as of June 2020, its types are generated with tsconfig that enables
-// `esModuleInterop` option.
-//
-// As a result, we can't depend on the package unless we enable the option
-// for our codebase. Instead of doing this, we abuse "require" to import module
-// without types.
-const ProxyAgent = require('https-proxy-agent');
 
 export const existsAsync = (path: string): Promise<boolean> => new Promise(resolve => fs.stat(path, err => resolve(!err)));
 
-export type HTTPRequestParams = {
-  url: string,
-  method?: string,
-  headers?: http.OutgoingHttpHeaders,
-  data?: string | Buffer,
-  timeout?: number,
-};
-
-function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) {
-  const parsedUrl = URL.parse(params.url);
-  let options: https.RequestOptions = { ...parsedUrl };
-  options.method = params.method || 'GET';
-  options.headers = params.headers;
-
-  const proxyURL = getProxyForUrl(params.url);
-  if (proxyURL) {
-    if (params.url.startsWith('http:')) {
-      const proxy = URL.parse(proxyURL);
-      options = {
-        path: parsedUrl.href,
-        host: proxy.hostname,
-        port: proxy.port,
-      };
-    } else {
-      const parsedProxyURL = URL.parse(proxyURL);
-      (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:';
-
-      options.agent = new ProxyAgent(parsedProxyURL);
-      options.rejectUnauthorized = false;
-    }
-  }
-
-  const requestCallback = (res: http.IncomingMessage) => {
-    const statusCode = res.statusCode || 0;
-    if (statusCode >= 300 && statusCode < 400 && res.headers.location)
-      httpRequest({ ...params, url: res.headers.location }, onResponse, onError);
-    else
-      onResponse(res);
-  };
-  const request = options.protocol === 'https:' ?
-    https.request(options, requestCallback) :
-    http.request(options, requestCallback);
-  request.on('error', onError);
-  if (params.timeout !== undefined) {
-    const rejectOnTimeout = () =>  {
-      onError(new Error(`Request to ${params.url} timed out after ${params.timeout}ms`));
-      request.abort();
-    };
-    if (params.timeout <= 0) {
-      rejectOnTimeout();
-      return;
-    }
-    request.setTimeout(params.timeout, rejectOnTimeout);
-  }
-  request.end(params.data);
-}
-
-export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequestParams, response: http.IncomingMessage) => Promise<Error>): Promise<string> {
-  return new Promise((resolve, reject) => {
-    httpRequest(params, async response => {
-      if (response.statusCode !== 200) {
-        const error = onError ? await onError(params, response) : new Error(`fetch failed: server returned code ${response.statusCode}. URL: ${params.url}`);
-        reject(error);
-        return;
-      }
-      let body = '';
-      response.on('data', (chunk: string) => body += chunk);
-      response.on('error', (error: any) => reject(error));
-      response.on('end', () => resolve(body));
-    }, reject);
-  });
-}
-
-type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void;
-type DownloadFileLogger = (message: string) => void;
-type DownloadFileOptions = {
-  progressCallback?: OnProgressCallback,
-  log?: DownloadFileLogger,
-  userAgent?: string
-};
-
-function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> {
-  const {
-    progressCallback,
-    log = () => {},
-  } = options;
-  log(`running download:`);
-  log(`-- from url: ${url}`);
-  log(`-- to location: ${destinationPath}`);
-  let fulfill: ({ error }: {error: any}) => void = ({ error }) => {};
-  let downloadedBytes = 0;
-  let totalBytes = 0;
-
-  const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; });
-
-  httpRequest({
-    url,
-    headers: options.userAgent ? {
-      'User-Agent': options.userAgent,
-    } : undefined,
-  }, response => {
-    log(`-- response status code: ${response.statusCode}`);
-    if (response.statusCode !== 200) {
-      const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`);
-      // consume response data to free up memory
-      response.resume();
-      fulfill({ error });
-      return;
-    }
-    const file = fs.createWriteStream(destinationPath);
-    file.on('finish', () => fulfill({ error: null }));
-    file.on('error', error => fulfill({ error }));
-    response.pipe(file);
-    totalBytes = parseInt(response.headers['content-length'] || '0', 10);
-    log(`-- total bytes: ${totalBytes}`);
-    if (progressCallback)
-      response.on('data', onData);
-  }, (error: any) => fulfill({ error }));
-  return promise;
-
-  function onData(chunk: string) {
-    downloadedBytes += chunk.length;
-    progressCallback!(downloadedBytes, totalBytes);
-  }
-}
-
-type DownloadOptions = {
-  progressBarName?: string,
-  retryCount?: number
-  log?: DownloadFileLogger
-  userAgent?: string
-};
-
-export async function download(
-  url: string,
-  destination: string,
-  options: DownloadOptions = {}
-) {
-  const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options;
-  for (let attempt = 1; attempt <= retryCount; ++attempt) {
-    log(
-        `downloading ${progressBarName} - attempt #${attempt}`
-    );
-    const { error } = await downloadFile(url, destination, {
-      progressCallback: getDownloadProgress(progressBarName),
-      log,
-      userAgent,
-    });
-    if (!error) {
-      log(`SUCCESS downloading ${progressBarName}`);
-      break;
-    }
-    const errorMessage = error?.message || '';
-    log(`attempt #${attempt} - ERROR: ${errorMessage}`);
-    if (
-      attempt < retryCount &&
-      (errorMessage.includes('ECONNRESET') ||
-        errorMessage.includes('ETIMEDOUT'))
-    ) {
-      // Maximum default delay is 3rd retry: 1337.5ms
-      const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt);
-      log(`sleeping ${millis}ms before retry...`);
-      await new Promise(c => setTimeout(c, millis));
-    } else {
-      throw error;
-    }
-  }
-}
-
-function getDownloadProgress(progressBarName: string): OnProgressCallback {
-  let progressBar: ProgressBar;
-  let lastDownloadedBytes = 0;
-
-  return (downloadedBytes: number, totalBytes: number) => {
-    if (!process.stderr.isTTY)
-      return;
-    if (!progressBar) {
-      progressBar = new ProgressBar(
-          `Downloading ${progressBarName} - ${toMegabytes(
-              totalBytes
-          )} [:bar] :percent :etas `,
-          {
-            complete: '=',
-            incomplete: ' ',
-            width: 20,
-            total: totalBytes,
-          }
-      );
-    }
-    const delta = downloadedBytes - lastDownloadedBytes;
-    lastDownloadedBytes = downloadedBytes;
-    progressBar.tick(delta);
-  };
-}
-
-function toMegabytes(bytes: number) {
-  const mb = bytes / 1024 / 1024;
-  return `${Math.round(mb * 10) / 10} Mb`;
-}
-
 export function spawnAsync(cmd: string, args: string[], options: SpawnOptions = {}): Promise<{stdout: string, stderr: string, code: number | null, error?: Error}> {
   const process = spawn(cmd, args, Object.assign({ windowsHide: true }, options));