diff --git a/packages/playwright-core/src/server/chromium/chromium.ts b/packages/playwright-core/src/server/chromium/chromium.ts index 00ec9634db..99422328ac 100644 --- a/packages/playwright-core/src/server/chromium/chromium.ts +++ b/packages/playwright-core/src/server/chromium/chromium.ts @@ -30,8 +30,9 @@ import { CRDevTools } from './crDevTools'; import type { BrowserOptions, BrowserProcess, PlaywrightOptions } from '../browser'; import { Browser } from '../browser'; import type * as types from '../types'; -import type { HTTPRequestParams } from '../../utils/utils'; -import { debugMode, fetchData, getUserAgent, headersArrayToObject, removeFolders, streamToString, wrapInASCIIBox } from '../../utils/utils'; +import type { HTTPRequestParams } from '../../utils'; +import { fetchData } from '../../utils'; +import { debugMode, getUserAgent, headersArrayToObject, removeFolders, streamToString, wrapInASCIIBox } from '../../utils/utils'; import { RecentLogsCollector } from '../../utils/debugLogger'; import type { Progress } from '../progress'; import { ProgressController } from '../progress'; diff --git a/packages/playwright-core/src/server/registry/browserFetcher.ts b/packages/playwright-core/src/server/registry/browserFetcher.ts index 06674aa8d1..8abce769e1 100644 --- a/packages/playwright-core/src/server/registry/browserFetcher.ts +++ b/packages/playwright-core/src/server/registry/browserFetcher.ts @@ -19,8 +19,9 @@ import extract from 'extract-zip'; import fs from 'fs'; import os from 'os'; import path from 'path'; -import { existsAsync, download, getUserAgent } from '../../utils/utils'; +import { existsAsync, getUserAgent } from '../../utils/utils'; import { debugLogger } from '../../utils/debugLogger'; +import { download } from './download'; export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise { const progressBarName = `Playwright build of ${title}`; diff --git a/packages/playwright-core/src/server/registry/download.ts b/packages/playwright-core/src/server/registry/download.ts new file mode 100644 index 0000000000..f59f072cd1 --- /dev/null +++ b/packages/playwright-core/src/server/registry/download.ts @@ -0,0 +1,146 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import fs from 'fs'; +import ProgressBar from 'progress'; +import { httpRequest } from '../../utils'; + +type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; +type DownloadFileLogger = (message: string) => void; +type DownloadFileOptions = { + progressCallback?: OnProgressCallback, + log?: DownloadFileLogger, + userAgent?: string +}; + +function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> { + const { + progressCallback, + log = () => {}, + } = options; + log(`running download:`); + log(`-- from url: ${url}`); + log(`-- to location: ${destinationPath}`); + let fulfill: ({ error }: {error: any}) => void = ({ error }) => {}; + let downloadedBytes = 0; + let totalBytes = 0; + + const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; }); + + httpRequest({ + url, + headers: options.userAgent ? { + 'User-Agent': options.userAgent, + } : undefined, + }, response => { + log(`-- response status code: ${response.statusCode}`); + if (response.statusCode !== 200) { + const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`); + // consume response data to free up memory + response.resume(); + fulfill({ error }); + return; + } + const file = fs.createWriteStream(destinationPath); + file.on('finish', () => fulfill({ error: null })); + file.on('error', error => fulfill({ error })); + response.pipe(file); + totalBytes = parseInt(response.headers['content-length'] || '0', 10); + log(`-- total bytes: ${totalBytes}`); + if (progressCallback) + response.on('data', onData); + }, (error: any) => fulfill({ error })); + return promise; + + function onData(chunk: string) { + downloadedBytes += chunk.length; + progressCallback!(downloadedBytes, totalBytes); + } +} + +type DownloadOptions = { + progressBarName?: string, + retryCount?: number + log?: DownloadFileLogger + userAgent?: string +}; + +export async function download( + url: string, + destination: string, + options: DownloadOptions = {} +) { + const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options; + for (let attempt = 1; attempt <= retryCount; ++attempt) { + log( + `downloading ${progressBarName} - attempt #${attempt}` + ); + const { error } = await downloadFile(url, destination, { + progressCallback: getDownloadProgress(progressBarName), + log, + userAgent, + }); + if (!error) { + log(`SUCCESS downloading ${progressBarName}`); + break; + } + const errorMessage = error?.message || ''; + log(`attempt #${attempt} - ERROR: ${errorMessage}`); + if ( + attempt < retryCount && + (errorMessage.includes('ECONNRESET') || + errorMessage.includes('ETIMEDOUT')) + ) { + // Maximum default delay is 3rd retry: 1337.5ms + const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt); + log(`sleeping ${millis}ms before retry...`); + await new Promise(c => setTimeout(c, millis)); + } else { + throw error; + } + } +} + +function getDownloadProgress(progressBarName: string): OnProgressCallback { + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + + return (downloadedBytes: number, totalBytes: number) => { + if (!process.stderr.isTTY) + return; + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${progressBarName} - ${toMegabytes( + totalBytes + )} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + }; +} + +function toMegabytes(bytes: number) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; +} diff --git a/packages/playwright-core/src/server/registry/index.ts b/packages/playwright-core/src/server/registry/index.ts index 9aa93643a4..a7e91890f2 100644 --- a/packages/playwright-core/src/server/registry/index.ts +++ b/packages/playwright-core/src/server/registry/index.ts @@ -21,7 +21,8 @@ import * as util from 'util'; import * as fs from 'fs'; import lockfile from 'proper-lockfile'; import { getUbuntuVersion } from '../../utils/ubuntuVersion'; -import { getFromENV, getAsBooleanFromENV, getClientLanguage, calculateSha1, removeFolders, existsAsync, hostPlatform, canAccessFile, spawnAsync, fetchData, wrapInASCIIBox, transformCommandsForRoot } from '../../utils/utils'; +import { fetchData } from '../../utils'; +import { getFromENV, getAsBooleanFromENV, getClientLanguage, calculateSha1, removeFolders, existsAsync, hostPlatform, canAccessFile, spawnAsync, wrapInASCIIBox, transformCommandsForRoot } from '../../utils/utils'; import type { DependencyGroup } from './dependencies'; import { installDependenciesLinux, installDependenciesWindows, validateDependenciesLinux, validateDependenciesWindows } from './dependencies'; import { downloadBrowserWithProgressBar, logPolitely } from './browserFetcher'; diff --git a/packages/playwright-core/src/utils/index.ts b/packages/playwright-core/src/utils/index.ts index 7b6f1b2958..635ed232dc 100644 --- a/packages/playwright-core/src/utils/index.ts +++ b/packages/playwright-core/src/utils/index.ts @@ -17,3 +17,5 @@ export { ManualPromise } from './manualPromise'; export { MultiMap } from './multimap'; export { raceAgainstTimeout, TimeoutRunner, TimeoutRunnerError } from './timeoutRunner'; +export type { HTTPRequestParams } from './netUtils'; +export { httpRequest, fetchData } from './netUtils'; diff --git a/packages/playwright-core/src/utils/netUtils.ts b/packages/playwright-core/src/utils/netUtils.ts index 3bb9fc324b..25f7180297 100644 --- a/packages/playwright-core/src/utils/netUtils.ts +++ b/packages/playwright-core/src/utils/netUtils.ts @@ -14,7 +14,11 @@ * limitations under the License. */ +import http from 'http'; +import https from 'https'; import net from 'net'; +import { getProxyForUrl } from 'proxy-from-env'; +import * as URL from 'url'; export async function createSocket(host: string, port: number): Promise { return new Promise((resolve, reject) => { @@ -23,3 +27,85 @@ export async function createSocket(host: string, port: number): Promise reject(error)); }); } + +// `https-proxy-agent` v5 is written in TypeScript and exposes generated types. +// However, as of June 2020, its types are generated with tsconfig that enables +// `esModuleInterop` option. +// +// As a result, we can't depend on the package unless we enable the option +// for our codebase. Instead of doing this, we abuse "require" to import module +// without types. +const ProxyAgent = require('https-proxy-agent'); + +export type HTTPRequestParams = { + url: string, + method?: string, + headers?: http.OutgoingHttpHeaders, + data?: string | Buffer, + timeout?: number, +}; + +export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) { + const parsedUrl = URL.parse(params.url); + let options: https.RequestOptions = { ...parsedUrl }; + options.method = params.method || 'GET'; + options.headers = params.headers; + + const proxyURL = getProxyForUrl(params.url); + if (proxyURL) { + if (params.url.startsWith('http:')) { + const proxy = URL.parse(proxyURL); + options = { + path: parsedUrl.href, + host: proxy.hostname, + port: proxy.port, + }; + } else { + const parsedProxyURL = URL.parse(proxyURL); + (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:'; + + options.agent = new ProxyAgent(parsedProxyURL); + options.rejectUnauthorized = false; + } + } + + const requestCallback = (res: http.IncomingMessage) => { + const statusCode = res.statusCode || 0; + if (statusCode >= 300 && statusCode < 400 && res.headers.location) + httpRequest({ ...params, url: res.headers.location }, onResponse, onError); + else + onResponse(res); + }; + const request = options.protocol === 'https:' ? + https.request(options, requestCallback) : + http.request(options, requestCallback); + request.on('error', onError); + if (params.timeout !== undefined) { + const rejectOnTimeout = () => { + onError(new Error(`Request to ${params.url} timed out after ${params.timeout}ms`)); + request.abort(); + }; + if (params.timeout <= 0) { + rejectOnTimeout(); + return; + } + request.setTimeout(params.timeout, rejectOnTimeout); + } + request.end(params.data); +} + +export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequestParams, response: http.IncomingMessage) => Promise): Promise { + return new Promise((resolve, reject) => { + httpRequest(params, async response => { + if (response.statusCode !== 200) { + const error = onError ? await onError(params, response) : new Error(`fetch failed: server returned code ${response.statusCode}. URL: ${params.url}`); + reject(error); + return; + } + let body = ''; + response.on('data', (chunk: string) => body += chunk); + response.on('error', (error: any) => reject(error)); + response.on('end', () => resolve(body)); + }, reject); + }); +} diff --git a/packages/playwright-core/src/utils/utils.ts b/packages/playwright-core/src/utils/utils.ts index b5c1b7e924..8a69c99fc7 100644 --- a/packages/playwright-core/src/utils/utils.ts +++ b/packages/playwright-core/src/utils/utils.ts @@ -14,233 +14,20 @@ * limitations under the License. */ -import path from 'path'; -import fs from 'fs'; -import type stream from 'stream'; -import removeFolder from 'rimraf'; -import * as crypto from 'crypto'; -import os from 'os'; -import http from 'http'; -import https from 'https'; import type { SpawnOptions } from 'child_process'; -import { spawn, execSync } from 'child_process'; -import { getProxyForUrl } from 'proxy-from-env'; +import { execSync, spawn } from 'child_process'; +import * as crypto from 'crypto'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import removeFolder from 'rimraf'; +import type stream from 'stream'; import * as URL from 'url'; -import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion'; import type { NameValue } from '../protocol/channels'; -import ProgressBar from 'progress'; - -// `https-proxy-agent` v5 is written in TypeScript and exposes generated types. -// However, as of June 2020, its types are generated with tsconfig that enables -// `esModuleInterop` option. -// -// As a result, we can't depend on the package unless we enable the option -// for our codebase. Instead of doing this, we abuse "require" to import module -// without types. -const ProxyAgent = require('https-proxy-agent'); +import { getUbuntuVersionSync, parseOSReleaseText } from './ubuntuVersion'; export const existsAsync = (path: string): Promise => new Promise(resolve => fs.stat(path, err => resolve(!err))); -export type HTTPRequestParams = { - url: string, - method?: string, - headers?: http.OutgoingHttpHeaders, - data?: string | Buffer, - timeout?: number, -}; - -function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) { - const parsedUrl = URL.parse(params.url); - let options: https.RequestOptions = { ...parsedUrl }; - options.method = params.method || 'GET'; - options.headers = params.headers; - - const proxyURL = getProxyForUrl(params.url); - if (proxyURL) { - if (params.url.startsWith('http:')) { - const proxy = URL.parse(proxyURL); - options = { - path: parsedUrl.href, - host: proxy.hostname, - port: proxy.port, - }; - } else { - const parsedProxyURL = URL.parse(proxyURL); - (parsedProxyURL as any).secureProxy = parsedProxyURL.protocol === 'https:'; - - options.agent = new ProxyAgent(parsedProxyURL); - options.rejectUnauthorized = false; - } - } - - const requestCallback = (res: http.IncomingMessage) => { - const statusCode = res.statusCode || 0; - if (statusCode >= 300 && statusCode < 400 && res.headers.location) - httpRequest({ ...params, url: res.headers.location }, onResponse, onError); - else - onResponse(res); - }; - const request = options.protocol === 'https:' ? - https.request(options, requestCallback) : - http.request(options, requestCallback); - request.on('error', onError); - if (params.timeout !== undefined) { - const rejectOnTimeout = () => { - onError(new Error(`Request to ${params.url} timed out after ${params.timeout}ms`)); - request.abort(); - }; - if (params.timeout <= 0) { - rejectOnTimeout(); - return; - } - request.setTimeout(params.timeout, rejectOnTimeout); - } - request.end(params.data); -} - -export function fetchData(params: HTTPRequestParams, onError?: (params: HTTPRequestParams, response: http.IncomingMessage) => Promise): Promise { - return new Promise((resolve, reject) => { - httpRequest(params, async response => { - if (response.statusCode !== 200) { - const error = onError ? await onError(params, response) : new Error(`fetch failed: server returned code ${response.statusCode}. URL: ${params.url}`); - reject(error); - return; - } - let body = ''; - response.on('data', (chunk: string) => body += chunk); - response.on('error', (error: any) => reject(error)); - response.on('end', () => resolve(body)); - }, reject); - }); -} - -type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; -type DownloadFileLogger = (message: string) => void; -type DownloadFileOptions = { - progressCallback?: OnProgressCallback, - log?: DownloadFileLogger, - userAgent?: string -}; - -function downloadFile(url: string, destinationPath: string, options: DownloadFileOptions = {}): Promise<{error: any}> { - const { - progressCallback, - log = () => {}, - } = options; - log(`running download:`); - log(`-- from url: ${url}`); - log(`-- to location: ${destinationPath}`); - let fulfill: ({ error }: {error: any}) => void = ({ error }) => {}; - let downloadedBytes = 0; - let totalBytes = 0; - - const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; }); - - httpRequest({ - url, - headers: options.userAgent ? { - 'User-Agent': options.userAgent, - } : undefined, - }, response => { - log(`-- response status code: ${response.statusCode}`); - if (response.statusCode !== 200) { - const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`); - // consume response data to free up memory - response.resume(); - fulfill({ error }); - return; - } - const file = fs.createWriteStream(destinationPath); - file.on('finish', () => fulfill({ error: null })); - file.on('error', error => fulfill({ error })); - response.pipe(file); - totalBytes = parseInt(response.headers['content-length'] || '0', 10); - log(`-- total bytes: ${totalBytes}`); - if (progressCallback) - response.on('data', onData); - }, (error: any) => fulfill({ error })); - return promise; - - function onData(chunk: string) { - downloadedBytes += chunk.length; - progressCallback!(downloadedBytes, totalBytes); - } -} - -type DownloadOptions = { - progressBarName?: string, - retryCount?: number - log?: DownloadFileLogger - userAgent?: string -}; - -export async function download( - url: string, - destination: string, - options: DownloadOptions = {} -) { - const { progressBarName = 'file', retryCount = 3, log = () => {}, userAgent } = options; - for (let attempt = 1; attempt <= retryCount; ++attempt) { - log( - `downloading ${progressBarName} - attempt #${attempt}` - ); - const { error } = await downloadFile(url, destination, { - progressCallback: getDownloadProgress(progressBarName), - log, - userAgent, - }); - if (!error) { - log(`SUCCESS downloading ${progressBarName}`); - break; - } - const errorMessage = error?.message || ''; - log(`attempt #${attempt} - ERROR: ${errorMessage}`); - if ( - attempt < retryCount && - (errorMessage.includes('ECONNRESET') || - errorMessage.includes('ETIMEDOUT')) - ) { - // Maximum default delay is 3rd retry: 1337.5ms - const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt); - log(`sleeping ${millis}ms before retry...`); - await new Promise(c => setTimeout(c, millis)); - } else { - throw error; - } - } -} - -function getDownloadProgress(progressBarName: string): OnProgressCallback { - let progressBar: ProgressBar; - let lastDownloadedBytes = 0; - - return (downloadedBytes: number, totalBytes: number) => { - if (!process.stderr.isTTY) - return; - if (!progressBar) { - progressBar = new ProgressBar( - `Downloading ${progressBarName} - ${toMegabytes( - totalBytes - )} [:bar] :percent :etas `, - { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - } - ); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - }; -} - -function toMegabytes(bytes: number) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; -} - export function spawnAsync(cmd: string, args: string[], options: SpawnOptions = {}): Promise<{stdout: string, stderr: string, code: number | null, error?: Error}> { const process = spawn(cmd, args, Object.assign({ windowsHide: true }, options));