chore(test runner): extract TestInfoImpl (#11725)

This commit is contained in:
Dmitry Gozman 2022-01-28 17:39:42 -08:00 committed by GitHub
parent 401cd9c0ee
commit eeebcd53ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 335 additions and 259 deletions

View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
import type { TestInfoImpl } from './types'; import type { TestInfoImpl } from './testInfo';
import { Suite } from './test'; import { Suite } from './test';
let currentTestInfoValue: TestInfoImpl | null = null; let currentTestInfoValue: TestInfoImpl | null = null;

View File

@ -22,10 +22,11 @@ import path from 'path';
import jpeg from 'jpeg-js'; import jpeg from 'jpeg-js';
import pixelmatch from 'pixelmatch'; import pixelmatch from 'pixelmatch';
import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch'; import { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } from '../third_party/diff_match_patch';
import { TestInfoImpl, UpdateSnapshots } from '../types'; import { UpdateSnapshots } from '../types';
import { addSuffixToFilePath } from '../util'; import { addSuffixToFilePath } from '../util';
import BlinkDiff from '../third_party/blink-diff'; import BlinkDiff from '../third_party/blink-diff';
import PNGImage from '../third_party/png-js'; import PNGImage from '../third_party/png-js';
import { TestInfoImpl } from '../testInfo';
// Note: we require the pngjs version of pixelmatch to avoid version mismatches. // Note: we require the pngjs version of pixelmatch to avoid version mismatches.
const { PNG } = require(require.resolve('pngjs', { paths: [require.resolve('pixelmatch')] })) as typeof import('pngjs'); const { PNG } = require(require.resolve('pngjs', { paths: [require.resolve('pixelmatch')] })) as typeof import('pngjs');
@ -128,12 +129,7 @@ export function compare(
return { pass: true, message }; return { pass: true, message };
} }
if (updateSnapshots === 'missing') { if (updateSnapshots === 'missing') {
if (testInfo.status === 'passed') testInfo._appendErrorMessage(message);
testInfo.status = 'failed';
if (!('error' in testInfo))
testInfo.error = { value: 'Error: ' + message };
else if (testInfo.error?.value)
testInfo.error.value += '\nError: ' + message;
return { pass: true, message }; return { pass: true, message };
} }
return { pass: false, message }; return { pass: false, message };

View File

@ -0,0 +1,268 @@
/**
* Copyright Microsoft Corporation. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import fs from 'fs';
import * as mime from 'mime';
import path from 'path';
import { TimeoutRunner, TimeoutRunnerError } from 'playwright-core/lib/utils/async';
import { calculateSha1 } from 'playwright-core/lib/utils/utils';
import type { FullConfig, FullProject, TestError, TestInfo, TestStatus } from '../types/test';
import { WorkerInitParams } from './ipc';
import { Loader } from './loader';
import { ProjectImpl } from './project';
import { TestCase } from './test';
import { Annotations, TestStepInternal } from './types';
import { addSuffixToFilePath, getContainedPath, monotonicTime, sanitizeForFilePath, serializeError, trimLongString } from './util';
export class TestInfoImpl implements TestInfo {
private _projectImpl: ProjectImpl;
private _addStepImpl: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal;
readonly _test: TestCase;
readonly _timeoutRunner: TimeoutRunner;
readonly _startTime: number;
readonly _startWallTime: number;
// ------------ TestInfo fields ------------
readonly repeatEachIndex: number;
readonly retry: number;
readonly workerIndex: number;
readonly parallelIndex: number;
readonly project: FullProject;
config: FullConfig;
readonly title: string;
readonly titlePath: string[];
readonly file: string;
readonly line: number;
readonly column: number;
readonly fn: Function;
expectedStatus: TestStatus;
duration: number = 0;
readonly annotations: Annotations = [];
readonly attachments: TestInfo['attachments'] = [];
status: TestStatus = 'passed';
readonly stdout: TestInfo['stdout'] = [];
readonly stderr: TestInfo['stderr'] = [];
timeout: number;
snapshotSuffix: string = '';
readonly outputDir: string;
readonly snapshotDir: string;
error: TestError | undefined = undefined;
constructor(
loader: Loader,
workerParams: WorkerInitParams,
test: TestCase,
retry: number,
addStepImpl: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal,
) {
this._projectImpl = loader.projects()[workerParams.projectIndex];
this._test = test;
this._addStepImpl = addStepImpl;
this._startTime = monotonicTime();
this._startWallTime = Date.now();
this.repeatEachIndex = workerParams.repeatEachIndex;
this.retry = retry;
this.workerIndex = workerParams.workerIndex;
this.parallelIndex = workerParams.parallelIndex;
this.project = this._projectImpl.config;
this.config = loader.fullConfig();
this.title = test.title;
this.titlePath = test.titlePath();
this.file = test.location.file;
this.line = test.location.line;
this.column = test.location.column;
this.fn = test.fn;
this.expectedStatus = test.expectedStatus;
this.timeout = this.project.timeout;
this._timeoutRunner = new TimeoutRunner(this.timeout);
this.outputDir = (() => {
const sameName = loader.projects().filter(project => project.config.name === this.project.name);
let uniqueProjectNamePathSegment: string;
if (sameName.length > 1)
uniqueProjectNamePathSegment = this.project.name + (sameName.indexOf(this._projectImpl) + 1);
else
uniqueProjectNamePathSegment = this.project.name;
const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this.workerIndex);
let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(trimLongString(fullTitleWithoutSpec));
if (uniqueProjectNamePathSegment)
testOutputDir += '-' + sanitizeForFilePath(uniqueProjectNamePathSegment);
if (this.retry)
testOutputDir += '-retry' + this.retry;
if (this.repeatEachIndex)
testOutputDir += '-repeat' + this.repeatEachIndex;
return path.join(this.project.outputDir, testOutputDir);
})();
this.snapshotDir = (() => {
const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile);
return path.join(this.project.snapshotDir, relativeTestFilePath + '-snapshots');
})();
}
private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
if (typeof modifierArgs[1] === 'function') {
throw new Error([
'It looks like you are calling test.skip() inside the test and pass a callback.',
'Pass a condition instead and optional description instead:',
`test('my test', async ({ page, isMobile }) => {`,
` test.skip(isMobile, 'This test is not applicable on mobile');`,
`});`,
].join('\n'));
}
if (modifierArgs.length >= 1 && !modifierArgs[0])
return;
const description = modifierArgs[1];
this.annotations.push({ type, description });
if (type === 'slow') {
this.setTimeout(this.timeout * 3);
} else if (type === 'skip' || type === 'fixme') {
this.expectedStatus = 'skipped';
throw new SkipError('Test is skipped: ' + (description || ''));
} else if (type === 'fail') {
if (this.expectedStatus !== 'skipped')
this.expectedStatus = 'failed';
}
}
async _runWithTimeout(cb: () => Promise<any>): Promise<void> {
try {
await this._timeoutRunner.run(cb);
} catch (error) {
if (!(error instanceof TimeoutRunnerError))
throw error;
// Do not overwrite existing failure upon hook/teardown timeout.
if (this.status === 'passed')
this.status = 'timedOut';
}
this.duration = monotonicTime() - this._startTime;
}
async _runFn(fn: Function, skips?: 'allowSkips'): Promise<TestError | undefined> {
try {
await fn();
} catch (error) {
if (skips === 'allowSkips' && error instanceof SkipError) {
if (this.status === 'passed')
this.status = 'skipped';
} else {
const serialized = serializeError(error);
this._failWithError(serialized);
return serialized;
}
}
}
_addStep(data: Omit<TestStepInternal, 'complete'>) {
return this._addStepImpl(data);
}
_failWithError(error: TestError) {
// Do not overwrite any previous error and error status.
// Some (but not all) scenarios include:
// - expect() that fails after uncaught exception.
// - fail after the timeout, e.g. due to fixture teardown.
if (this.status === 'passed')
this.status = 'failed';
if (this.error === undefined)
this.error = error;
}
_appendErrorMessage(message: string) {
// Do not overwrite any previous error status.
if (this.status === 'passed')
this.status = 'failed';
if (this.error === undefined)
this.error = { value: 'Error: ' + message };
else if (this.error.value)
this.error.value += '\nError: ' + message;
}
// ------------ TestInfo methods ------------
async attach(name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) {
if ((options.path !== undefined ? 1 : 0) + (options.body !== undefined ? 1 : 0) !== 1)
throw new Error(`Exactly one of "path" and "body" must be specified`);
if (options.path !== undefined) {
const hash = calculateSha1(options.path);
const dest = this.outputPath('attachments', hash + path.extname(options.path));
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
await fs.promises.copyFile(options.path, dest);
const contentType = options.contentType ?? (mime.getType(path.basename(options.path)) || 'application/octet-stream');
this.attachments.push({ name, contentType, path: dest });
} else {
const contentType = options.contentType ?? (typeof options.body === 'string' ? 'text/plain' : 'application/octet-stream');
this.attachments.push({ name, contentType, body: typeof options.body === 'string' ? Buffer.from(options.body) : options.body });
}
}
outputPath(...pathSegments: string[]){
fs.mkdirSync(this.outputDir, { recursive: true });
const joinedPath = path.join(...pathSegments);
const outputPath = getContainedPath(this.outputDir, joinedPath);
if (outputPath)
return outputPath;
throw new Error(`The outputPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\toutputPath: ${joinedPath}`);
}
snapshotPath(...pathSegments: string[]) {
let suffix = '';
const projectNamePathSegment = sanitizeForFilePath(this.project.name);
if (projectNamePathSegment)
suffix += '-' + projectNamePathSegment;
if (this.snapshotSuffix)
suffix += '-' + this.snapshotSuffix;
const subPath = addSuffixToFilePath(path.join(...pathSegments), suffix);
const snapshotPath = getContainedPath(this.snapshotDir, subPath);
if (snapshotPath)
return snapshotPath;
throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`);
}
skip(...args: [arg?: any, description?: string]) {
this._modifier('skip', args);
}
fixme(...args: [arg?: any, description?: string]) {
this._modifier('fixme', args);
}
fail(...args: [arg?: any, description?: string]) {
this._modifier('fail', args);
}
slow(...args: [arg?: any, description?: string]) {
this._modifier('slow', args);
}
setTimeout(timeout: number) {
if (!this.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
this.timeout = timeout;
this._timeoutRunner.updateTimeout(timeout);
}
}
class SkipError extends Error {
}

View File

@ -14,7 +14,7 @@
* limitations under the License. * limitations under the License.
*/ */
import type { Fixtures, TestError, TestInfo } from '../types/test'; import type { Fixtures, TestError } from '../types/test';
import type { Location } from '../types/testReporter'; import type { Location } from '../types/testReporter';
export * from '../types/test'; export * from '../types/test';
export { Location } from '../types/testReporter'; export { Location } from '../types/testReporter';
@ -34,8 +34,4 @@ export interface TestStepInternal {
location?: Location; location?: Location;
} }
export interface TestInfoImpl extends TestInfo {
_addStep: (data: Omit<TestStepInternal, 'complete'>) => TestStepInternal;
}
export type TestCaseType = 'beforeAll' | 'afterAll' | 'test'; export type TestCaseType = 'beforeAll' | 'afterAll' | 'test';

View File

@ -33,7 +33,7 @@ global.console = new Console({
process.stdout.write = (chunk: string | Buffer) => { process.stdout.write = (chunk: string | Buffer) => {
const outPayload: TestOutputPayload = { const outPayload: TestOutputPayload = {
testId: workerRunner?._currentTest?.testId, testId: workerRunner?._currentTest?._test._id,
...chunkToParams(chunk) ...chunkToParams(chunk)
}; };
sendMessageToParent('stdOut', outPayload); sendMessageToParent('stdOut', outPayload);
@ -43,7 +43,7 @@ process.stdout.write = (chunk: string | Buffer) => {
if (!process.env.PW_RUNNER_DEBUG) { if (!process.env.PW_RUNNER_DEBUG) {
process.stderr.write = (chunk: string | Buffer) => { process.stderr.write = (chunk: string | Buffer) => {
const outPayload: TestOutputPayload = { const outPayload: TestOutputPayload = {
testId: workerRunner?._currentTest?.testId, testId: workerRunner?._currentTest?._test._id,
...chunkToParams(chunk) ...chunkToParams(chunk)
}; };
sendMessageToParent('stdErr', outPayload); sendMessageToParent('stdErr', outPayload);

View File

@ -14,44 +14,36 @@
* limitations under the License. * limitations under the License.
*/ */
import fs from 'fs';
import path from 'path';
import rimraf from 'rimraf'; import rimraf from 'rimraf';
import * as mime from 'mime';
import util from 'util'; import util from 'util';
import colors from 'colors/safe'; import colors from 'colors/safe';
import { EventEmitter } from 'events'; import { EventEmitter } from 'events';
import { monotonicTime, serializeError, sanitizeForFilePath, getContainedPath, addSuffixToFilePath, prependToTestError, trimLongString, formatLocation } from './util'; import { serializeError, prependToTestError, formatLocation } from './util';
import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc'; import { TestBeginPayload, TestEndPayload, RunPayload, TestEntry, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload } from './ipc';
import { setCurrentTestInfo } from './globals'; import { setCurrentTestInfo } from './globals';
import { Loader } from './loader'; import { Loader } from './loader';
import { Modifier, Suite, TestCase } from './test'; import { Modifier, Suite, TestCase } from './test';
import { Annotations, TestCaseType, TestError, TestInfo, TestInfoImpl, TestStepInternal, WorkerInfo } from './types'; import { Annotations, TestError, TestInfo, TestStepInternal, WorkerInfo } from './types';
import { ProjectImpl } from './project'; import { ProjectImpl } from './project';
import { FixtureRunner } from './fixtures'; import { FixtureRunner } from './fixtures';
import { TimeoutRunner, raceAgainstTimeout, TimeoutRunnerError } from 'playwright-core/lib/utils/async'; import { raceAgainstTimeout } from 'playwright-core/lib/utils/async';
import { calculateSha1 } from 'playwright-core/lib/utils/utils'; import { TestInfoImpl } from './testInfo';
const removeFolderAsync = util.promisify(rimraf); const removeFolderAsync = util.promisify(rimraf);
type TestData = { testId: string, testInfo: TestInfoImpl, type: TestCaseType };
export class WorkerRunner extends EventEmitter { export class WorkerRunner extends EventEmitter {
private _params: WorkerInitParams; private _params: WorkerInitParams;
private _loader!: Loader; private _loader!: Loader;
private _project!: ProjectImpl; private _project!: ProjectImpl;
private _workerInfo!: WorkerInfo; private _workerInfo!: WorkerInfo;
private _projectNamePathSegment = '';
private _uniqueProjectNamePathSegment = '';
private _fixtureRunner: FixtureRunner; private _fixtureRunner: FixtureRunner;
private _failedTest: TestData | undefined; private _failedTest: TestInfoImpl | undefined;
private _fatalError: TestError | undefined; private _fatalError: TestError | undefined;
private _entries = new Map<string, TestEntry>(); private _entries = new Map<string, TestEntry>();
private _isStopped = false; private _isStopped = false;
private _runFinished = Promise.resolve(); private _runFinished = Promise.resolve();
private _currentTimeoutRunner: TimeoutRunner | undefined; _currentTest: TestInfoImpl | null = null;
_currentTest: TestData | null = null;
constructor(params: WorkerInitParams) { constructor(params: WorkerInitParams) {
super(); super();
@ -64,11 +56,11 @@ export class WorkerRunner extends EventEmitter {
this._isStopped = true; this._isStopped = true;
// Interrupt current action. // Interrupt current action.
this._currentTimeoutRunner?.interrupt(); this._currentTest?._timeoutRunner.interrupt();
// TODO: mark test as 'interrupted' instead. // TODO: mark test as 'interrupted' instead.
if (this._currentTest && this._currentTest.testInfo.status === 'passed') if (this._currentTest && this._currentTest.status === 'passed')
this._currentTest.testInfo.status = 'skipped'; this._currentTest.status = 'skipped';
} }
return this._runFinished; return this._runFinished;
} }
@ -102,11 +94,8 @@ export class WorkerRunner extends EventEmitter {
// a test runner. In the latter case, the worker state could be messed up, // a test runner. In the latter case, the worker state could be messed up,
// and continuing to run tests in the same worker is problematic. Therefore, // and continuing to run tests in the same worker is problematic. Therefore,
// we turn this into a fatal error and restart the worker anyway. // we turn this into a fatal error and restart the worker anyway.
if (this._currentTest && this._currentTest.type === 'test' && this._currentTest.testInfo.expectedStatus !== 'failed') { if (this._currentTest && this._currentTest._test._type === 'test' && this._currentTest.expectedStatus !== 'failed') {
if (!this._currentTest.testInfo.error) { this._currentTest._failWithError(serializeError(error));
this._currentTest.testInfo.status = 'failed';
this._currentTest.testInfo.error = serializeError(error);
}
} else { } else {
// No current test - fatal error. // No current test - fatal error.
if (!this._fatalError) if (!this._fatalError)
@ -122,15 +111,6 @@ export class WorkerRunner extends EventEmitter {
this._loader = await Loader.deserialize(this._params.loader); this._loader = await Loader.deserialize(this._params.loader);
this._project = this._loader.projects()[this._params.projectIndex]; this._project = this._loader.projects()[this._params.projectIndex];
this._projectNamePathSegment = sanitizeForFilePath(this._project.config.name);
const sameName = this._loader.projects().filter(project => project.config.name === this._project.config.name);
if (sameName.length > 1)
this._uniqueProjectNamePathSegment = this._project.config.name + (sameName.indexOf(this._project) + 1);
else
this._uniqueProjectNamePathSegment = this._project.config.name;
this._uniqueProjectNamePathSegment = sanitizeForFilePath(this._uniqueProjectNamePathSegment);
this._workerInfo = { this._workerInfo = {
workerIndex: this._params.workerIndex, workerIndex: this._params.workerIndex,
parallelIndex: this._params.parallelIndex, parallelIndex: this._params.parallelIndex,
@ -218,101 +198,8 @@ export class WorkerRunner extends EventEmitter {
} }
private async _runTestOrAllHook(test: TestCase, annotations: Annotations, retry: number) { private async _runTestOrAllHook(test: TestCase, annotations: Annotations, retry: number) {
const startTime = monotonicTime();
const startWallTime = Date.now();
let timeoutRunner: TimeoutRunner;
const testId = test._id;
const baseOutputDir = (() => {
const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile.replace(/\.(spec|test)\.(js|ts|mjs)$/, ''));
const sanitizedRelativePath = relativeTestFilePath.replace(process.platform === 'win32' ? new RegExp('\\\\', 'g') : new RegExp('/', 'g'), '-');
const fullTitleWithoutSpec = test.titlePath().slice(1).join(' ') + (test._type === 'test' ? '' : '-worker' + this._params.workerIndex);
let testOutputDir = sanitizedRelativePath + '-' + sanitizeForFilePath(trimLongString(fullTitleWithoutSpec));
if (this._uniqueProjectNamePathSegment)
testOutputDir += '-' + this._uniqueProjectNamePathSegment;
if (retry)
testOutputDir += '-retry' + retry;
if (this._params.repeatEachIndex)
testOutputDir += '-repeat' + this._params.repeatEachIndex;
return path.join(this._project.config.outputDir, testOutputDir);
})();
const snapshotDir = (() => {
const relativeTestFilePath = path.relative(this._project.config.testDir, test._requireFile);
return path.join(this._project.config.snapshotDir, relativeTestFilePath + '-snapshots');
})();
let lastStepId = 0; let lastStepId = 0;
const testInfo: TestInfoImpl = { const testInfo = new TestInfoImpl(this._loader, this._params, test, retry, data => {
workerIndex: this._params.workerIndex,
parallelIndex: this._params.parallelIndex,
project: this._project.config,
config: this._loader.fullConfig(),
title: test.title,
titlePath: test.titlePath(),
file: test.location.file,
line: test.location.line,
column: test.location.column,
fn: test.fn,
repeatEachIndex: this._params.repeatEachIndex,
retry,
expectedStatus: test.expectedStatus,
annotations: [],
attachments: [],
attach: async (name: string, options: { path?: string, body?: string | Buffer, contentType?: string } = {}) => {
if ((options.path !== undefined ? 1 : 0) + (options.body !== undefined ? 1 : 0) !== 1)
throw new Error(`Exactly one of "path" and "body" must be specified`);
if (options.path !== undefined) {
const hash = calculateSha1(options.path);
const dest = testInfo.outputPath('attachments', hash + path.extname(options.path));
await fs.promises.mkdir(path.dirname(dest), { recursive: true });
await fs.promises.copyFile(options.path, dest);
const contentType = options.contentType ?? (mime.getType(path.basename(options.path)) || 'application/octet-stream');
testInfo.attachments.push({ name, contentType, path: dest });
} else {
const contentType = options.contentType ?? (typeof options.body === 'string' ? 'text/plain' : 'application/octet-stream');
testInfo.attachments.push({ name, contentType, body: typeof options.body === 'string' ? Buffer.from(options.body) : options.body });
}
},
duration: 0,
status: 'passed',
stdout: [],
stderr: [],
timeout: this._project.config.timeout,
snapshotSuffix: '',
outputDir: baseOutputDir,
snapshotDir,
outputPath: (...pathSegments: string[]): string => {
fs.mkdirSync(baseOutputDir, { recursive: true });
const joinedPath = path.join(...pathSegments);
const outputPath = getContainedPath(baseOutputDir, joinedPath);
if (outputPath) return outputPath;
throw new Error(`The outputPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\toutputPath: ${joinedPath}`);
},
snapshotPath: (...pathSegments: string[]): string => {
let suffix = '';
if (this._projectNamePathSegment)
suffix += '-' + this._projectNamePathSegment;
if (testInfo.snapshotSuffix)
suffix += '-' + testInfo.snapshotSuffix;
const subPath = addSuffixToFilePath(path.join(...pathSegments), suffix);
const snapshotPath = getContainedPath(snapshotDir, subPath);
if (snapshotPath) return snapshotPath;
throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`);
},
skip: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'skip', args),
fixme: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fixme', args),
fail: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'fail', args),
slow: (...args: [arg?: any, description?: string]) => modifier(testInfo, 'slow', args),
setTimeout: (timeout: number) => {
if (!testInfo.timeout)
return; // Zero timeout means some debug mode - do not set a timeout.
testInfo.timeout = timeout;
if (timeoutRunner)
timeoutRunner.updateTimeout(timeout);
},
_addStep: data => {
const stepId = `${data.category}@${data.title}@${++lastStepId}`; const stepId = `${data.category}@${data.title}@${++lastStepId}`;
let callbackHandled = false; let callbackHandled = false;
const step: TestStepInternal = { const step: TestStepInternal = {
@ -324,7 +211,7 @@ export class WorkerRunner extends EventEmitter {
if (error instanceof Error) if (error instanceof Error)
error = serializeError(error); error = serializeError(error);
const payload: StepEndPayload = { const payload: StepEndPayload = {
testId, testId: test._id,
stepId, stepId,
wallTime: Date.now(), wallTime: Date.now(),
error, error,
@ -336,7 +223,7 @@ export class WorkerRunner extends EventEmitter {
// Sanitize location that comes from user land, it might have extra properties. // Sanitize location that comes from user land, it might have extra properties.
const location = data.location && hasLocation ? { file: data.location.file, line: data.location.line, column: data.location.column } : undefined; const location = data.location && hasLocation ? { file: data.location.file, line: data.location.line, column: data.location.column } : undefined;
const payload: StepBeginPayload = { const payload: StepBeginPayload = {
testId, testId: test._id,
stepId, stepId,
...data, ...data,
location, location,
@ -344,8 +231,7 @@ export class WorkerRunner extends EventEmitter {
}; };
this.emit('stepBegin', payload); this.emit('stepBegin', payload);
return step; return step;
}, });
};
// Inherit test.setTimeout() from parent suites. // Inherit test.setTimeout() from parent suites.
for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) { for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) {
@ -373,40 +259,35 @@ export class WorkerRunner extends EventEmitter {
} }
} }
const testData: TestData = { testInfo, testId, type: test._type }; this._currentTest = testInfo;
this._currentTest = testData;
setCurrentTestInfo(testInfo); setCurrentTestInfo(testInfo);
this.emit('testBegin', buildTestBeginPayload(testData, startWallTime)); this.emit('testBegin', buildTestBeginPayload(testInfo));
if (testInfo.expectedStatus === 'skipped') { if (testInfo.expectedStatus === 'skipped') {
testInfo.status = 'skipped'; testInfo.status = 'skipped';
this.emit('testEnd', buildTestEndPayload(testData)); this.emit('testEnd', buildTestEndPayload(testInfo));
return; return;
} }
// Update the fixture pool - it may differ between tests, but only in test-scoped fixtures. // Update the fixture pool - it may differ between tests, but only in test-scoped fixtures.
this._fixtureRunner.setPool(test._pool!); this._fixtureRunner.setPool(test._pool!);
this._currentTimeoutRunner = timeoutRunner = new TimeoutRunner(testInfo.timeout); await testInfo._runWithTimeout(() => this._runTestWithBeforeHooks(test, testInfo));
await this._runWithTimeout(timeoutRunner, () => this._runTestWithBeforeHooks(test, testInfo), testInfo);
testInfo.duration = monotonicTime() - startTime;
if (testInfo.status === 'timedOut') { if (testInfo.status === 'timedOut') {
// A timed-out test gets a full additional timeout to run after hooks. // A timed-out test gets a full additional timeout to run after hooks.
timeoutRunner.resetTimeout(testInfo.timeout); testInfo._timeoutRunner.resetTimeout(testInfo.timeout);
} }
await this._runWithTimeout(timeoutRunner, () => this._runAfterHooks(test, testInfo), testInfo); await testInfo._runWithTimeout(() => this._runAfterHooks(test, testInfo));
testInfo.duration = monotonicTime() - startTime;
this._currentTimeoutRunner = undefined;
this._currentTest = null; this._currentTest = null;
setCurrentTestInfo(null); setCurrentTestInfo(null);
const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus; const isFailure = testInfo.status !== 'skipped' && testInfo.status !== testInfo.expectedStatus;
if (isFailure) { if (isFailure) {
// Delay reporting testEnd result until after teardownScopes is done. // Delay reporting testEnd result until after teardownScopes is done.
this._failedTest = testData; this._failedTest = testInfo;
if (test._type !== 'test') { if (test._type !== 'test') {
// beforeAll/afterAll hook failure skips any remaining tests in the worker. // beforeAll/afterAll hook failure skips any remaining tests in the worker.
if (!this._fatalError) if (!this._fatalError)
@ -417,7 +298,7 @@ export class WorkerRunner extends EventEmitter {
} }
this.stop(); this.stop();
} else { } else {
this.emit('testEnd', buildTestEndPayload(testData)); this.emit('testEnd', buildTestEndPayload(testInfo));
} }
const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' || const preserveOutput = this._loader.fullConfig().preserveOutput === 'always' ||
@ -433,7 +314,7 @@ export class WorkerRunner extends EventEmitter {
canHaveChildren: true, canHaveChildren: true,
forceNoParent: true forceNoParent: true
}); });
const maybeError = await this._runFn(async () => { const maybeError = await testInfo._runFn(async () => {
if (test._type === 'test') { if (test._type === 'test') {
const beforeEachModifiers: Modifier[] = []; const beforeEachModifiers: Modifier[] = [];
for (let s: Suite | undefined = test.parent; s; s = s.parent) { for (let s: Suite | undefined = test.parent; s; s = s.parent) {
@ -452,7 +333,7 @@ export class WorkerRunner extends EventEmitter {
step.complete(); // Report fixture hooks step as completed. step.complete(); // Report fixture hooks step as completed.
const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]"). const fn = test.fn; // Extract a variable to get a better stack trace ("myTest" vs "TestCase.myTest [as fn]").
await fn(params, testInfo); await fn(params, testInfo);
}, testInfo, 'allowSkips'); }, 'allowSkips');
step.complete(maybeError); // Second complete is a no-op. step.complete(maybeError); // Second complete is a no-op.
} }
@ -466,10 +347,10 @@ export class WorkerRunner extends EventEmitter {
let teardownError1: TestError | undefined; let teardownError1: TestError | undefined;
if (test._type === 'test') if (test._type === 'test')
teardownError1 = await this._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo), testInfo); teardownError1 = await testInfo._runFn(() => this._runHooks(test.parent!, 'afterEach', testInfo));
// Continue teardown even after the failure. // Continue teardown even after the failure.
const teardownError2 = await this._runFn(() => this._fixtureRunner.teardownScope('test'), testInfo); const teardownError2 = await testInfo._runFn(() => this._fixtureRunner.teardownScope('test'));
step.complete(teardownError1 || teardownError2); step.complete(teardownError1 || teardownError2);
} }
@ -494,40 +375,6 @@ export class WorkerRunner extends EventEmitter {
throw error; throw error;
} }
private async _runWithTimeout(timeoutRunner: TimeoutRunner, cb: () => Promise<any>, testInfo: TestInfoImpl): Promise<void> {
try {
await timeoutRunner.run(cb);
} catch (error) {
if (!(error instanceof TimeoutRunnerError))
throw error;
// Do not overwrite existing failure upon hook/teardown timeout.
if (testInfo.status === 'passed')
testInfo.status = 'timedOut';
}
}
private async _runFn(fn: Function, testInfo: TestInfoImpl, skips?: 'allowSkips'): Promise<TestError | undefined> {
try {
await fn();
} catch (error) {
if (skips === 'allowSkips' && error instanceof SkipError) {
if (testInfo.status === 'passed')
testInfo.status = 'skipped';
} else {
const serialized = serializeError(error);
// Do not overwrite any previous error and error status.
// Some (but not all) scenarios include:
// - expect() that fails after uncaught exception.
// - fail after the timeout, e.g. due to fixture teardown.
if (testInfo.status === 'passed')
testInfo.status = 'failed';
if (!('error' in testInfo))
testInfo.error = serialized;
return serialized;
}
}
}
private _reportDone() { private _reportDone() {
const donePayload: DonePayload = { fatalError: this._fatalError }; const donePayload: DonePayload = { fatalError: this._fatalError };
this.emit('done', donePayload); this.emit('done', donePayload);
@ -536,17 +383,16 @@ export class WorkerRunner extends EventEmitter {
} }
} }
function buildTestBeginPayload(testData: TestData, startWallTime: number): TestBeginPayload { function buildTestBeginPayload(testInfo: TestInfoImpl): TestBeginPayload {
return { return {
testId: testData.testId, testId: testInfo._test._id,
startWallTime, startWallTime: testInfo._startWallTime,
}; };
} }
function buildTestEndPayload(testData: TestData): TestEndPayload { function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload {
const { testId, testInfo } = testData;
return { return {
testId, testId: testInfo._test._id,
duration: testInfo.duration, duration: testInfo.duration,
status: testInfo.status!, status: testInfo.status!,
error: testInfo.error, error: testInfo.error,
@ -561,33 +407,3 @@ function buildTestEndPayload(testData: TestData): TestEndPayload {
})) }))
}; };
} }
function modifier(testInfo: TestInfo, type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
if (typeof modifierArgs[1] === 'function') {
throw new Error([
'It looks like you are calling test.skip() inside the test and pass a callback.',
'Pass a condition instead and optional description instead:',
`test('my test', async ({ page, isMobile }) => {`,
` test.skip(isMobile, 'This test is not applicable on mobile');`,
`});`,
].join('\n'));
}
if (modifierArgs.length >= 1 && !modifierArgs[0])
return;
const description = modifierArgs[1];
testInfo.annotations.push({ type, description });
if (type === 'slow') {
testInfo.setTimeout(testInfo.timeout * 3);
} else if (type === 'skip' || type === 'fixme') {
testInfo.expectedStatus = 'skipped';
throw new SkipError('Test is skipped: ' + (description || ''));
} else if (type === 'fail') {
if (testInfo.expectedStatus !== 'skipped')
testInfo.expectedStatus = 'failed';
}
}
class SkipError extends Error {
}