test: take a screenshot upon failure example (#3556)

test-runner/src/fixtures.ts

@@ -38,6 +38,19 @@ export function setParameters(params: any) {
     registerWorkerFixture(name as keyof WorkerState, async ({}, test) => await test(parameters[name] as never));
 }
 
+type TestInfo = {
+  file: string;
+  title: string;
+  timeout: number;
+  outputDir: string;
+  testDir: string;
+};
+
+type TestResult = {
+  success: boolean;
+  info: TestInfo;
+  error?: Error;
+};
 
 class Fixture {
   pool: FixturePool;
@@ -82,13 +95,13 @@ class Fixture {
     this._tearDownComplete = this.fn(params, async (value: any) => {
       this.value = value;
       setupFenceFulfill();
-      await teardownFence;
+      return await teardownFence;
     }).catch((e: any) => setupFenceReject(e));
     await setupFence;
     this._setup = true;
   }
 
-  async teardown() {
+  async teardown(testResult: TestResult) {
     if (this.hasGeneratorValue)
       return;
     if (this._teardown)
@@ -98,11 +111,11 @@ class Fixture {
       const fixture = this.pool.instances.get(name);
       if (!fixture)
         continue;
-      await fixture.teardown();
+      await fixture.teardown(testResult);
     }
     if (this._setup) {
       debug('pw:test:hook')(`teardown "${this.name}"`);
-      this._teardownFenceCallback();
+      this._teardownFenceCallback(testResult);
     }
     await this._tearDownComplete;
     this.pool.instances.delete(this.name);
@@ -129,31 +142,45 @@ export class FixturePool {
     return fixture;
   }
 
-  async teardownScope(scope: string) {
+  async teardownScope(scope: string, testResult?: TestResult) {
     for (const [name, fixture] of this.instances) {
       if (fixture.scope === scope)
-        await fixture.teardown();
+        await fixture.teardown(testResult);
     }
   }
 
-  async resolveParametersAndRun(fn: (arg0: {}) => any) {
+  async resolveParametersAndRun(fn: (arg0: {}) => any, timeout: number) {
     const names = fixtureParameterNames(fn);
     for (const name of names)
       await this.setupFixture(name);
     const params = {};
     for (const n of names)
       params[n] = this.instances.get(n).value;
-    return fn(params);
+
+    if (!timeout)
+      return fn(params);
+
+    let timer;
+    let timerPromise = new Promise(f => timer = setTimeout(f, timeout));
+    return Promise.race([
+      Promise.resolve(fn(params)).then(() => clearTimeout(timer)),
+      timerPromise.then(() => Promise.reject(new Error(`Timeout of ${timeout}ms exceeded`)))
+    ]);
   }
 
-  wrapTestCallback(callback: any) {
+  wrapTestCallback(callback: any, timeout: number, info: TestInfo) {
     if (!callback)
       return callback;
+    const testResult: TestResult = { success: true, info };
     return async() => {
       try {
-        return await this.resolveParametersAndRun(callback);
+        await this.resolveParametersAndRun(callback, timeout);
+      } catch (e) {
+        testResult.success = false;
+        testResult.error = e;
+        throw e;
       } finally {
-        await this.teardownScope('test');
+        await this.teardownScope('test', testResult);
       }
     };
   }
@@ -188,15 +215,6 @@ function fixtureParameterNames(fn: { toString: () => any; }) {
   return signature.split(',').map((t: string) => t.trim());
 }
 
-function optionParameterNames(fn: { toString: () => any; }) {
-  const text = fn.toString();
-  const match = text.match(/(?:\s+function)?\s*\(\s*{\s*([^}]*)\s*}/);
-  if (!match || !match[1].trim())
-    return [];
-  let signature = match[1];
-  return signature.split(',').map((t: string) => t.trim());
-}
-
 function innerRegisterFixture(name: any, scope: string, fn: any, caller: Function) {
   const obj = {stack: ''};
   Error.captureStackTrace(obj, caller);
@@ -210,7 +228,7 @@ function innerRegisterFixture(name: any, scope: string, fn: any, caller: Functio
   registrationsByFile.get(file).push(registration);
 };
 
-export function registerFixture<T extends keyof TestState>(name: T, fn: (params: FixtureParameters & WorkerState & TestState, test: (arg: TestState[T]) => Promise<void>) => Promise<void>) {
+export function registerFixture<T extends keyof TestState>(name: T, fn: (params: FixtureParameters & WorkerState & TestState, test: (arg: TestState[T]) => Promise<TestResult>) => Promise<void>) {
   innerRegisterFixture(name, 'test', fn, registerFixture);
 };
 

test-runner/src/fixturesUI.js

@@ -63,7 +63,7 @@ function fixturesUI(wrappers, suite) {
 
       if (suite.isPending())
         fn = null;
-      const wrapper = fn ? wrappers.testWrapper(fn) : undefined;
+      const wrapper = fn ? wrappers.testWrapper(fn, title, file, specs.slow && specs.slow[0]) : undefined;
       if (wrapper) {
         wrapper.toString = () => fn.toString();
         wrapper.__original = fn;
@@ -72,8 +72,6 @@ function fixturesUI(wrappers, suite) {
       test.file = file;
       suite.addTest(test);
       const only = wrappers.ignoreOnly ? false : specs.only && specs.only[0];
-      if (specs.slow && specs.slow[0])
-        test.timeout(90000);
       if (only)
         test.__only = true;
       if (!only && specs.skip && specs.skip[0])

test-runner/src/testRunner.ts

@@ -31,49 +31,55 @@ declare global {
 global.expect = require('expect');
 const GoldenUtils = require('./GoldenUtils');
 
+export type TestRunnerEntry = {
+  file: string;
+  ordinals: number[];
+  configuredFile: string;
+  configurationObject: any;
+};
+
 class NullReporter {}
 
 export class TestRunner extends EventEmitter {
   mocha: any;
-  _currentOrdinal: number;
+  private _currentOrdinal = -1;
-  _failedWithError: boolean;
+  private _failedWithError = false;
-  _file: any;
+  private _file: any;
-  _ordinals: Set<unknown>;
+  private _ordinals: Set<number>;
-  _remaining: Set<unknown>;
+  private _remaining: Set<number>;
-  _trialRun: any;
+  private _trialRun: any;
-  _passes: number;
+  private _passes = 0;
-  _failures: number;
+  private _failures = 0;
-  _pending: number;
+  private _pending = 0;
-  _configuredFile: any;
+  private _configuredFile: any;
-  _configurationObject: any;
+  private _configurationObject: any;
-  _configurationString: any;
+  private _parsedGeneratorConfiguration: any = {};
-  _parsedGeneratorConfiguration: {};
+  private _relativeTestFile: string;
-  _relativeTestFile: string;
+  private _runner: Mocha.Runner;
-  _runner: any;
+  private _outDir: string;
-  constructor(entry, options, workerId) {
+  private _timeout: number;
+  private _testDir: string;
+
+  constructor(entry: TestRunnerEntry, options, workerId) {
     super();
     this.mocha = new Mocha({
       reporter: NullReporter,
-      timeout: options.timeout,
+      timeout: 0,
       ui: fixturesUI.bind(null, {
-        testWrapper: fn => this._testWrapper(fn),
+        testWrapper: (fn, title, file, isSlow) => this._testWrapper(fn, title, file, isSlow),
         hookWrapper: (hook, fn) => this._hookWrapper(hook, fn),
         ignoreOnly: true
       }),
     });
-    this._currentOrdinal = -1;
-    this._failedWithError = false;
     this._file = entry.file;
     this._ordinals = new Set(entry.ordinals);
     this._remaining = new Set(entry.ordinals);
     this._trialRun = options.trialRun;
-    this._passes = 0;
+    this._timeout = options.timeout;
-    this._failures = 0;
+    this._testDir = options.testDir;
-    this._pending = 0;
+    this._outDir = options.outputDir;
     this._configuredFile = entry.configuredFile;
     this._configurationObject = entry.configurationObject;
-    this._configurationString = entry.configurationString;
-    this._parsedGeneratorConfiguration = {};
     for (const {name, value} of this._configurationObject) {
       this._parsedGeneratorConfiguration[name] = value;
       // @ts-ignore
@@ -168,8 +174,15 @@ export class TestRunner extends EventEmitter {
     return true;
   }
 
-  _testWrapper(fn) {
+  _testWrapper(fn, title, file, isSlow) {
-    const wrapped = fixturePool.wrapTestCallback(fn);
+    const timeout = isSlow ? this._timeout * 3 : this._timeout;
+    const wrapped = fixturePool.wrapTestCallback(fn, timeout, {
+      outputDir: this._outDir,
+      testDir: this._testDir,
+      title,
+      file,
+      timeout
+    });
     return wrapped ? (done, ...args) => {
       if (!this._shouldRunTest()) {
         done();
@@ -183,7 +196,7 @@ export class TestRunner extends EventEmitter {
     if (!this._shouldRunTest(true))
       return;
     return hook(async () => {
-      return await fixturePool.resolveParametersAndRun(fn);
+      return await fixturePool.resolveParametersAndRun(fn, 0);
     });
   }
 

test/playwright.fixtures.ts

@@ -41,9 +41,9 @@ declare global {
     browserType: BrowserType<Browser>;
     browser: Browser;
     httpService: {server: TestServer, httpsServer: TestServer}
+    toImpl: (rpcObject: any) => any;
   }
   interface TestState {
-    toImpl: (rpcObject: any) => any;
     context: BrowserContext;
     server: TestServer;
     page: Page;
@@ -142,7 +142,7 @@ registerWorkerFixture('playwright', async({browserName, wire}, test) => {
 
 });
 
-registerFixture('toImpl', async ({playwright}, test) => {
+registerWorkerFixture('toImpl', async ({playwright}, test) => {
   await test((playwright as any)._toImpl);
 });
 
@@ -165,6 +165,14 @@ registerWorkerFixture('browser', async ({browserType, defaultBrowserOptions}, te
   await browser.close();
 });
 
+registerWorkerFixture('asset', async ({}, test) => {
+  await test(p => path.join(__dirname, `assets`, p));
+});
+
+registerWorkerFixture('golden', async ({browserName}, test) => {
+  await test(p => path.join(browserName, p));
+});
+
 registerFixture('context', async ({browser}, test) => {
   const context = await browser.newContext();
   await test(context);
@@ -173,7 +181,13 @@ registerFixture('context', async ({browser}, test) => {
 
 registerFixture('page', async ({context}, test) => {
   const page = await context.newPage();
-  await test(page);
+  const { success, info } = await test(page);
+  if (!success) {
+    const relativePath = path.relative(info.testDir, info.file).replace(/\.spec\.[jt]s/, '');
+    const sanitizedTitle = info.title.replace(/[^\w\d]+/g, '_');
+    const assetPath = path.join(info.outputDir, relativePath, sanitizedTitle) + '-failed.png';
+    await page.screenshot({ path: assetPath });
+  }
 });
 
 registerFixture('server', async ({httpService}, test) => {
@@ -192,14 +206,6 @@ registerFixture('tmpDir', async ({}, test) => {
   await removeFolderAsync(tmpDir).catch(e => {});
 });
 
-registerWorkerFixture('asset', async ({}, test) => {
-  await test(p => path.join(__dirname, `assets`, p));
-});
-
-registerWorkerFixture('golden', async ({browserName}, test) => {
-  await test(p => path.join(browserName, p));
-});
-
 export const options = {
   CHROMIUM: parameters.browserName === 'chromium',
   FIREFOX: parameters.browserName === 'firefox',

test/test-runner.spec.ts

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import './test-runner-helper';
+import { registerFixture } from '../test-runner';
+
+declare global {
+  interface TestState {
+    a: string;
+    b: string;
+    ab: string;
+    zero: number;
+    tear: string;
+  }
+}
+
+let zero = 0;
+registerFixture('zero', async ({}, test) => {
+  await test(zero++);
+});
+
+registerFixture('a', async ({zero}, test) => {
+  await test('a' + zero);
+});
+
+registerFixture('b', async ({zero}, test) => {
+  await test('b' + zero);
+});
+
+registerFixture('ab', async ({a, b}, test) => {
+  await test(a + b);
+});
+
+it('should eval fixture once', async ({ab}) => {
+  expect(ab).toBe('a0b0');
+});