feat(testrunner): convert reporter to an interface (#3588)

test-runner/src/cli.ts

@@ -18,7 +18,15 @@ import program from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
 import { collectTests, runTests, RunnerConfig } from '.';
-import { reporters } from './reporters';
+import { DotReporter } from './reporters/dot';
+import { ListReporter } from './reporters/list';
+import { JSONReporter } from './reporters/json';
+
+export const reporters = {
+  'dot': DotReporter,
+  'list': ListReporter,
+  'json': JSONReporter
+};
 
 program
   .version('Version ' + /** @type {any} */ (require)('../package.json').version)
@@ -66,8 +74,8 @@ program
       process.exit(1);
     }
 
-    const reporterFactory = reporters[command.reporter || 'dot'];
-    await runTests(config, suite, reporterFactory);
+    const reporter = new (reporters[command.reporter || 'dot'])();
+    await runTests(config, suite, reporter);
     const hasFailures = suite.eachTest(t => t.error);
     process.exit(hasFailures ? 1 : 0);
   });

test-runner/src/index.ts

@@ -20,13 +20,14 @@ import * as path from 'path';
 import './builtin.fixtures';
 import './expect';
 import { registerFixture as registerFixtureT, registerWorkerFixture as registerWorkerFixtureT } from './fixtures';
-import { reporters } from './reporters';
+import { Reporter } from './reporter';
 import { Runner } from './runner';
 import { RunnerConfig } from './runnerConfig';
 import { Suite, Test } from './test';
 import { Matrix, TestCollector } from './testCollector';
 import { installTransform } from './transform';
 export { parameters, registerParameter } from './fixtures';
+export { Reporter } from './reporter';
 export { RunnerConfig } from './runnerConfig';
 export { Suite, Test } from './test';
 
@@ -76,11 +77,10 @@ export function collectTests(config: RunnerConfig, files: string[]): Suite {
   return testCollector.suite;
 }
 
-export async function runTests(config: RunnerConfig, suite: Suite, reporterFactory: any) {
+export async function runTests(config: RunnerConfig, suite: Suite, reporter: Reporter) {
   // Trial run does not need many workers, use one.
   const jobs = (config.trialRun || config.debug) ? 1 : config.jobs;
-  const runner = new Runner(suite, { ...config, jobs });
-  new reporterFactory(runner);
+  const runner = new Runner(suite, { ...config, jobs }, reporter);
 
   try {
     for (const f of beforeFunctions)

test-runner/src/reporter.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RunnerConfig } from './runnerConfig';
+import { Suite, Test } from './test';
+
+export interface Reporter {
+  onBegin(config: RunnerConfig, suite: Suite): void;
+  onTest(test: Test): void;
+  onPending(test: Test): void;
+  onPass(test: Test): void;
+  onFail(test: Test): void;
+  onEnd(): void;
+}

test-runner/src/reporters.ts

@@ -1,234 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import colors from 'colors/safe';
-import milliseconds from 'ms';
-import { codeFrameColumns } from '@babel/code-frame';
-import path from 'path';
-import fs from 'fs';
-import os from 'os';
-import terminalLink from 'terminal-link';
-import StackUtils from 'stack-utils';
-import { Test, Suite } from './test';
-import { EventEmitter } from 'ws';
-import { RunnerConfig } from './runnerConfig';
-
-const stackUtils = new StackUtils();
-
-class BaseReporter {
-  pending: Test[] = [];
-  passes: Test[] = [];
-  failures: Test[] = [];
-  duration = 0;
-  startTime: number;
-  config: RunnerConfig;
-  suite: Suite;
-
-  constructor(runner: EventEmitter) {
-    process.on('SIGINT', async () => {
-      this.epilogue();
-      process.exit(130);
-    });
-
-    runner.on('pending', (test: Test) => {
-      this.pending.push(test);
-    });
-
-    runner.on('pass', (test: Test) => {
-      this.passes.push(test);
-    });
-
-    runner.on('fail', (test: Test) => {
-      this.failures.push(test);
-    });
-
-    runner.once('begin', (options: { config: RunnerConfig, suite: Suite }) => {
-      this.startTime = Date.now();
-      this.config = options.config;
-      this.suite = options.suite;
-    });
-
-    runner.once('end', () => {
-      this.duration = Date.now() - this.startTime;
-    });
-  }
-
-  epilogue() {
-    console.log('');
-
-    console.log(colors.green(`  ${this.passes.length} passing`) + colors.dim(` (${milliseconds(this.duration)})`));  
-
-    if (this.pending.length)
-      console.log(colors.yellow(`  ${this.pending.length} skipped`));
-
-    if (this.failures.length) {  
-      console.log(colors.red(`  ${this.failures.length} failing`));
-      console.log('');
-      this.failures.forEach((failure, index) => {
-        const relativePath = path.relative(process.cwd(), failure.file);
-        const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
-        console.log(colors.bold(colors.red(header)));
-        const stack = failure.error.stack;
-        if (stack) {
-          console.log('');
-          const messageLocation = failure.error.stack.indexOf(failure.error.message);
-          const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
-          console.log(indent(preamble, '    '));
-          const position = positionInFile(stack, failure.file);
-          if (position) {
-            const source = fs.readFileSync(failure.file, 'utf8');
-            console.log('');
-            console.log(indent(codeFrameColumns(source, {
-                start: position,
-              },
-              { highlightCode: true}
-            ), '    '));
-          }
-          console.log('');
-          console.log(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
-        } else {
-          console.log('');
-          console.log(indent(String(failure.error), '    '));
-        }
-        console.log('');
-      });
-    }
-  }
-}
-
-export class DotReporter extends BaseReporter {
-  constructor(runner: EventEmitter) {
-    super(runner);
-
-    runner.on('pending', () => {
-      process.stdout.write(colors.yellow('∘'))
-    });
-
-    runner.on('pass', () => {
-      process.stdout.write(colors.green('\u00B7'));
-    });
-
-    runner.on('fail', (test: Test) => {
-      if (test.duration >= test.timeout)
-        process.stdout.write(colors.red('T'));
-      else
-        process.stdout.write(colors.red('F'));
-    });
-
-    runner.once('end', () => {
-      process.stdout.write('\n');
-      this.epilogue();
-    });
-  }
-}
-
-export class ListReporter extends BaseReporter {
-  constructor(runner: EventEmitter) {
-    super(runner);
-
-    runner.on('begin', () => {
-      console.log();
-    });
-
-    runner.on('test', test => {
-      process.stdout.write('    ' + colors.gray(test.fullTitle() + ': '));
-    });
-
-    runner.on('pending', test => {
-      process.stdout.write(colors.green('  - ') + colors.cyan(test.fullTitle()));
-      process.stdout.write('\n');
-    });
-
-    runner.on('pass', test => {
-      process.stdout.write('\u001b[2K\u001b[0G');
-      process.stdout.write(colors.green('  ✓ ') + colors.gray(test.fullTitle()));
-      process.stdout.write('\n');
-    });
-
-    let failure = 0;
-    runner.on('fail', (test: Test) => {
-      process.stdout.write('\u001b[2K\u001b[0G');
-      process.stdout.write(colors.red(`  ${++failure}) ` + test.fullTitle()));
-      process.stdout.write('\n');
-    });
-
-    runner.once('end', () => {
-      process.stdout.write('\n');
-      this.epilogue();
-    });
-  }
-}
-
-export class JSONReporter extends BaseReporter {
-  constructor(runner: EventEmitter) {
-    super(runner);
-
-    runner.once('end', () => {
-      const result = {
-        config: this.config,
-        suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s)
-      };
-      console.log(JSON.stringify(result, undefined, 2));
-    });
-  }
-
-  private _serializeSuite(suite: Suite): any {
-    if (!suite.eachTest(test => true))
-      return null;
-    const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
-    return {
-      title: suite.title,
-      file: suite.file,
-      configuration: suite.configuration,
-      tests: suite.tests.map(test => this._serializeTest(test)),
-      suites: suites.length ? suites : undefined
-    };
-  }
-
-  private _serializeTest(test: Test): any {
-    return {
-      title: test.title,
-      file: test.file,
-      only: test.only,
-      pending: test.pending,
-      slow: test.slow,
-      duration: test.duration,
-      timeout: test.timeout,
-      error: test.error
-    };
-  }
-}
-
-function indent(lines: string, tab: string) {
-  return lines.replace(/^/gm, tab);
-}
-
-function positionInFile(stack: string, file: string): { column: number; line: number; } {
-  for (const line of stack.split('\n')) {
-    const parsed = stackUtils.parseLine(line);
-    if (!parsed)
-      continue;
-    if (path.resolve(process.cwd(), parsed.file) === file)
-      return {column: parsed.column, line: parsed.line};
-  }
-  return null;
-}
-
-export const reporters = {
-  'dot': DotReporter,
-  'list': ListReporter,
-  'json': JSONReporter
-};

test-runner/src/reporters/base.ts

@@ -0,0 +1,128 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { codeFrameColumns } from '@babel/code-frame';
+import colors from 'colors/safe';
+import fs from 'fs';
+import milliseconds from 'ms';
+import os from 'os';
+import path from 'path';
+import StackUtils from 'stack-utils';
+import terminalLink from 'terminal-link';
+import { Reporter } from '../reporter';
+import { RunnerConfig } from '../runnerConfig';
+import { Suite, Test } from '../test';
+
+const stackUtils = new StackUtils()
+
+export class BaseReporter implements Reporter  {
+  pending: Test[] = [];
+  passes: Test[] = [];
+  failures: Test[] = [];
+  duration = 0;
+  startTime: number;
+  config: RunnerConfig;
+  suite: Suite;
+
+  constructor() {
+    process.on('SIGINT', async () => {
+      this.epilogue();
+      process.exit(130);
+    });
+  }
+
+  onBegin(config: RunnerConfig, suite: Suite) {
+    this.startTime = Date.now();
+    this.config = config;
+    this.suite = suite;
+  }
+
+  onTest(test: Test) {
+  }
+
+  onPending(test: Test) {
+    this.pending.push(test);
+  }
+
+  onPass(test: Test) {
+    this.passes.push(test);
+  }
+
+  onFail(test: Test) {
+    this.failures.push(test);
+  }
+
+  onEnd() {
+    this.duration = Date.now() - this.startTime;
+  }
+
+  epilogue() {
+    console.log('');
+
+    console.log(colors.green(`  ${this.passes.length} passing`) + colors.dim(` (${milliseconds(this.duration)})`));  
+
+    if (this.pending.length)
+      console.log(colors.yellow(`  ${this.pending.length} skipped`));
+
+    if (this.failures.length) {  
+      console.log(colors.red(`  ${this.failures.length} failing`));
+      console.log('');
+      this.failures.forEach((failure, index) => {
+        const relativePath = path.relative(process.cwd(), failure.file);
+        const header = `  ${index +1}. ${terminalLink(relativePath, `file://${os.hostname()}${failure.file}`)} › ${failure.title}`;
+        console.log(colors.bold(colors.red(header)));
+        const stack = failure.error.stack;
+        if (stack) {
+          console.log('');
+          const messageLocation = failure.error.stack.indexOf(failure.error.message);
+          const preamble = failure.error.stack.substring(0, messageLocation + failure.error.message.length);
+          console.log(indent(preamble, '    '));
+          const position = positionInFile(stack, failure.file);
+          if (position) {
+            const source = fs.readFileSync(failure.file, 'utf8');
+            console.log('');
+            console.log(indent(codeFrameColumns(source, {
+                start: position,
+              },
+              { highlightCode: true}
+            ), '    '));
+          }
+          console.log('');
+          console.log(indent(colors.dim(stack.substring(preamble.length + 1)), '    '));
+        } else {
+          console.log('');
+          console.log(indent(String(failure.error), '    '));
+        }
+        console.log('');
+      });
+    }
+  }
+}
+
+function indent(lines: string, tab: string) {
+  return lines.replace(/^/gm, tab);
+}
+
+function positionInFile(stack: string, file: string): { column: number; line: number; } {
+  for (const line of stack.split('\n')) {
+    const parsed = stackUtils.parseLine(line);
+    if (!parsed)
+      continue;
+    if (path.resolve(process.cwd(), parsed.file) === file)
+      return {column: parsed.column, line: parsed.line};
+  }
+  return null;
+}

test-runner/src/reporters/dot.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import { BaseReporter } from './base';
+import { Test } from '../test';
+
+export class DotReporter extends BaseReporter {  
+  onPending(test: Test) {
+    super.onPending(test);
+    process.stdout.write(colors.yellow('∘'))
+  }
+  
+  onPass(test: Test) {
+    super.onPass(test);
+    process.stdout.write(colors.green('\u00B7'));
+  }
+  
+  onFail(test: Test) {
+    super.onFail(test);
+    if (test.duration >= test.timeout)
+      process.stdout.write(colors.red('T'));
+    else
+      process.stdout.write(colors.red('F'));
+  } 
+ 
+  onEnd() {
+    super.onEnd();
+    process.stdout.write('\n');
+    this.epilogue();
+  }
+}

test-runner/src/reporters/json.ts

@@ -0,0 +1,55 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { BaseReporter } from './base';
+import { Suite, Test } from '../test';
+
+export class JSONReporter extends BaseReporter {
+  onEnd() {
+    super.onEnd();
+    const result = {
+      config: this.config,
+      suites: this.suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s)
+    };
+    console.log(JSON.stringify(result, undefined, 2));
+  }
+
+  private _serializeSuite(suite: Suite): any {
+    if (!suite.eachTest(test => true))
+      return null;
+    const suites = suite.suites.map(suite => this._serializeSuite(suite)).filter(s => s);
+    return {
+      title: suite.title,
+      file: suite.file,
+      configuration: suite.configuration,
+      tests: suite.tests.map(test => this._serializeTest(test)),
+      suites: suites.length ? suites : undefined
+    };
+  }
+
+  private _serializeTest(test: Test): any {
+    return {
+      title: test.title,
+      file: test.file,
+      only: test.only,
+      pending: test.pending,
+      slow: test.slow,
+      duration: test.duration,
+      timeout: test.timeout,
+      error: test.error
+    };
+  }
+}

test-runner/src/reporters/list.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import colors from 'colors/safe';
+import { BaseReporter } from './base';
+import { RunnerConfig } from '../runnerConfig';
+import { Suite, Test } from '../test';
+
+export class ListReporter extends BaseReporter {
+  _failure = 0;
+
+  onBegin(config: RunnerConfig, suite: Suite) {
+    super.onBegin(config, suite);
+    console.log();
+  }
+
+  onTest(test: Test) {
+    super.onTest(test);
+    process.stdout.write('    ' + colors.gray(test.fullTitle() + ': '));
+  }
+
+  onPending(test: Test) {
+    super.onPending(test);
+    process.stdout.write(colors.green('  - ') + colors.cyan(test.fullTitle()));
+    process.stdout.write('\n');
+}
+
+  onPass(test: Test) {
+    super.onPass(test);
+    process.stdout.write('\u001b[2K\u001b[0G');
+    process.stdout.write(colors.green('  ✓ ') + colors.gray(test.fullTitle()));
+    process.stdout.write('\n');
+}
+
+  onFail(test: Test) {
+    super.onFail(test);
+    process.stdout.write('\u001b[2K\u001b[0G');
+    process.stdout.write(colors.red(`  ${++this._failure}) ` + test.fullTitle()));
+    process.stdout.write('\n');
+}
+
+  onEnd() {
+    super.onEnd();
+    process.stdout.write('\n');
+    this.epilogue();
+  }
+}

test-runner/src/runner.ts

@@ -19,11 +19,12 @@ import crypto from 'crypto';
 import path from 'path';
 import { EventEmitter } from 'events';
 import { lookupRegistrations, FixturePool } from './fixtures';
-import { Suite, Test, Configuration } from './test';
+import { Suite, Test } from './test';
 import { TestRunnerEntry } from './testRunner';
 import { RunnerConfig } from './runnerConfig';
+import { Reporter } from './reporter';
 
-export class Runner extends EventEmitter {
+export class Runner {
   private _workers = new Set<Worker>();
   private _freeWorkers: Worker[] = [];
   private _workerClaimers: (() => void)[] = [];
@@ -34,11 +35,11 @@ export class Runner extends EventEmitter {
   private _stopCallback: () => void;
   readonly _config: RunnerConfig;
   private _suite: Suite;
+  private _reporter: Reporter;
 
-  constructor(suite: Suite, config: RunnerConfig) {
-    super();
-
+  constructor(suite: Suite, config: RunnerConfig, reporter: Reporter) {
     this._config = config;
+    this._reporter = reporter;
     this.stats = {
       duration: 0,
       failures: 0,
@@ -80,12 +81,12 @@ export class Runner extends EventEmitter {
   }
 
   async run() {
-    this.emit('begin', { config: this._config, suite: this._suite });
+    this._reporter.onBegin(this._config, this._suite);
     this._queue = this._filesSortedByWorkerHash();
     // Loop in case job schedules more jobs
     while (this._queue.length)
       await this._dispatchQueue();
-    this.emit('end', {});
+    this._reporter.onEnd();
   }
 
   async _dispatchQueue() {
@@ -146,16 +147,16 @@ export class Runner extends EventEmitter {
     const worker = this._config.debug ? new InProcessWorker(this) : new OopWorker(this);
     worker.on('test', params => {
       ++this.stats.tests;
-      this.emit('test', this._updateTest(params.test));
+      this._reporter.onTest(this._updateTest(params.test));
     });
     worker.on('pending', params => {
       ++this.stats.tests;
       ++this.stats.pending;
-      this.emit('pending', this._updateTest(params.test));
+      this._reporter.onPending(this._updateTest(params.test));
     });
     worker.on('pass', params => {
       ++this.stats.passes;
-      this.emit('pass', this._updateTest(params.test));
+      this._reporter.onPass(this._updateTest(params.test));
     });
     worker.on('fail', params => {
       ++this.stats.failures;
@@ -165,7 +166,7 @@ export class Runner extends EventEmitter {
       const err = worker.takeErr();
       if (err.length)
         params.test.error.stack += '\n\x1b[33mstderr: ' + err.join('\n') + '\x1b[0m';
-      this.emit('fail', this._updateTest(params.test));
+        this._reporter.onFail(this._updateTest(params.test));
     });
     worker.on('exit', () => {
       this._workers.delete(worker);

test-runner/src/spec.ts

@@ -49,7 +49,7 @@ function specBuilder(modifiers, specCallback) {
   return builder({}, null);
 }
 
-export function fixturesUI(suite: Suite, file: string, timeout: number): () => void {
+export function spec(suite: Suite, file: string, timeout: number): () => void {
   const suites = [suite];
   suite.file = file;
 

test-runner/src/testCollector.ts

@@ -17,7 +17,7 @@
 import path from 'path';
 import { fixturesForCallback } from './fixtures';
 import { Test, Suite } from './test';
-import { fixturesUI } from './fixturesUI';
+import { spec } from './spec';
 import { RunnerConfig } from './runnerConfig';
 
 export type Matrix = {
@@ -53,7 +53,7 @@ export class TestCollector {
 
   private _addFile(file: string) {
     const suite = new Suite('');
-    const revertBabelRequire = fixturesUI(suite, file, this._config.timeout);
+    const revertBabelRequire = spec(suite, file, this._config.timeout);
     require(file);
     revertBabelRequire();
     suite._renumber();

test-runner/src/testRunner.ts

@@ -18,7 +18,7 @@ import { FixturePool, rerunRegistrations, setParameters } from './fixtures';
 import { EventEmitter } from 'events';
 import { setCurrentTestFile } from './expect';
 import { Test, Suite, Configuration } from './test';
-import { fixturesUI } from './fixturesUI';
+import { spec } from './spec';
 import { RunnerConfig } from './runnerConfig';
 
 export const fixturePool = new FixturePool<RunnerConfig>();
@@ -67,7 +67,7 @@ export class TestRunner extends EventEmitter {
     setParameters(this._parsedGeneratorConfiguration);
 
     const suite = new Suite('');
-    const revertBabelRequire = fixturesUI(suite, this._file, this._timeout);
+    const revertBabelRequire = spec(suite, this._file, this._timeout);
     require(this._file);
     revertBabelRequire();
     suite._renumber();