chore: experimental test server implementation (#16033)

packages/playwright-core/src/cli/cli.ts

@@ -315,9 +315,7 @@ if (!process.env.PW_LANG_NAME) {
   } catch {}
 
   if (playwrightTestPackagePath) {
-    require(playwrightTestPackagePath).addTestCommand(program);
+    require(playwrightTestPackagePath).addTestCommands(program);
-    require(playwrightTestPackagePath).addShowReportCommand(program);
-    require(playwrightTestPackagePath).addListFilesCommand(program);
   } else {
     {
       const command = program.command('test').allowUnknownOption(true);

packages/playwright-test/src/cli.ts

@@ -27,7 +27,14 @@ import type { FilePatternFilter } from './util';
 import { showHTMLReport } from './reporters/html';
 import { baseFullConfig, defaultTimeout, fileIsModule } from './loader';
 
-export function addTestCommand(program: Command) {
+export function addTestCommands(program: Command) {
+  addTestCommand(program);
+  addShowReportCommand(program);
+  addListFilesCommand(program);
+  addTestServerCommand(program);
+}
+
+function addTestCommand(program: Command) {
   const command = program.command('test [test-filter...]');
   command.description('Run tests with Playwright Test');
   command.option('--browser <browser>', `Browser to use for tests, one of "all", "chromium", "firefox" or "webkit" (default: "chromium")`);
@@ -71,7 +78,7 @@ Examples:
   $ npx playwright test --browser=webkit`);
 }
 
-export function addListFilesCommand(program: Command) {
+function addListFilesCommand(program: Command) {
   const command = program.command('list-files [file-filter...]', { hidden: true });
   command.description('List files with Playwright Test tests');
   command.option('-c, --config <file>', `Configuration file, or a test directory with optional ${kDefaultConfigFiles.map(file => `"${file}"`).join('/')}`);
@@ -86,7 +93,21 @@ export function addListFilesCommand(program: Command) {
   });
 }
 
-export function addShowReportCommand(program: Command) {
+function addTestServerCommand(program: Command) {
+  const command = program.command('test-server', { hidden: true });
+  command.option('-c, --config <file>', `Configuration file, or a test directory with optional ${kDefaultConfigFiles.map(file => `"${file}"`).join('/')}`);
+  command.option('--reporter <reporter>', `Reporter to use, comma-separated, can be ${builtInReporters.map(name => `"${name}"`).join(', ')} (default: "${baseFullConfig.reporter[0]}")`);
+  command.action(async opts => {
+    try {
+      await runTestServer(opts);
+    } catch (e) {
+      console.error(e);
+      process.exit(1);
+    }
+  });
+}
+
+function addShowReportCommand(program: Command) {
   const command = program.command('show-report [report]');
   command.description('show HTML report');
   command.action(report => showHTMLReport(report));
@@ -176,6 +197,25 @@ async function listTestFiles(opts: { [key: string]: any }) {
   });
 }
 
+async function runTestServer(opts: { [key: string]: any }) {
+  const overrides = overridesFromOptions(opts);
+
+  overrides.use = { headless: false };
+  overrides.maxFailures = 1;
+  overrides.timeout = 0;
+  overrides.workers = 1;
+
+  // When no --config option is passed, let's look for the config file in the current directory.
+  const configFileOrDirectory = opts.config ? path.resolve(process.cwd(), opts.config) : process.cwd();
+  const resolvedConfigFile = Runner.resolveConfigFile(configFileOrDirectory)!;
+  if (restartWithExperimentalTsEsm(resolvedConfigFile))
+    return;
+
+  const runner = new Runner(overrides);
+  await runner.loadConfigFromResolvedFile(resolvedConfigFile);
+  await runner.runTestServer();
+}
+
 function forceRegExp(pattern: string): RegExp {
   const match = pattern.match(/^\/(.*)\/([gi]*)$/);
   if (match)

packages/playwright-test/src/dispatcher.ts

@@ -17,11 +17,13 @@
 import child_process from 'child_process';
 import path from 'path';
 import { EventEmitter } from 'events';
-import type { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, StepBeginPayload, StepEndPayload, SerializedLoaderData, TeardownErrorsPayload } from './ipc';
+import type { RunPayload, TestBeginPayload, TestEndPayload, DonePayload, TestOutputPayload, WorkerInitParams, StepBeginPayload, StepEndPayload, SerializedLoaderData, TeardownErrorsPayload, TestServerTestResolvedPayload } from './ipc';
 import type { TestResult, Reporter, TestStep, TestError } from '../types/testReporter';
-import type { Suite, TestCase } from './test';
+import type { Suite } from './test';
 import type { Loader } from './loader';
+import { TestCase } from './test';
 import { ManualPromise } from 'playwright-core/lib/utils/manualPromise';
+import { TestTypeImpl } from './testType';
 
 export type TestGroup = {
   workerHash: string;
@@ -29,6 +31,7 @@ export type TestGroup = {
   repeatEachIndex: number;
   projectId: string;
   tests: TestCase[];
+  testServerTestLine?: number;
 };
 
 type TestResultData = {
@@ -172,6 +175,7 @@ export class Dispatcher {
     let doneCallback = () => {};
     const result = new Promise<void>(f => doneCallback = f);
     const doneWithJob = () => {
+      worker.removeListener('testServer:testResolved', onTestServerTestResolved);
       worker.removeListener('testBegin', onTestBegin);
       worker.removeListener('testEnd', onTestEnd);
       worker.removeListener('stepBegin', onStepBegin);
@@ -184,6 +188,12 @@ export class Dispatcher {
     const remainingByTestId = new Map(testGroup.tests.map(e => [ e.id, e ]));
     const failedTestIds = new Set<string>();
 
+    const onTestServerTestResolved = (params: TestServerTestResolvedPayload) => {
+      const test = new TestCase(params.title, () => {}, new TestTypeImpl([]), params.location);
+      this._testById.set(params.testId, { test, resultByWorkerIndex: new Map() });
+    };
+    worker.addListener('testServer:testResolved', onTestServerTestResolved);
+
     const onTestBegin = (params: TestBeginPayload) => {
       const data = this._testById.get(params.testId)!;
       if (this._hasReachedMaxFailures())
@@ -549,6 +559,7 @@ class Worker extends EventEmitter {
       entries: testGroup.tests.map(test => {
         return { testId: test.id, retry: test.results.length };
       }),
+      testServerTestLine: testGroup.testServerTestLine,
     };
     this.send({ method: 'run', params: runPayload });
   }

packages/playwright-test/src/ipc.ts

@@ -40,6 +40,12 @@ export type WorkerInitParams = {
   stderrParams: TtyParams;
 };
 
+export type TestServerTestResolvedPayload = {
+  testId: string;
+  title: string;
+  location: { file: string, line: number, column: number };
+};
+
 export type TestBeginPayload = {
   testId: string;
   startWallTime: number;  // milliseconds since unix epoch
@@ -83,6 +89,7 @@ export type TestEntry = {
 export type RunPayload = {
   file: string;
   entries: TestEntry[];
+  testServerTestLine?: number;
 };
 
 export type DonePayload = {

packages/playwright-test/src/reporters/multiplexer.ts

@@ -76,6 +76,14 @@ export class Multiplexer implements Reporter {
     for (const reporter of this._reporters)
       (reporter as any).onStepEnd?.(test, result, step);
   }
+
+  _nextTest(): Promise<any | null> {
+    for (const reporter of this._reporters) {
+      if ((reporter as any)._nextTest)
+        return (reporter as any)._nextTest();
+    }
+    return Promise.resolve(null);
+  }
 }
 
 function wrap(callback: () => void) {

packages/playwright-test/src/runner.ts

@@ -45,6 +45,7 @@ import type { TestRunnerPlugin } from './plugins';
 import { setRunnerToAddPluginsTo } from './plugins';
 import { webServerPluginsForConfig } from './plugins/webServerPlugin';
 import { MultiMap } from 'playwright-core/lib/utils/multimap';
+import { createGuid } from 'playwright-core/lib/utils';
 
 const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
@@ -209,6 +210,37 @@ export class Runner {
     return report;
   }
 
+  async runTestServer(): Promise<void> {
+    const config = this._loader.fullConfig();
+    this._reporter = await this._createReporter(false);
+    const rootSuite = new Suite('', 'root');
+    this._reporter.onBegin?.(config, rootSuite);
+    const result: FullResult = { status: 'passed' };
+    const globalTearDown = await this._performGlobalSetup(config, rootSuite, result);
+    if (result.status !== 'passed')
+      return;
+
+    while (true) {
+      const nextTest = await (this._reporter as any)._nextTest!();
+      if (!nextTest)
+        break;
+      const { projectId, file, line } = nextTest;
+      const testGroup: TestGroup = {
+        workerHash: createGuid(), // Create new worker for each test.
+        requireFile: file,
+        repeatEachIndex: 0,
+        projectId,
+        tests: [],
+        testServerTestLine: line,
+      };
+      const dispatcher = new Dispatcher(this._loader, [testGroup], this._reporter);
+      await dispatcher.run();
+    }
+
+    await globalTearDown?.();
+    await this._reporter.onEnd?.(result);
+  }
+
   private async _run(list: boolean, testFileReFilters: FilePatternFilter[], projectNames?: string[]): Promise<FullResult> {
     const filesByProject = await this._collectFiles(testFileReFilters, projectNames);
     return await this._runFiles(list, filesByProject, testFileReFilters);

packages/playwright-test/src/worker.ts

@@ -69,7 +69,7 @@ process.on('message', async message => {
     initConsoleParameters(initParams);
     startProfiling();
     workerRunner = new WorkerRunner(initParams);
-    for (const event of ['testBegin', 'testEnd', 'stepBegin', 'stepEnd', 'done', 'teardownErrors'])
+    for (const event of ['testServer:testResolved', 'testBegin', 'testEnd', 'stepBegin', 'stepEnd', 'done', 'teardownErrors'])
       workerRunner.on(event, sendMessageToParent.bind(null, event));
     return;
   }

packages/playwright-test/src/workerRunner.ts

@@ -18,7 +18,7 @@ import { colors, rimraf } from 'playwright-core/lib/utilsBundle';
 import util from 'util';
 import { EventEmitter } from 'events';
 import { relativeFilePath, serializeError } from './util';
-import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload } from './ipc';
+import type { TestBeginPayload, TestEndPayload, RunPayload, DonePayload, WorkerInitParams, StepBeginPayload, StepEndPayload, TeardownErrorsPayload, TestServerTestResolvedPayload } from './ipc';
 import { setCurrentTestInfo } from './globals';
 import { Loader } from './loader';
 import type { Suite, TestCase } from './test';
@@ -172,6 +172,15 @@ export class WorkerRunner extends EventEmitter {
       await this._loadIfNeeded();
       const fileSuite = await this._loader.loadTestFile(runPayload.file, 'worker');
       const suite = this._loader.buildFileSuiteForProject(this._project, fileSuite, this._params.repeatEachIndex, test => {
+        if (test.location.line === runPayload.testServerTestLine) {
+          const testResolvedPayload: TestServerTestResolvedPayload = {
+            testId: test.id,
+            title: test.title,
+            location: test.location
+          };
+          this.emit('testServer:testResolved', testResolvedPayload);
+          entries.set(test.id, { testId: test.id, retry: 0 });
+        }
         if (!entries.has(test.id))
           return false;
         return true;
@@ -180,7 +189,7 @@ export class WorkerRunner extends EventEmitter {
         this._extraSuiteAnnotations = new Map();
         this._activeSuites = new Set();
         this._didRunFullCleanup = false;
-        const tests = suite.allTests().filter(test => entries.has(test.id));
+        const tests = suite.allTests();
         for (let i = 0; i < tests.length; i++) {
           // Do not run tests after full cleanup, because we are entirely done.
           if (this._isStopped && this._didRunFullCleanup)