chore: support esm loader on Node 20 (#23269)

.github/workflows/tests_primary.yml

@@ -151,6 +151,12 @@ jobs:
           - os: ubuntu-latest
             node-version: 18
             shard: 2/2
+          - os: ubuntu-latest
+            node-version: 20
+            shard: 1/2
+          - os: ubuntu-latest
+            node-version: 20
+            shard: 2/2
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3

packages/playwright-test/package.json

@@ -20,8 +20,8 @@
     "./lib/cli": "./lib/cli.js",
     "./lib/transform/babelBundle": "./lib/transform/babelBundle.js",
     "./lib/transform/compilationCache": "./lib/transform/compilationCache.js",
+    "./lib/transform/esmLoader": "./lib/transform/esmLoader.js",
     "./lib/internalsForTest": "./lib/internalsForTest.js",
-    "./lib/experimentalLoader": "./lib/experimentalLoader.js",
     "./lib/plugins": "./lib/plugins/index.js",
     "./jsx-runtime": {
       "import": "./jsx-runtime.mjs",

packages/playwright-test/src/DEPS.list

@@ -9,8 +9,5 @@ common/
 [index.ts]
 @testIsomorphic/**
 
-[experimentalLoader.ts]
-./transform/
-
 [internalsForTest.ts]
 **

packages/playwright-test/src/common/configLoader.ts

@@ -23,6 +23,8 @@ import type { Config, Project } from '../../types/test';
 import { errorWithFile } from '../util';
 import { setCurrentConfig } from './globals';
 import { FullConfigInternal } from './config';
+import { addToCompilationCache } from '../transform/compilationCache';
+import { initializeEsmLoader } from './esmLoaderHost';
 
 const kDefineConfigWasUsed = Symbol('defineConfigWasUsed');
 export const defineConfig = (config: any) => {
@@ -39,8 +41,11 @@ export class ConfigLoader {
   }
 
   static async deserialize(data: SerializedConfig): Promise<FullConfigInternal> {
-    const loader = new ConfigLoader(data.configCLIOverrides);
     setBabelPlugins(data.babelTransformPlugins);
+    addToCompilationCache(data.compilationCache);
+    await initializeEsmLoader();
+
+    const loader = new ConfigLoader(data.configCLIOverrides);
 
     if (data.configFile)
       return await loader.loadConfigFile(data.configFile);

packages/playwright-test/src/common/esmLoaderHost.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { addToCompilationCache, serializeCompilationCache } from '../transform/compilationCache';
+import { getBabelPlugins } from '../transform/transform';
+import { PortTransport } from '../transform/portTransport';
+
+const port = (globalThis as any).__esmLoaderPort;
+
+const loaderChannel = port ? new PortTransport(port, async (method, params) => {
+  if (method === 'pushToCompilationCache')
+    addToCompilationCache(params.cache);
+}) : undefined;
+
+export async function startCollectingFileDeps() {
+  if (!loaderChannel)
+    return;
+  await loaderChannel.send('startCollectingFileDeps', {});
+}
+
+export async function stopCollectingFileDeps(file: string) {
+  if (!loaderChannel)
+    return;
+  await loaderChannel.send('stopCollectingFileDeps', { file });
+}
+
+export async function incorporateCompilationCache() {
+  if (!loaderChannel)
+    return;
+  const result = await loaderChannel.send('getCompilationCache', {});
+  addToCompilationCache(result.cache);
+}
+
+export async function initializeEsmLoader() {
+  if (!loaderChannel)
+    return;
+  await loaderChannel.send('setBabelPlugins', { plugins: getBabelPlugins() });
+  await loaderChannel.send('addToCompilationCache', { cache: serializeCompilationCache() });
+}

packages/playwright-test/src/common/testLoader.ts

@@ -22,6 +22,7 @@ import { Suite } from './test';
 import { requireOrImport } from '../transform/transform';
 import { filterStackTrace } from '../util';
 import { startCollectingFileDeps, stopCollectingFileDeps } from '../transform/compilationCache';
+import * as esmLoaderHost from './esmLoaderHost';
 
 export const defaultTimeout = 30000;
 
@@ -37,8 +38,10 @@ export async function loadTestFile(file: string, rootDir: string, testErrors?: T
   suite.location = { file, line: 0, column: 0 };
 
   setCurrentlyLoadingFileSuite(suite);
-  if (!isWorkerProcess())
+  if (!isWorkerProcess()) {
     startCollectingFileDeps();
+    await esmLoaderHost.startCollectingFileDeps();
+  }
   try {
     await requireOrImport(file);
     cachedFileSuites.set(file, suite);
@@ -47,8 +50,11 @@ export async function loadTestFile(file: string, rootDir: string, testErrors?: T
       throw e;
     testErrors.push(serializeLoadError(file, e));
   } finally {
-    stopCollectingFileDeps(file);
     setCurrentlyLoadingFileSuite(undefined);
+    if (!isWorkerProcess()) {
+      stopCollectingFileDeps(file);
+      await esmLoaderHost.stopCollectingFileDeps(file);
+    }
   }
 
   {

packages/playwright-test/src/loader/loaderMain.ts

@@ -20,8 +20,9 @@ import { ProcessRunner } from '../common/process';
 import type { FullConfigInternal } from '../common/config';
 import { loadTestFile } from '../common/testLoader';
 import type { TestError } from '../../reporter';
-import { addToCompilationCache, serializeCompilationCache } from '../transform/compilationCache';
+import { serializeCompilationCache } from '../transform/compilationCache';
 import { PoolBuilder } from '../common/poolBuilder';
+import { incorporateCompilationCache } from '../common/esmLoaderHost';
 
 export class LoaderMain extends ProcessRunner {
   private _serializedConfig: SerializedConfig;
@@ -30,7 +31,6 @@ export class LoaderMain extends ProcessRunner {
 
   constructor(serializedConfig: SerializedConfig) {
     super();
-    addToCompilationCache(serializedConfig.compilationCache);
     this._serializedConfig = serializedConfig;
   }
 
@@ -48,7 +48,8 @@ export class LoaderMain extends ProcessRunner {
     return { fileSuite: fileSuite._deepSerialize(), testErrors };
   }
 
-  async serializeCompilationCache() {
+  async getCompilationCacheFromLoader() {
+    await incorporateCompilationCache();
     return serializeCompilationCache();
   }
 }

packages/playwright-test/src/runner/loadUtils.ts

@@ -95,6 +95,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |
   // Load test files.
   const fileSuiteByFile = new Map<string, Suite>();
   const loaderHost = mode === 'out-of-process' ? new OutOfProcessLoaderHost(config) : new InProcessLoaderHost(config);
+  await loaderHost.start();
   for (const file of allTestFiles) {
     const fileSuite = await loaderHost.loadTestFile(file, errors);
     fileSuiteByFile.set(file, fileSuite);

packages/playwright-test/src/runner/loaderHost.ts

@@ -23,6 +23,7 @@ import type { FullConfigInternal } from '../common/config';
 import { PoolBuilder } from '../common/poolBuilder';
 import { addToCompilationCache } from '../transform/compilationCache';
 import { setBabelPlugins } from '../transform/transform';
+import { incorporateCompilationCache, initializeEsmLoader } from '../common/esmLoaderHost';
 
 export class InProcessLoaderHost {
   private _config: FullConfigInternal;
@@ -31,10 +32,14 @@ export class InProcessLoaderHost {
   constructor(config: FullConfigInternal) {
     this._config = config;
     this._poolBuilder = PoolBuilder.createForLoader();
+  }
+
+  async start() {
     const babelTransformPlugins: [string, any?][] = [];
-    for (const plugin of config.plugins)
+    for (const plugin of this._config.plugins)
       babelTransformPlugins.push(...plugin.babelPlugins || []);
     setBabelPlugins(babelTransformPlugins);
+    await initializeEsmLoader();
   }
 
   async loadTestFile(file: string, testErrors: TestError[]): Promise<Suite> {
@@ -43,28 +48,32 @@ export class InProcessLoaderHost {
     return result;
   }
 
-  async stop() {}
+  async stop() {
+    await incorporateCompilationCache();
+  }
 }
 
 export class OutOfProcessLoaderHost {
-  private _startPromise: Promise<void>;
+  private _config: FullConfigInternal;
   private _processHost: ProcessHost;
 
   constructor(config: FullConfigInternal) {
+    this._config = config;
     this._processHost = new ProcessHost(require.resolve('../loader/loaderMain.js'), 'loader', {});
-    this._startPromise = this._processHost.startRunner(serializeConfig(config), true);
+  }
+
+  async start() {
+    await this._processHost.startRunner(serializeConfig(this._config), true);
   }
 
   async loadTestFile(file: string, testErrors: TestError[]): Promise<Suite> {
-    await this._startPromise;
     const result = await this._processHost.sendMessage({ method: 'loadTestFile', params: { file } }) as any;
     testErrors.push(...result.testErrors);
     return Suite._deepParse(result.fileSuite);
   }
 
   async stop() {
-    await this._startPromise;
+    const result = await this._processHost.sendMessage({ method: 'getCompilationCacheFromLoader' }) as any;
-    const result = await this._processHost.sendMessage({ method: 'serializeCompilationCache' }) as any;
     addToCompilationCache(result);
     await this._processHost.stop();
   }

packages/playwright-test/src/transform/esmLoader.ts

@@ -16,8 +16,9 @@
 
 import fs from 'fs';
 import url from 'url';
-import { belongsToNodeModules, currentFileDepsCollector } from './transform/compilationCache';
+import { addToCompilationCache, belongsToNodeModules, currentFileDepsCollector, serializeCompilationCache, startCollectingFileDeps, stopCollectingFileDeps } from './compilationCache';
-import { transformHook, resolveHook } from './transform/transform';
+import { transformHook, resolveHook, setBabelPlugins } from './transform';
+import { PortTransport } from './portTransport';
 
 // Node < 18.6: defaultResolve takes 3 arguments.
 // Node >= 18.6: nextResolve from the chain takes 2 arguments.
@@ -54,9 +55,46 @@ async function load(moduleUrl: string, context: { format?: string }, defaultLoad
 
   const code = fs.readFileSync(filename, 'utf-8');
   const source = transformHook(code, filename, moduleUrl);
+
+  // Flush the source maps to the main thread.
+  await transport?.send('pushToCompilationCache', { cache: serializeCompilationCache() });
+
   // Output format is always the same as input format, if it was unknown, we always report modules.
-  // shortCurcuit is required by Node >= 18.6 to designate no more loaders should be called.
+  // shortCircuit is required by Node >= 18.6 to designate no more loaders should be called.
   return { format: context.format || 'module', source, shortCircuit: true };
 }
 
-module.exports = { resolve, load };
+let transport: PortTransport | undefined;
+
+function globalPreload(context: { port: MessagePort }) {
+  transport = new PortTransport(context.port, async (method, params) => {
+    if (method === 'setBabelPlugins') {
+      setBabelPlugins(params.plugins);
+      return;
+    }
+
+    if (method === 'addToCompilationCache') {
+      addToCompilationCache(params.cache);
+      return;
+    }
+
+    if (method === 'getCompilationCache')
+      return { cache: serializeCompilationCache() };
+
+    if (method === 'startCollectingFileDeps') {
+      startCollectingFileDeps();
+      return;
+    }
+
+    if (method === 'stopCollectingFileDeps') {
+      stopCollectingFileDeps(params.file);
+      return;
+    }
+  });
+
+  return `
+    globalThis.__esmLoaderPort = port;
+  `;
+}
+
+module.exports = { resolve, load, globalPreload };

packages/playwright-test/src/transform/portTransport.ts

@@ -0,0 +1,49 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export class PortTransport {
+  private _lastId = 0;
+  private _port: MessagePort;
+  private _callbacks = new Map<number, (result: any) => void>();
+
+  constructor(port: MessagePort, handler: (method: string, params: any) => Promise<any>) {
+    this._port = port;
+    port.onmessage = async event => {
+      const message = event.data;
+      const { id, ackId, method, params, result } = message;
+      if (id) {
+        const result = await handler(method, params);
+        this._port.postMessage({ ackId: id, result });
+        return;
+      }
+
+      if (ackId) {
+        const callback = this._callbacks.get(ackId);
+        this._callbacks.delete(ackId);
+        callback?.(result);
+        return;
+      }
+    };
+  }
+
+  async send(method: string, params: any) {
+    return await new Promise<any>(f => {
+      const id = ++this._lastId;
+      this._callbacks.set(id, f);
+      this._port.postMessage({ id, method, params });
+    });
+  }
+}

packages/playwright-test/src/transform/transform.ts

@@ -41,6 +41,10 @@ export function setBabelPlugins(plugins: BabelPlugin[]) {
   babelPlugins = plugins;
 }
 
+export function getBabelPlugins(): BabelPlugin[] {
+  return babelPlugins;
+}
+
 function validateTsConfig(tsconfig: TsConfigLoaderResult): ParsedTsConfigData | undefined {
   if (!tsconfig.tsConfigPath || !tsconfig.baseUrl)
     return;

packages/playwright-test/src/util.ts

@@ -300,7 +300,7 @@ function folderIsModule(folder: string): boolean {
 }
 
 export function experimentalLoaderOption() {
-  return ` --no-warnings --experimental-loader=${url.pathToFileURL(require.resolve('@playwright/test/lib/experimentalLoader')).toString()}`;
+  return ` --no-warnings --experimental-loader=${url.pathToFileURL(require.resolve('@playwright/test/lib/transform/esmLoader')).toString()}`;
 }
 
 export function envWithoutExperimentalLoaderOptions(): NodeJS.ProcessEnv {

packages/playwright-test/src/worker/workerMain.ts

@@ -30,7 +30,6 @@ import { ProcessRunner } from '../common/process';
 import { loadTestFile } from '../common/testLoader';
 import { buildFileSuiteForProject, filterTestsRemoveEmptySuites } from '../common/suiteUtils';
 import { PoolBuilder } from '../common/poolBuilder';
-import { addToCompilationCache } from '../transform/compilationCache';
 import type { TestInfoError } from '../../types/test';
 
 const removeFolderAsync = util.promisify(rimraf);
@@ -67,7 +66,6 @@ export class WorkerMain extends ProcessRunner {
     process.env.TEST_WORKER_INDEX = String(params.workerIndex);
     process.env.TEST_PARALLEL_INDEX = String(params.parallelIndex);
     setIsWorkerProcess();
-    addToCompilationCache(params.config.compilationCache);
 
     this._params = params;
     this._fixtureRunner = new FixtureRunner();