fix(test runner): make sure options, trace and screenshot apply to all contexts (#8083)

- Uses some auto fixtures to set default options and instrumentation on BrowserType.
- Moves screenshot, trace and video to worker-scoped fixtures.
- Throws in page/context when used from beforeAll/afterAll.
- Plumbs around BrowserType to be accessible from Browser and BrowserContext.

src/client/browser.ts

@@ -23,12 +23,14 @@ import { BrowserContextOptions } from './types';
 import { isSafeCloseError } from '../utils/errors';
 import * as api from '../../types/types';
 import { CDPSession } from './cdpSession';
+import type { BrowserType } from './browserType';
 
 export class Browser extends ChannelOwner<channels.BrowserChannel, channels.BrowserInitializer> implements api.Browser {
   readonly _contexts = new Set<BrowserContext>();
   private _isConnected = true;
   private _closedPromise: Promise<void>;
   _remoteType: 'owns-connection' | 'uses-connection' | null = null;
+  private _browserType!: BrowserType;
   readonly _name: string;
 
   static from(browser: channels.BrowserChannel): Browser {
@@ -46,13 +48,22 @@ export class Browser extends ChannelOwner<channels.BrowserChannel, channels.Brow
     this._closedPromise = new Promise(f => this.once(Events.Browser.Disconnected, f));
   }
 
+  _setBrowserType(browserType: BrowserType) {
+    this._browserType = browserType;
+    for (const context of this._contexts)
+      context._setBrowserType(browserType);
+  }
+
   async newContext(options: BrowserContextOptions = {}): Promise<BrowserContext> {
     return this._wrapApiCall(async (channel: channels.BrowserChannel) => {
+      options = { ...this._browserType._defaultContextOptions, ...options };
       const contextOptions = await prepareBrowserContextParams(options);
       const context = BrowserContext.from((await channel.newContext(contextOptions)).context);
       context._options = contextOptions;
       this._contexts.add(context);
       context._logger = options.logger || this._logger;
+      context._setBrowserType(this._browserType);
+      await this._browserType._onDidCreateContext?.(context);
       return context;
     });
   }

src/client/browserContext.ts

@@ -33,11 +33,13 @@ import * as api from '../../types/types';
 import * as structs from '../../types/structs';
 import { CDPSession } from './cdpSession';
 import { Tracing } from './tracing';
+import type { BrowserType } from './browserType';
 
 export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel, channels.BrowserContextInitializer> implements api.BrowserContext {
   _pages = new Set<Page>();
   private _routes: { url: URLMatch, handler: network.RouteHandler }[] = [];
   readonly _browser: Browser | null = null;
+  private _browserType: BrowserType | undefined;
   readonly _bindings = new Map<string, (source: structs.BindingSource, ...args: any[]) => any>();
   _timeoutSettings = new TimeoutSettings();
   _ownerPage: Page | undefined;
@@ -89,6 +91,11 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
     this._closedPromise = new Promise(f => this.once(Events.BrowserContext.Close, f));
   }
 
+  _setBrowserType(browserType: BrowserType) {
+    this._browserType = browserType;
+    browserType._contexts.add(this);
+  }
+
   private _onPage(page: Page): void {
     this._pages.add(page);
     this.emit(Events.BrowserContext.Page, page);
@@ -311,12 +318,14 @@ export class BrowserContext extends ChannelOwner<channels.BrowserContextChannel,
   _onClose() {
     if (this._browser)
       this._browser._contexts.delete(this);
+    this._browserType?._contexts?.delete(this);
     this.emit(Events.BrowserContext.Close, this);
   }
 
   async close(): Promise<void> {
     try {
       await this._wrapApiCall(async (channel: channels.BrowserContextChannel) => {
+        await this._browserType?._onWillCloseContext?.(this);
         await channel.close();
         await this._closedPromise;
       });

src/client/browserType.ts

@@ -18,7 +18,7 @@ import * as channels from '../protocol/channels';
 import { Browser } from './browser';
 import { BrowserContext, prepareBrowserContextParams } from './browserContext';
 import { ChannelOwner } from './channelOwner';
-import { LaunchOptions, LaunchServerOptions, ConnectOptions, LaunchPersistentContextOptions } from './types';
+import { LaunchOptions, LaunchServerOptions, ConnectOptions, LaunchPersistentContextOptions, BrowserContextOptions } from './types';
 import WebSocket from 'ws';
 import { Connection } from './connection';
 import { Events } from './events';
@@ -45,6 +45,13 @@ export interface BrowserServer extends api.BrowserServer {
 export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, channels.BrowserTypeInitializer> implements api.BrowserType {
   private _timeoutSettings = new TimeoutSettings();
   _serverLauncher?: BrowserServerLauncher;
+  _contexts = new Set<BrowserContext>();
+
+  // Instrumentation.
+  _defaultContextOptions: BrowserContextOptions = {};
+  _defaultLaunchOptions: LaunchOptions = {};
+  _onDidCreateContext?: (context: BrowserContext) => Promise<void>;
+  _onWillCloseContext?: (context: BrowserContext) => Promise<void>;
 
   static from(browserType: channels.BrowserTypeChannel): BrowserType {
     return (browserType as any)._object;
@@ -69,6 +76,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
     return this._wrapApiCall(async (channel: channels.BrowserTypeChannel) => {
       assert(!(options as any).userDataDir, 'userDataDir option is not supported in `browserType.launch`. Use `browserType.launchPersistentContext` instead');
       assert(!(options as any).port, 'Cannot specify a port without launching as a server.');
+      options = { ...this._defaultLaunchOptions, ...options };
       const launchOptions: channels.BrowserTypeLaunchParams = {
         ...options,
         ignoreDefaultArgs: Array.isArray(options.ignoreDefaultArgs) ? options.ignoreDefaultArgs : undefined,
@@ -77,6 +85,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
       };
       const browser = Browser.from((await channel.launch(launchOptions)).browser);
       browser._logger = logger;
+      browser._setBrowserType(this);
       return browser;
     }, logger);
   }
@@ -90,6 +99,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
   async launchPersistentContext(userDataDir: string, options: LaunchPersistentContextOptions = {}): Promise<BrowserContext> {
     return this._wrapApiCall(async (channel: channels.BrowserTypeChannel) => {
       assert(!(options as any).port, 'Cannot specify a port without launching as a server.');
+      options = { ...this._defaultLaunchOptions, ...this._defaultContextOptions, ...options };
       const contextParams = await prepareBrowserContextParams(options);
       const persistentParams: channels.BrowserTypeLaunchPersistentContextParams = {
         ...contextParams,
@@ -103,6 +113,8 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
       const context = BrowserContext.from(result.context);
       context._options = contextParams;
       context._logger = options.logger;
+      context._setBrowserType(this);
+      await this._onDidCreateContext?.(context);
       return context;
     }, options.logger);
   }
@@ -181,6 +193,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
           const browser = Browser.from(playwright._initializer.preLaunchedBrowser!);
           browser._logger = logger;
           browser._remoteType = 'owns-connection';
+          browser._setBrowserType((playwright as any)[browser._name]);
           const closeListener = () => {
             // Emulate all pages, contexts and the browser closing upon disconnect.
             for (const context of browser.contexts()) {
@@ -252,6 +265,7 @@ export class BrowserType extends ChannelOwner<channels.BrowserTypeChannel, chann
         browser._contexts.add(BrowserContext.from(result.defaultContext));
       browser._remoteType = 'uses-connection';
       browser._logger = logger;
+      browser._setBrowserType(this);
       return browser;
     }, logger);
   }

src/client/channelOwner.ts

@@ -47,7 +47,6 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
     if (this._parent) {
       this._parent._objects.set(guid, this);
       this._logger = this._parent._logger;
-      this._csi = this._parent._csi;
     }
 
     this._channel = this._createChannel(new EventEmitter(), null);
@@ -95,10 +94,15 @@ export abstract class ChannelOwner<T extends channels.Channel = channels.Channel
     const stackTrace = captureStackTrace();
     const { apiName, frameTexts } = stackTrace;
     const channel = this._createChannel({}, stackTrace);
+
+    let ancestorWithCSI: ChannelOwner<any> = this;
+    while (!ancestorWithCSI._csi && ancestorWithCSI._parent)
+      ancestorWithCSI = ancestorWithCSI._parent;
     let csiCallback: ((e?: Error) => void) | undefined;
+
     try {
       logApiCall(logger, `=> ${apiName} started`);
-      csiCallback = this._csi?.onApiCall(apiName);
+      csiCallback = ancestorWithCSI._csi?.onApiCall(apiName);
       const result = await func(channel as any, stackTrace);
       csiCallback?.();
       logApiCall(logger, `<= ${apiName} succeeded`);

src/client/page.ts

@@ -487,9 +487,10 @@ export class Page extends ChannelOwner<channels.PageChannel, channels.PageInitia
   async close(options: { runBeforeUnload?: boolean } = {runBeforeUnload: undefined}) {
     try {
       await this._wrapApiCall(async (channel: channels.PageChannel) => {
-        await channel.close(options);
         if (this._ownedContext)
           await this._ownedContext.close();
+        else
+          await channel.close(options);
       });
     } catch (e) {
       if (isSafeCloseError(e))

src/test/index.ts

@@ -16,27 +16,51 @@
 
 import * as fs from 'fs';
 import * as path from 'path';
-import * as os from 'os';
+import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext, BrowserType } from '../../types/types';
-import type { LaunchOptions, BrowserContextOptions, Page, BrowserContext } from '../../types/types';
 import type { TestType, PlaywrightTestArgs, PlaywrightTestOptions, PlaywrightWorkerArgs, PlaywrightWorkerOptions } from '../../types/test';
 import { rootTestType } from './testType';
 import { createGuid, removeFolders } from '../utils/utils';
 export { expect } from './expect';
 export const _baseTest: TestType<{}, {}> = rootTestType.test;
 
-const artifactsFolder = path.join(os.tmpdir(), 'pwt-' + createGuid());
+type TestFixtures = PlaywrightTestArgs & PlaywrightTestOptions & {
+  _combinedContextOptions: BrowserContextOptions,
+  _setupContextOptionsAndArtifacts: void;
+};
+type WorkerAndFileFixtures = PlaywrightWorkerArgs & PlaywrightWorkerOptions & {
+  _browserType: BrowserType;
+  _artifactsDir: () => string,
+};
 
-export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions, PlaywrightWorkerArgs & PlaywrightWorkerOptions>({
+export const test = _baseTest.extend<TestFixtures, WorkerAndFileFixtures>({
   defaultBrowserType: [ 'chromium', { scope: 'worker' } ],
   browserName: [ ({ defaultBrowserType }, use) => use(defaultBrowserType), { scope: 'worker' } ],
   playwright: [ require('../inprocess'), { scope: 'worker' } ],
   headless: [ undefined, { scope: 'worker' } ],
   channel: [ undefined, { scope: 'worker' } ],
   launchOptions: [ {}, { scope: 'worker' } ],
+  screenshot: [ 'off', { scope: 'worker' } ],
+  video: [ 'off', { scope: 'worker' } ],
+  trace: [ 'off', { scope: 'worker' } ],
 
-  browser: [ async ({ playwright, browserName, headless, channel, launchOptions }, use) => {
+  _artifactsDir: [async ({}, use, workerInfo) => {
+    let dir: string | undefined;
+    await use(() => {
+      if (!dir) {
+        dir = path.join(workerInfo.project.outputDir, '.playwright-artifacts-' + workerInfo.workerIndex);
+        fs.mkdirSync(dir, { recursive: true });
+      }
+      return dir;
+    });
+    if (dir)
+      await removeFolders([dir]);
+  }, { scope: 'worker' }],
+
+  _browserType: [async ({ playwright, browserName, headless, channel, launchOptions }, use) => {
     if (!['chromium', 'firefox', 'webkit'].includes(browserName))
       throw new Error(`Unexpected browserName "${browserName}", must be one of "chromium", "firefox" or "webkit"`);
+    const browserType = playwright[browserName];
+
     const options: LaunchOptions = {
       handleSIGINT: false,
       timeout: 0,
@@ -46,15 +70,18 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
       options.headless = headless;
     if (channel !== undefined)
       options.channel = channel;
-    const browser = await playwright[browserName].launch(options);
+
+    (browserType as any)._defaultLaunchOptions = options;
+    await use(browserType);
+    (browserType as any)._defaultLaunchOptions = undefined;
+  }, { scope: 'worker' }],
+
+  browser: [ async ({ _browserType }, use) => {
+    const browser = await _browserType.launch();
     await use(browser);
     await browser.close();
-    await removeFolders([artifactsFolder]);
   }, { scope: 'worker' } ],
 
-  screenshot: 'off',
-  video: 'off',
-  trace: 'off',
   acceptDownloads: undefined,
   bypassCSP: undefined,
   colorScheme: undefined,
@@ -81,11 +108,7 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
   },
   contextOptions: {},
 
-  createContext: async ({
+  _combinedContextOptions: async ({
-    browser,
-    screenshot,
-    trace,
-    video,
     acceptDownloads,
     bypassCSP,
     colorScheme,
@@ -107,22 +130,7 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
     userAgent,
     baseURL,
     contextOptions,
-    actionTimeout,
+  }, use) => {
-    navigationTimeout
-  }, use, testInfo) => {
-    testInfo.snapshotSuffix = process.platform;
-    if (process.env.PWDEBUG)
-      testInfo.setTimeout(0);
-
-    let videoMode = typeof video === 'string' ? video : video.mode;
-    if (videoMode === 'retry-with-video')
-      videoMode = 'on-first-retry';
-    if (trace === 'retry-with-trace')
-      trace = 'on-first-retry';
-
-    const captureVideo = (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1));
-    const captureTrace = (trace === 'on' || trace === 'retain-on-failure' || (trace === 'on-first-retry' && testInfo.retry === 1));
-
     const options: BrowserContextOptions = {};
     if (acceptDownloads !== undefined)
       options.acceptDownloads = acceptDownloads;
@@ -164,6 +172,124 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
       options.viewport = viewport;
     if (baseURL !== undefined)
       options.baseURL = baseURL;
+    await use({
+      ...contextOptions,
+      ...options,
+    });
+  },
+
+  _setupContextOptionsAndArtifacts: [async ({ _browserType, _combinedContextOptions, _artifactsDir, trace, screenshot, actionTimeout, navigationTimeout }, use, testInfo) => {
+    testInfo.snapshotSuffix = process.platform;
+    if (process.env.PWDEBUG)
+      testInfo.setTimeout(0);
+
+    if (trace === 'retry-with-trace')
+      trace = 'on-first-retry';
+    const captureTrace = (trace === 'on' || trace === 'retain-on-failure' || (trace === 'on-first-retry' && testInfo.retry === 1));
+    const temporaryTraceFiles: string[] = [];
+    const temporaryScreenshots: string[] = [];
+
+    const onDidCreateContext = async (context: BrowserContext) => {
+      context.setDefaultTimeout(actionTimeout || 0);
+      context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0);
+      if (captureTrace)
+        await context.tracing.start({ screenshots: true, snapshots: true });
+      (context as any)._csi = {
+        onApiCall: (name: string) => {
+          return (testInfo as any)._addStep('pw:api', name);
+        },
+      };
+    };
+
+    const onWillCloseContext = async (context: BrowserContext) => {
+      if (captureTrace) {
+        // Export trace for now. We'll know whether we have to preserve it
+        // after the test finishes.
+        const tracePath = path.join(_artifactsDir(), createGuid() + '.zip');
+        temporaryTraceFiles.push(tracePath);
+        await (context.tracing as any)._export({ path: tracePath });
+      }
+      if (screenshot === 'on' || screenshot === 'only-on-failure') {
+        // Capture screenshot for now. We'll know whether we have to preserve them
+        // after the test finishes.
+        await Promise.all(context.pages().map(async page => {
+          const screenshotPath = path.join(_artifactsDir(), createGuid() + '.png');
+          temporaryScreenshots.push(screenshotPath);
+          await page.screenshot({ timeout: 5000, path: screenshotPath }).catch(() => {});
+        }));
+      }
+    };
+
+    // 1. Setup instrumentation and process existing contexts.
+    const oldOnDidCreateContext = (_browserType as any)._onDidCreateContext;
+    (_browserType as any)._onDidCreateContext = onDidCreateContext;
+    (_browserType as any)._onWillCloseContext = onWillCloseContext;
+    (_browserType as any)._defaultContextOptions = _combinedContextOptions;
+    const existingContexts = Array.from((_browserType as any)._contexts) as BrowserContext[];
+    await Promise.all(existingContexts.map(onDidCreateContext));
+
+    // 2. Run the test.
+    await use();
+
+    // 3. Determine whether we need the artifacts.
+    const testFailed = testInfo.status !== testInfo.expectedStatus;
+    const isHook = testInfo.title === 'beforeAll' || testInfo.title === 'afterAll';
+    const preserveTrace = captureTrace && !isHook && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1));
+    const captureScreenshots = !isHook && (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed));
+
+    const traceAttachments: string[] = [];
+    const addTraceAttachment = () => {
+      const tracePath = testInfo.outputPath(`trace${traceAttachments.length ? '-' + traceAttachments.length : ''}.zip`);
+      traceAttachments.push(tracePath);
+      testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
+      return tracePath;
+    };
+
+    const screenshotAttachments: string[] = [];
+    const addScreenshotAttachment = () => {
+      const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${screenshotAttachments.length + 1}.png`);
+      screenshotAttachments.push(screenshotPath);
+      testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' });
+      return screenshotPath;
+    };
+
+    // 4. Cleanup instrumentation.
+    const leftoverContexts = Array.from((_browserType as any)._contexts) as BrowserContext[];
+    (_browserType as any)._onDidCreateContext = oldOnDidCreateContext;
+    (_browserType as any)._onWillCloseContext = undefined;
+    (_browserType as any)._defaultContextOptions = undefined;
+    leftoverContexts.forEach(context => (context as any)._csi = undefined);
+
+    // 5. Collect artifacts from any non-closed contexts.
+    await Promise.all(leftoverContexts.map(async context => {
+      if (preserveTrace)
+        await (context.tracing as any)._export({ path: addTraceAttachment() });
+      if (captureScreenshots)
+        await Promise.all(context.pages().map(page => page.screenshot({ timeout: 5000, path: addScreenshotAttachment() }).catch(() => {})));
+    }));
+
+    // 6. Either remove or attach temporary traces and screenshots for contexts closed
+    // before the test has finished.
+    await Promise.all(temporaryTraceFiles.map(async file => {
+      if (preserveTrace)
+        await fs.promises.rename(file, addTraceAttachment()).catch(() => {});
+      else
+        await fs.promises.unlink(file).catch(() => {});
+    }));
+    await Promise.all(temporaryScreenshots.map(async file => {
+      if (captureScreenshots)
+        await fs.promises.rename(file, addScreenshotAttachment()).catch(() => {});
+      else
+        await fs.promises.unlink(file).catch(() => {});
+    }));
+  }, { auto: true }],
+
+  createContext: async ({ browser, video, _artifactsDir }, use, testInfo) => {
+    let videoMode = typeof video === 'string' ? video : video.mode;
+    if (videoMode === 'retry-with-video')
+      videoMode = 'on-first-retry';
+
+    const captureVideo = (videoMode === 'on' || videoMode === 'retain-on-failure' || (videoMode === 'on-first-retry' && testInfo.retry === 1));
 
     const allContexts: BrowserContext[] = [];
     const allPages: Page[] = [];
@@ -171,63 +297,20 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
     await use(async (additionalOptions = {}) => {
       let recordVideoDir: string | null = null;
       const recordVideoSize = typeof video === 'string' ? undefined : video.size;
-      if (captureVideo) {
+      if (captureVideo)
-        await fs.promises.mkdir(artifactsFolder, { recursive: true });
+        recordVideoDir = _artifactsDir();
-        recordVideoDir = artifactsFolder;
-      }
 
       const combinedOptions: BrowserContextOptions = {
         recordVideo: recordVideoDir ? { dir: recordVideoDir, size: recordVideoSize } : undefined,
-        ...contextOptions,
-        ...options,
         ...additionalOptions,
       };
       const context = await browser.newContext(combinedOptions);
-      (context as any)._csi = {
-        onApiCall: (name: string) => {
-          return (testInfo as any)._addStep('pw:api', name);
-        },
-      };
-      context.setDefaultTimeout(actionTimeout || 0);
-      context.setDefaultNavigationTimeout(navigationTimeout || actionTimeout || 0);
       context.on('page', page => allPages.push(page));
 
-      if (captureTrace) {
-        const name = path.relative(testInfo.project.outputDir, testInfo.outputDir).replace(/[\/\\]/g, '-');
-        const suffix = allContexts.length ? '-' + allContexts.length : '';
-        await context.tracing.start({ name: name + suffix, screenshots: true, snapshots: true });
-      }
-
       allContexts.push(context);
       return context;
     });
 
-    const testFailed = testInfo.status !== testInfo.expectedStatus;
-
-    await Promise.all(allContexts.map(async (context, contextIndex) => {
-      const preserveTrace = captureTrace && (trace === 'on' || (testFailed && trace === 'retain-on-failure') || (trace === 'on-first-retry' && testInfo.retry === 1));
-      if (preserveTrace) {
-        const suffix = contextIndex ? '-' + contextIndex : '';
-        const tracePath = testInfo.outputPath(`trace${suffix}.zip`);
-        await context.tracing.stop({ path: tracePath });
-        testInfo.attachments.push({ name: 'trace', path: tracePath, contentType: 'application/zip' });
-      } else if (captureTrace) {
-        await context.tracing.stop();
-      }
-    }));
-
-    const captureScreenshots = (screenshot === 'on' || (screenshot === 'only-on-failure' && testFailed));
-    if (captureScreenshots) {
-      await Promise.all(allPages.map(async (page, index) => {
-        const screenshotPath = testInfo.outputPath(`test-${testFailed ? 'failed' : 'finished'}-${index + 1}.png`);
-        try {
-          await page.screenshot({ timeout: 5000, path: screenshotPath });
-          testInfo.attachments.push({ name: 'screenshot', path: screenshotPath, contentType: 'image/png' });
-        } catch {
-        }
-      }));
-    }
-
     const prependToError = (testInfo.status === 'timedOut' && allContexts.length) ?
       formatPendingCalls((allContexts[0] as any)._connection.pendingProtocolCalls()) : '';
     await Promise.all(allContexts.map(context => context.close()));
@@ -241,6 +324,7 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
       }
     }
 
+    const testFailed = testInfo.status !== testInfo.expectedStatus;
     const preserveVideo = captureVideo && (videoMode === 'on' || (testFailed && videoMode === 'retain-on-failure') || (videoMode === 'on-first-retry' && testInfo.retry === 1));
     if (preserveVideo) {
       await Promise.all(allPages.map(async page => {
@@ -259,7 +343,9 @@ export const test = _baseTest.extend<PlaywrightTestArgs & PlaywrightTestOptions,
     }
   },
 
-  context: async ({ createContext }, use) => {
+  context: async ({ createContext }, use, testInfo) => {
+    if (testInfo.title === 'beforeAll' || testInfo.title === 'afterAll')
+      throw new Error(`"context" and "page" fixtures are not suppoted in ${testInfo.title}. Use browser.newContext() instead.`);
     await use(await createContext());
   },
 

tests/playwright-test/golden.spec.ts

@@ -19,11 +19,23 @@ import * as fs from 'fs';
 import * as path from 'path';
 import { test, expect, stripAscii } from './playwright-test-fixtures';
 
+const files = {
+  'helper.ts': `
+    export const test = pwt.test.extend({
+      auto: [ async ({}, run, testInfo) => {
+        testInfo.snapshotSuffix = '';
+        await run();
+      }, { auto: true } ]
+    });
+  `
+};
+
 test('should support golden', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot('snapshot.txt');
       });
@@ -34,6 +46,7 @@ test('should support golden', async ({runInlineTest}) => {
 
 test('should fail on wrong golden', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': `Line1
 Line2
 Line3
@@ -42,7 +55,7 @@ Line5
 Line6
 Line7`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         const data = [];
         data.push('Line1');
@@ -67,9 +80,10 @@ Line7`,
 
 test('should write detailed failure result to an output folder', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world updated').toMatchSnapshot('snapshot.txt');
       });
@@ -89,9 +103,10 @@ test('should write detailed failure result to an output folder', async ({runInli
 
 test("doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher", async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world updated').not.toMatchSnapshot('snapshot.txt');
       });
@@ -110,9 +125,10 @@ test("doesn\'t create comparison artifacts in an output folder for passed negate
 
 test('should pass on different snapshots with negate matcher', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world updated').not.toMatchSnapshot('snapshot.txt');
       });
@@ -124,9 +140,10 @@ test('should pass on different snapshots with negate matcher', async ({runInline
 
 test('should fail on same snapshots with negate matcher', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').not.toMatchSnapshot('snapshot.txt');
       });
@@ -140,8 +157,9 @@ test('should fail on same snapshots with negate matcher', async ({runInlineTest}
 
 test('should write missing expectations locally', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot('snapshot.txt');
       });
@@ -157,8 +175,9 @@ test('should write missing expectations locally', async ({runInlineTest}, testIn
 
 test('shouldn\'t write missing expectations locally for negated matcher', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').not.toMatchSnapshot('snapshot.txt');
       });
@@ -175,9 +194,10 @@ test('should update snapshot with the update-snapshots flag', async ({runInlineT
   const EXPECTED_SNAPSHOT = 'Hello world';
   const ACTUAL_SNAPSHOT = 'Hello world updated';
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt');
       });
@@ -195,9 +215,10 @@ test('shouldn\'t update snapshot with the update-snapshots flag for negated matc
   const EXPECTED_SNAPSHOT = 'Hello world';
   const ACTUAL_SNAPSHOT = 'Hello world updated';
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.txt': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('${ACTUAL_SNAPSHOT}').not.toMatchSnapshot('snapshot.txt');
       });
@@ -213,8 +234,9 @@ test('shouldn\'t update snapshot with the update-snapshots flag for negated matc
 test('should silently write missing expectations locally with the update-snapshots flag', async ({runInlineTest}, testInfo) => {
   const ACTUAL_SNAPSHOT = 'Hello world new';
   const result = await runInlineTest({
+    ...files,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('${ACTUAL_SNAPSHOT}').toMatchSnapshot('snapshot.txt');
       });
@@ -230,8 +252,9 @@ test('should silently write missing expectations locally with the update-snapsho
 
 test('should silently write missing expectations locally with the update-snapshots flag for negated matcher', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').not.toMatchSnapshot('snapshot.txt');
       });
@@ -246,11 +269,12 @@ test('should silently write missing expectations locally with the update-snapsho
 
 test('should match multiple snapshots', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot1.txt': `Snapshot1`,
     'a.spec.js-snapshots/snapshot2.txt': `Snapshot2`,
     'a.spec.js-snapshots/snapshot3.txt': `Snapshot3`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Snapshot1').toMatchSnapshot('snapshot1.txt');
         expect('Snapshot2').toMatchSnapshot('snapshot2.txt');
@@ -263,6 +287,7 @@ test('should match multiple snapshots', async ({runInlineTest}) => {
 
 test('should match snapshots from multiple projects', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'playwright.config.ts': `
       import * as path from 'path';
       module.exports = { projects: [
@@ -271,14 +296,14 @@ test('should match snapshots from multiple projects', async ({runInlineTest}) =>
       ]};
     `,
     'p1/a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('../helper');
       test('is a test', ({}) => {
         expect('Snapshot1').toMatchSnapshot('snapshot.txt');
       });
     `,
     'p1/a.spec.js-snapshots/snapshot.txt': `Snapshot1`,
     'p2/a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('../helper');
       test('is a test', ({}) => {
         expect('Snapshot2').toMatchSnapshot('snapshot.txt');
       });
@@ -290,9 +315,10 @@ test('should match snapshots from multiple projects', async ({runInlineTest}) =>
 
 test('should use provided name', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/provided.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot('provided.txt');
       });
@@ -303,8 +329,9 @@ test('should use provided name', async ({runInlineTest}) => {
 
 test('should throw without a name', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot();
       });
@@ -316,9 +343,10 @@ test('should throw without a name', async ({runInlineTest}) => {
 
 test('should use provided name via options', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/provided.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot({ name: 'provided.txt' });
       });
@@ -329,9 +357,10 @@ test('should use provided name via options', async ({runInlineTest}) => {
 
 test('should compare binary', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.dat': Buffer.from([1,2,3,4]),
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect(Buffer.from([1,2,3,4])).toMatchSnapshot('snapshot.dat');
       });
@@ -342,10 +371,11 @@ test('should compare binary', async ({runInlineTest}) => {
 
 test('should compare PNG images', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.png':
         Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'),
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64')).toMatchSnapshot('snapshot.png');
       });
@@ -356,10 +386,11 @@ test('should compare PNG images', async ({runInlineTest}) => {
 
 test('should compare different PNG images', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.png':
         Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+P+/HgAFhAJ/wlseKgAAAABJRU5ErkJggg==', 'base64'),
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect(Buffer.from('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII==', 'base64')).toMatchSnapshot('snapshot.png');
       });
@@ -384,10 +415,11 @@ test('should respect threshold', async ({runInlineTest}) => {
   const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png'));
   const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png'));
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/snapshot.png': expected,
     'a.spec.js-snapshots/snapshot2.png': expected,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 });
         expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png', { threshold: 0.2 });
@@ -403,6 +435,7 @@ test('should respect project threshold', async ({runInlineTest}) => {
   const expected = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-expected.png'));
   const actual = fs.readFileSync(path.join(__dirname, 'assets/screenshot-canvas-actual.png'));
   const result = await runInlineTest({
+    ...files,
     'playwright.config.ts': `
       module.exports = { projects: [
         { expect: { toMatchSnapshot: { threshold: 0.2 } } },
@@ -411,7 +444,7 @@ test('should respect project threshold', async ({runInlineTest}) => {
     'a.spec.js-snapshots/snapshot.png': expected,
     'a.spec.js-snapshots/snapshot2.png': expected,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect(Buffer.from('${actual.toString('base64')}', 'base64')).toMatchSnapshot('snapshot.png', { threshold: 0.3 });
         expect(Buffer.from('${actual.toString('base64')}', 'base64')).not.toMatchSnapshot('snapshot.png');
@@ -425,9 +458,10 @@ test('should respect project threshold', async ({runInlineTest}) => {
 
 test('should sanitize snapshot name', async ({runInlineTest}) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js-snapshots/-snapshot-.txt': `Hello world`,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot('../../snapshot!.txt');
       });
@@ -438,8 +472,9 @@ test('should sanitize snapshot name', async ({runInlineTest}) => {
 
 test('should write missing expectations with sanitized snapshot name', async ({runInlineTest}, testInfo) => {
   const result = await runInlineTest({
+    ...files,
     'a.spec.js': `
-      const { test } = pwt;
+      const { test } = require('./helper');
       test('is a test', ({}) => {
         expect('Hello world').toMatchSnapshot('../../snapshot!.txt');
       });

tests/playwright-test/playwright.artifacts.spec.ts

@@ -0,0 +1,280 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect } from './playwright-test-fixtures';
+import fs from 'fs';
+import path from 'path';
+
+function listFiles(dir: string): string[] {
+  const result: string[] = [];
+  const entries = fs.readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name));
+  for (const entry of entries) {
+    result.push(entry.name);
+    if (entry.isDirectory())
+      result.push(...listFiles(path.join(dir, entry.name)).map(x => '  ' + x));
+  }
+  return result;
+}
+
+const testFiles = {
+  'artifacts.spec.ts': `
+    import fs from 'fs';
+    import os from 'os';
+    import path from 'path';
+    import rimraf from 'rimraf';
+
+    const { test } = pwt;
+
+    test.describe('shared', () => {
+      let page;
+      test.beforeAll(async ({ browser }) => {
+        page = await browser.newPage({});
+        await page.setContent('<button>Click me</button><button>And me</button>');
+      });
+
+      test.afterAll(async () => {
+        await page.close();
+      });
+
+      test('shared passing', async ({ }) => {
+        await page.click('text=Click me');
+      });
+
+      test('shared  failing', async ({ }) => {
+        await page.click('text=And me');
+        expect(1).toBe(2);
+      });
+    });
+
+    test('passing', async ({ page }) => {
+      await page.setContent('I am the page');
+    });
+
+    test('two contexts', async ({ page, createContext }) => {
+      await page.setContent('I am the page');
+
+      const context2 = await createContext();
+      const page2 = await context2.newPage();
+      await page2.setContent('I am the page');
+    });
+
+    test('failing', async ({ page }) => {
+      await page.setContent('I am the page');
+      expect(1).toBe(2);
+    });
+
+    test('two contexts failing', async ({ page, createContext }) => {
+      await page.setContent('I am the page');
+
+      const context2 = await createContext();
+      const page2 = await context2.newPage();
+      await page2.setContent('I am the page');
+
+      expect(1).toBe(2);
+    });
+
+    test('own context passing', async ({ browser }) => {
+      const page = await browser.newPage();
+      await page.setContent('<button>Click me</button><button>And me</button>');
+      await page.click('text=Click me');
+      await page.close();
+    });
+
+    test('own context failing', async ({ browser }) => {
+      const page = await browser.newPage();
+      await page.setContent('<button>Click me</button><button>And me</button>');
+      await page.click('text=Click me');
+      await page.close();
+      expect(1).toBe(2);
+    });
+
+    const testPersistent = test.extend({
+      page: async ({ playwright, browserName }, use) => {
+        const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'user-data-dir-'));
+        const context = await playwright[browserName].launchPersistentContext(dir);
+        await use(context.pages()[0]);
+        await context.close();
+        rimraf.sync(dir);
+      },
+    });
+
+    testPersistent('persistent passing', async ({ page }) => {
+      await page.setContent('<button>Click me</button><button>And me</button>');
+    });
+
+    testPersistent('persistent failing', async ({ page }) => {
+      await page.setContent('<button>Click me</button><button>And me</button>');
+      expect(1).toBe(2);
+    });
+  `,
+};
+
+test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { screenshot: 'on' } };
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    'artifacts-failing',
+    '  test-failed-1.png',
+    'artifacts-own-context-failing',
+    '  test-failed-1.png',
+    'artifacts-own-context-passing',
+    '  test-finished-1.png',
+    'artifacts-passing',
+    '  test-finished-1.png',
+    'artifacts-persistent-failing',
+    '  test-failed-1.png',
+    'artifacts-persistent-passing',
+    '  test-finished-1.png',
+    'artifacts-shared-failing',
+    '  test-failed-1.png',
+    'artifacts-shared-passing',
+    '  test-finished-1.png',
+    'artifacts-two-contexts',
+    '  test-finished-1.png',
+    '  test-finished-2.png',
+    'artifacts-two-contexts-failing',
+    '  test-failed-1.png',
+    '  test-failed-2.png',
+    'report.json',
+  ]);
+});
+
+test('should work with screenshot: only-on-failure', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { screenshot: 'only-on-failure' } };
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    'artifacts-failing',
+    '  test-failed-1.png',
+    'artifacts-own-context-failing',
+    '  test-failed-1.png',
+    'artifacts-persistent-failing',
+    '  test-failed-1.png',
+    'artifacts-shared-failing',
+    '  test-failed-1.png',
+    'artifacts-two-contexts-failing',
+    '  test-failed-1.png',
+    '  test-failed-2.png',
+    'report.json',
+  ]);
+});
+
+test('should work with trace: on', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { trace: 'on' } };
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    'artifacts-failing',
+    '  trace.zip',
+    'artifacts-own-context-failing',
+    '  trace.zip',
+    'artifacts-own-context-passing',
+    '  trace.zip',
+    'artifacts-passing',
+    '  trace.zip',
+    'artifacts-persistent-failing',
+    '  trace.zip',
+    'artifacts-persistent-passing',
+    '  trace.zip',
+    'artifacts-shared-failing',
+    '  trace.zip',
+    'artifacts-shared-passing',
+    '  trace.zip',
+    'artifacts-two-contexts',
+    '  trace-1.zip',
+    '  trace.zip',
+    'artifacts-two-contexts-failing',
+    '  trace-1.zip',
+    '  trace.zip',
+    'report.json',
+  ]);
+});
+
+test('should work with trace: retain-on-failure', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { trace: 'retain-on-failure' } };
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    'artifacts-failing',
+    '  trace.zip',
+    'artifacts-own-context-failing',
+    '  trace.zip',
+    'artifacts-persistent-failing',
+    '  trace.zip',
+    'artifacts-shared-failing',
+    '  trace.zip',
+    'artifacts-two-contexts-failing',
+    '  trace-1.zip',
+    '  trace.zip',
+    'report.json',
+  ]);
+});
+
+test('should work with trace: on-first-retry', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    ...testFiles,
+    'playwright.config.ts': `
+      module.exports = { use: { trace: 'on-first-retry' } };
+    `,
+  }, { workers: 1, retries: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(5);
+  expect(result.failed).toBe(5);
+  expect(listFiles(testInfo.outputPath('test-results'))).toEqual([
+    'artifacts-failing-retry1',
+    '  trace.zip',
+    'artifacts-own-context-failing-retry1',
+    '  trace.zip',
+    'artifacts-persistent-failing-retry1',
+    '  trace.zip',
+    'artifacts-shared-failing-retry1',
+    '  trace.zip',
+    'artifacts-two-contexts-failing-retry1',
+    '  trace-1.zip',
+    '  trace.zip',
+    'report.json',
+  ]);
+});

tests/playwright-test/playwright.spec.ts

@@ -161,6 +161,65 @@ test('should override use:browserName with --browser', async ({ runInlineTest })
   ]);
 });
 
+test('should respect context options in various contexts', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { use: { viewport: { width: 500, height: 500 } } };
+    `,
+    'a.test.ts': `
+      import fs from 'fs';
+      import os from 'os';
+      import path from 'path';
+      import rimraf from 'rimraf';
+
+      const { test } = pwt;
+      test.use({ locale: 'fr-CH' });
+
+      let context;
+      test.beforeAll(async ({ browser }) => {
+        context = await browser.newContext();
+      });
+
+      test.afterAll(async () => {
+        await context.close();
+      });
+
+      test('shared context', async ({}) => {
+        const page = await context.newPage();
+        expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
+        expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
+      });
+
+      test('own context', async ({ browser }) => {
+        const page = await browser.newPage();
+        expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
+        expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
+        await page.close();
+      });
+
+      test('default context', async ({ page }) => {
+        expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
+        expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
+      });
+
+      test('persistent context', async ({ playwright, browserName }) => {
+        const dir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'user-data-dir-'));
+        const context = await playwright[browserName].launchPersistentContext(dir);
+        const page = context.pages()[0];
+
+        expect(page.viewportSize()).toEqual({ width: 500, height: 500 });
+        expect(await page.evaluate(() => navigator.language)).toBe('fr-CH');
+
+        await context.close();
+        rimraf.sync(dir);
+      });
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(4);
+});
+
 test('should report error and pending operations on timeout', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     'a.test.ts': `
@@ -185,6 +244,22 @@ test('should report error and pending operations on timeout', async ({ runInline
   expect(stripAscii(result.output)).toContain(`10 |           page.textContent('text=More missing'),`);
 });
 
+test('should throw when using page in beforeAll', async ({ runInlineTest }, testInfo) => {
+  const result = await runInlineTest({
+    'a.test.ts': `
+      const { test } = pwt;
+      test.beforeAll(async ({ page }) => {
+      });
+      test('ok', async ({ page }) => {
+      });
+    `,
+  }, { workers: 1 });
+
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(0);
+  expect(result.output).toContain(`Error: "context" and "page" fixtures are not suppoted in beforeAll. Use browser.newContext() instead.`);
+});
+
 test('should report click error on sigint', async ({ runInlineTest }) => {
   test.skip(process.platform === 'win32', 'No sending SIGINT on Windows');
 
@@ -208,37 +283,6 @@ test('should report click error on sigint', async ({ runInlineTest }) => {
   expect(stripAscii(result.output)).toContain(`8 |         const promise = page.click('text=Missing');`);
 });
 
-test('should work with screenshot: only-on-failure', async ({ runInlineTest }, testInfo) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-      module.exports = { use: { screenshot: 'only-on-failure' }, name: 'chromium' };
-    `,
-    'a.test.ts': `
-      const { test } = pwt;
-      test('pass', async ({ page }) => {
-        await page.setContent('<div>PASS</div>');
-        test.expect(1 + 1).toBe(2);
-      });
-      test('fail', async ({ page }) => {
-        await page.setContent('<div>FAIL</div>');
-        const page2 = await page.context().newPage();
-        await page2.setContent('<div>FAIL</div>');
-        test.expect(1 + 1).toBe(1);
-      });
-    `,
-  }, { workers: 1 });
-
-  expect(result.exitCode).toBe(1);
-  expect(result.passed).toBe(1);
-  expect(result.failed).toBe(1);
-  const screenshotPass = testInfo.outputPath('test-results', 'a-pass-chromium', 'test-failed-1.png');
-  const screenshotFail1 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-1.png');
-  const screenshotFail2 = testInfo.outputPath('test-results', 'a-fail-chromium', 'test-failed-2.png');
-  expect(fs.existsSync(screenshotPass)).toBe(false);
-  expect(fs.existsSync(screenshotFail1)).toBe(true);
-  expect(fs.existsSync(screenshotFail2)).toBe(true);
-});
-
 test('should work with video: retain-on-failure', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
     'playwright.config.ts': `
@@ -339,36 +383,3 @@ test('should work with video size', async ({ runInlineTest }, testInfo) => {
   expect(videoPlayer.videoWidth).toBe(220);
   expect(videoPlayer.videoHeight).toBe(110);
 });
-
-test('should work with multiple contexts and trace: on', async ({ runInlineTest }, testInfo) => {
-  const result = await runInlineTest({
-    'playwright.config.ts': `
-      module.exports = { use: { trace: 'on' } };
-    `,
-    'a.test.ts': `
-      const { test } = pwt;
-      test('pass', async ({ page, createContext }) => {
-        await page.setContent('<div>PASS</div>');
-
-        const context1 = await createContext();
-        const page1 = await context1.newPage();
-        await page1.setContent('<div>PASS</div>');
-
-        const context2 = await createContext({ locale: 'en-US' });
-        const page2 = await context2.newPage();
-        await page2.setContent('<div>PASS</div>');
-
-        test.expect(1 + 1).toBe(2);
-      });
-    `,
-  }, { workers: 1 });
-
-  expect(result.exitCode).toBe(0);
-  expect(result.passed).toBe(1);
-  const traceDefault = testInfo.outputPath('test-results', 'a-pass', 'trace.zip');
-  const trace1 = testInfo.outputPath('test-results', 'a-pass', 'trace-1.zip');
-  const trace2 = testInfo.outputPath('test-results', 'a-pass', 'trace-2.zip');
-  expect(fs.existsSync(traceDefault)).toBe(true);
-  expect(fs.existsSync(trace1)).toBe(true);
-  expect(fs.existsSync(trace2)).toBe(true);
-});

tests/playwright-test/reporter.spec.ts

@@ -114,7 +114,6 @@ test('should work with custom reporter', async ({ runInlineTest }) => {
   ]);
 });
 
-
 test('should work without a file extension', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'reporter.ts': smallReporterJS,
@@ -162,6 +161,9 @@ test('should load reporter from node_modules', async ({ runInlineTest }) => {
 test('should report expect steps', async ({ runInlineTest }) => {
   const expectReporterJS = `
     class Reporter {
+      onStdOut(chunk) {
+        process.stdout.write(chunk);
+      }
       onStepBegin(test, result, step) {
         const copy = { ...step, startTime: undefined, duration: undefined };
         console.log('%%%% begin', JSON.stringify(copy));
@@ -232,6 +234,15 @@ test('should report expect steps', async ({ runInlineTest }) => {
 test('should report api steps', async ({ runInlineTest }) => {
   const expectReporterJS = `
     class Reporter {
+      onStdOut(chunk) {
+        process.stdout.write(chunk);
+      }
+      onTestBegin(test) {
+        console.log('%%%% test begin ' + test.title);
+      }
+      onTestEnd(test) {
+        console.log('%%%% test end ' + test.title);
+      }
       onStepBegin(test, result, step) {
         const copy = { ...step, startTime: undefined, duration: undefined };
         console.log('%%%% begin', JSON.stringify(copy));
@@ -259,11 +270,31 @@ test('should report api steps', async ({ runInlineTest }) => {
         await page.setContent('<button></button>');
         await page.click('button');
       });
+
+      test.describe('suite', () => {
+        let myPage;
+        test.beforeAll(async ({ browser }) => {
+          myPage = await browser.newPage();
+          await myPage.setContent('<button></button>');
+        });
+
+        test('pass1', async () => {
+          await myPage.click('button');
+        });
+        test('pass2', async () => {
+          await myPage.click('button');
+        });
+
+        test.afterAll(async () => {
+          await myPage.close();
+        });
+      });
     `
   }, { reporter: '', workers: 1 });
 
   expect(result.exitCode).toBe(0);
   expect(result.output.split('\n').filter(line => line.startsWith('%%')).map(stripEscapedAscii)).toEqual([
+    `%%%% test begin pass`,
     `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
     `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
     `%% begin {\"title\":\"browserContext.newPage\",\"category\":\"pw:api\"}`,
@@ -276,6 +307,23 @@ test('should report api steps', async ({ runInlineTest }) => {
     `%% begin {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
     `%% end {\"title\":\"browserContext.close\",\"category\":\"pw:api\"}`,
     `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%%%% test end pass`,
+    `%%%% test begin pass1`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%%%% test end pass1`,
+    `%%%% test begin pass2`,
+    `%% begin {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"Before Hooks\",\"category\":\"hook\"}`,
+    `%% begin {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% end {\"title\":\"page.click\",\"category\":\"pw:api\"}`,
+    `%% begin {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%% end {\"title\":\"After Hooks\",\"category\":\"hook\"}`,
+    `%%%% test end pass2`,
   ]);
 });
 
@@ -283,6 +331,9 @@ test('should report api steps', async ({ runInlineTest }) => {
 test('should report api step failure', async ({ runInlineTest }) => {
   const expectReporterJS = `
     class Reporter {
+      onStdOut(chunk) {
+        process.stdout.write(chunk);
+      }
       onStepBegin(test, result, step) {
         const copy = { ...step, startTime: undefined, duration: undefined };
         console.log('%%%% begin', JSON.stringify(copy));
@@ -333,6 +384,9 @@ test('should report api step failure', async ({ runInlineTest }) => {
 test('should report test.step', async ({ runInlineTest }) => {
   const expectReporterJS = `
     class Reporter {
+      onStdOut(chunk) {
+        process.stdout.write(chunk);
+      }
       onStepBegin(test, result, step) {
         const copy = { ...step, startTime: undefined, duration: undefined };
         console.log('%%%% begin', JSON.stringify(copy));

tests/playwright-test/retry.spec.ts

@@ -68,7 +68,7 @@ test('should retry timeout', async ({ runInlineTest }) => {
         await new Promise(f => setTimeout(f, 10000));
       });
     `
-  }, { timeout: 100, retries: 2 });
+  }, { timeout: 1000, retries: 2 });
   expect(exitCode).toBe(1);
   expect(passed).toBe(0);
   expect(failed).toBe(1);

tests/playwright-test/test-output-dir.spec.ts

@@ -88,6 +88,7 @@ test('should include the project name', async ({ runInlineTest }) => {
     'helper.ts': `
       export const test = pwt.test.extend({
         auto: [ async ({}, run, testInfo) => {
+          testInfo.snapshotSuffix = '';
           await run();
         }, { auto: true } ]
       });

types/test.d.ts

@@ -2297,6 +2297,35 @@ export interface PlaywrightWorkerOptions {
    * [fixtures.channel](https://playwright.dev/docs/api/class-fixtures#fixtures-channel) take priority over this.
    */
   launchOptions: LaunchOptions;
+  /**
+   * Whether to automatically capture a screenshot after each test. Defaults to `'off'`.
+   * - `'off'`: Do not capture screenshots.
+   * - `'on'`: Capture screenshot after each test.
+   * - `'only-on-failure'`: Capture screenshot after each test failure.
+   *
+   * Learn more about [automatic screenshots](https://playwright.dev/docs/test-configuration#automatic-screenshots).
+   */
+  screenshot: 'off' | 'on' | 'only-on-failure';
+  /**
+   * Whether to record a trace for each test. Defaults to `'off'`.
+   * - `'off'`: Do not record a trace.
+   * - `'on'`: Record a trace for each test.
+   * - `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs.
+   * - `'on-first-retry'`: Record a trace only when retrying a test for the first time.
+   *
+   * Learn more about [recording trace](https://playwright.dev/docs/test-configuration#record-test-trace).
+   */
+  trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
+  /**
+   * Whether to record video for each test. Defaults to `'off'`.
+   * - `'off'`: Do not record video.
+   * - `'on'`: Record video for each test.
+   * - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
+   * - `'on-first-retry'`: Record video only when retrying a test for the first time.
+   *
+   * Learn more about [recording video](https://playwright.dev/docs/test-configuration#record-video).
+   */
+  video: VideoMode | { mode: VideoMode, size: ViewportSize };
 }
 
 export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video';
@@ -2390,35 +2419,6 @@ export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' |
  *
  */
 export interface PlaywrightTestOptions {
-  /**
-   * Whether to automatically capture a screenshot after each test. Defaults to `'off'`.
-   * - `'off'`: Do not capture screenshots.
-   * - `'on'`: Capture screenshot after each test.
-   * - `'only-on-failure'`: Capture screenshot after each test failure.
-   *
-   * Learn more about [automatic screenshots](https://playwright.dev/docs/test-configuration#automatic-screenshots).
-   */
-  screenshot: 'off' | 'on' | 'only-on-failure';
-  /**
-   * Whether to record a trace for each test. Defaults to `'off'`.
-   * - `'off'`: Do not record a trace.
-   * - `'on'`: Record a trace for each test.
-   * - `'retain-on-failure'`: Record a trace for each test, but remove it from successful test runs.
-   * - `'on-first-retry'`: Record a trace only when retrying a test for the first time.
-   *
-   * Learn more about [recording trace](https://playwright.dev/docs/test-configuration#record-test-trace).
-   */
-  trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
-  /**
-   * Whether to record video for each test. Defaults to `'off'`.
-   * - `'off'`: Do not record video.
-   * - `'on'`: Record video for each test.
-   * - `'retain-on-failure'`: Record video for each test, but remove all videos from successful test runs.
-   * - `'on-first-retry'`: Record video only when retrying a test for the first time.
-   *
-   * Learn more about [recording video](https://playwright.dev/docs/test-configuration#record-video).
-   */
-  video: VideoMode | { mode: VideoMode, size: ViewportSize };
   /**
    * Whether to automatically download all the attachments. Defaults to `false` where all the downloads are canceled.
    */

utils/generate_types/overrides-test.d.ts

@@ -284,14 +284,14 @@ export interface PlaywrightWorkerOptions {
   headless: boolean | undefined;
   channel: BrowserChannel | undefined;
   launchOptions: LaunchOptions;
+  screenshot: 'off' | 'on' | 'only-on-failure';
+  trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
+  video: VideoMode | { mode: VideoMode, size: ViewportSize };
 }
 
 export type VideoMode = 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-video';
 
 export interface PlaywrightTestOptions {
-  screenshot: 'off' | 'on' | 'only-on-failure';
-  trace: 'off' | 'on' | 'retain-on-failure' | 'on-first-retry' | /** deprecated */ 'retry-with-trace';
-  video: VideoMode | { mode: VideoMode, size: ViewportSize };
   acceptDownloads: boolean | undefined;
   bypassCSP: boolean | undefined;
   colorScheme: ColorScheme | undefined;