feat: support screenshotsDir option (#12642)

The `screenshotsDir` option controls the expectation storage
for `toHaveScreenshot()` function.

The new expectation management for screenshots has the following
key properties:
- All screenshots are stored in a single folder called `screenshotsDir`.
- Screenshot names **do not** respect `snapshotDir` and `snapshotSuffix`
  configurations.
- `screenshotsDir` is configurable per project. This way a "smoke tests"
  project can re-use screenshots from "all tests" project.
- Host platform is a top-level folder.

For example, given the following config:

```js
// playwright.config.ts
module.exports = {
  projects: [
    { name: 'Mobile Safari' },
    { name: 'Desktop Chrome' },
  ],
};
```

And the following test structure:

```
smoke-tests/
└── basic.spec.ts
```

Will result in the following screenshots folder structure by default:

```
__screenshots__/
└── darwin/
    ├── Mobile Safari/
    │   └── smoke-tests/
    │       └── basic.spec.ts/
    │           └── screenshot-expectation.png
    └── Desktop Chrome/
        └── smoke-tests/
            └── basic.spec.ts/
                └── screenshot-expectation.png
```

docs/src/test-api/class-testconfig.md

@@ -298,10 +298,46 @@ test('example test', async ({}, testInfo) => {
 });
 ```
 
+## property: TestConfig.screenshotsDir
+- type: <[string]>
+
+The base directory, relative to the config file, for screenshot files created with `toHaveScreenshot`. Defaults to
+
+```
+<directory-of-configuration-file>/__screenshots__/<platform name>/<project name>
+```
+
+This path will serve as the base directory for each test file screenshot directory. For example, the following test structure:
+
+```
+smoke-tests/
+└── basic.spec.ts
+```
+
+will result in the following screenshots folder structure:
+
+```
+__screenshots__/
+└── darwin/
+    ├── Mobile Safari/
+    │   └── smoke-tests/
+    │       └── basic.spec.ts/
+    │           └── screenshot-expectation.png
+    └── Desktop Chrome/
+        └── smoke-tests/
+            └── basic.spec.ts/
+                └── screenshot-expectation.png
+```
+
+where:
+* `darwin/` - a platform name folder
+* `Mobile Safari` and `Desktop Chrome` - project names
+
+
 ## property: TestConfig.snapshotDir
 - type: <[string]>
 
-The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and `toHaveScreenshot`. Defaults to [`property: TestConfig.testDir`].
+The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to [`property: TestConfig.testDir`].
 
 The directory for each test can be accessed by [`property: TestInfo.snapshotDir`] and [`method: TestInfo.snapshotPath`].
 

docs/src/test-api/class-testinfo.md

@@ -382,7 +382,7 @@ The name of the snapshot or the path segments to define the snapshot file path.
 ## property: TestInfo.snapshotSuffix
 - type: <[string]>
 
-Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case both `expect(value).toMatchSnapshot(snapshotName)` and `expect(page).toHaveScreenshot(snapshotName)` will use different snapshots depending on the platform. Learn more about [snapshots](./test-snapshots.md).
+Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case `expect(value).toMatchSnapshot(snapshotName)` will use different snapshots depending on the platform. Learn more about [snapshots](./test-snapshots.md).
 
 ## property: TestInfo.status
 - type: <[void]|[TestStatus]<"passed"|"failed"|"timedOut"|"skipped">>

docs/src/test-api/class-testproject.md

@@ -150,10 +150,45 @@ Any JSON-serializable metadata that will be put directly to the test report.
 
 Project name is visible in the report and during test execution.
 
+## property: TestProject.screenshotsDir
+- type: <[string]>
+
+The base directory, relative to the config file, for screenshot files created with `toHaveScreenshot`. Defaults to
+
+```
+<directory-of-configuration-file>/__screenshots__/<platform name>/<project name>
+```
+
+This path will serve as the base directory for each test file screenshot directory. For example, the following test structure:
+
+```
+smoke-tests/
+└── basic.spec.ts
+```
+
+will result in the following screenshots folder structure:
+
+```
+__screenshots__/
+└── darwin/
+    ├── Mobile Safari/
+    │   └── smoke-tests/
+    │       └── basic.spec.ts/
+    │           └── screenshot-expectation.png
+    └── Desktop Chrome/
+        └── smoke-tests/
+            └── basic.spec.ts/
+                └── screenshot-expectation.png
+```
+
+where:
+* `darwin/` - a platform name folder
+* `Mobile Safari` and `Desktop Chrome` - project names
+
 ## property: TestProject.snapshotDir
 - type: <[string]>
 
-The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and `toHaveScreenshot`. Defaults to [`property: TestProject.testDir`].
+The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to [`property: TestProject.testDir`].
 
 The directory for each test can be accessed by [`property: TestInfo.snapshotDir`] and [`method: TestInfo.snapshotPath`].
 

packages/playwright-test/src/loader.ts

@@ -202,6 +202,10 @@ export class Loader {
     let snapshotDir = takeFirst(this._configOverrides.snapshotDir, projectConfig.snapshotDir, this._config.snapshotDir, testDir);
     if (!path.isAbsolute(snapshotDir))
       snapshotDir = path.resolve(configDir, snapshotDir);
+    const name = takeFirst(this._configOverrides.name, projectConfig.name, this._config.name, '');
+    let screenshotsDir = takeFirst(this._configOverrides.screenshotsDir, projectConfig.screenshotsDir, this._config.screenshotsDir, path.join(rootDir, '__screenshots__', process.platform, name));
+    if (!path.isAbsolute(screenshotsDir))
+      screenshotsDir = path.resolve(configDir, screenshotsDir);
     const fullProject: FullProject = {
       fullyParallel: takeFirst(this._configOverrides.fullyParallel, projectConfig.fullyParallel, this._config.fullyParallel, undefined),
       expect: takeFirst(this._configOverrides.expect, projectConfig.expect, this._config.expect, undefined),
@@ -211,9 +215,10 @@ export class Loader {
       repeatEach: takeFirst(this._configOverrides.repeatEach, projectConfig.repeatEach, this._config.repeatEach, 1),
       retries: takeFirst(this._configOverrides.retries, projectConfig.retries, this._config.retries, 0),
       metadata: takeFirst(this._configOverrides.metadata, projectConfig.metadata, this._config.metadata, undefined),
-      name: takeFirst(this._configOverrides.name, projectConfig.name, this._config.name, ''),
+      name,
       testDir,
       snapshotDir,
+      screenshotsDir,
       testIgnore: takeFirst(this._configOverrides.testIgnore, projectConfig.testIgnore, this._config.testIgnore, []),
       testMatch: takeFirst(this._configOverrides.testMatch, projectConfig.testMatch, this._config.testMatch, '**/?(*.)@(spec|test).*'),
       timeout: takeFirst(this._configOverrides.timeout, projectConfig.timeout, this._config.timeout, 10000),

packages/playwright-test/src/matchers/toMatchSnapshot.ts

@@ -44,9 +44,18 @@ export function getSnapshotName(
   nameOrOptions: NameOrSegments | { name?: NameOrSegments } & MatchSnapshotOptions = {},
   optOptions: MatchSnapshotOptions = {}
 ) {
-  const anonymousSnapshotExtension = typeof received === 'string' || Buffer.isBuffer(received) ? determineFileExtension(received) : 'png';
+  const [
+    anonymousSnapshotExtension,
+    snapshotPathResolver,
+  ] = typeof received === 'string' || Buffer.isBuffer(received) ? [
+    determineFileExtension(received),
+    testInfo.snapshotPath.bind(testInfo),
+  ] : [
+    'png',
+    testInfo._screenshotPath.bind(testInfo),
+  ];
   const helper = new SnapshotHelper(
-      testInfo, anonymousSnapshotExtension, {},
+      testInfo, snapshotPathResolver, anonymousSnapshotExtension, {},
       nameOrOptions, optOptions, true /* dryRun */);
   return path.basename(helper.snapshotPath);
 }
@@ -65,6 +74,7 @@ class SnapshotHelper<T extends ImageComparatorOptions> {
 
   constructor(
     testInfo: TestInfoImpl,
+    snapshotPathResolver: (...pathSegments: string[]) => string,
     anonymousSnapshotExtension: string,
     configOptions: ImageComparatorOptions,
     nameOrOptions: NameOrSegments | { name?: NameOrSegments } & T,
@@ -105,7 +115,7 @@ class SnapshotHelper<T extends ImageComparatorOptions> {
 
     // sanitizes path if string
     const pathSegments = Array.isArray(name) ? name : [addSuffixToFilePath(name, '', undefined, true)];
-    const snapshotPath = testInfo.snapshotPath(...pathSegments);
+    const snapshotPath = snapshotPathResolver(...pathSegments);
     const outputFile = testInfo.outputPath(...pathSegments);
     const expectedPath = addSuffixToFilePath(outputFile, '-expected');
     const actualPath = addSuffixToFilePath(outputFile, '-actual');
@@ -234,7 +244,7 @@ export function toMatchSnapshot(
   if (!testInfo)
     throw new Error(`toMatchSnapshot() must be called during the test`);
   const helper = new SnapshotHelper(
-      testInfo, determineFileExtension(received),
+      testInfo, testInfo.snapshotPath.bind(testInfo), determineFileExtension(received),
       testInfo.project.expect?.toMatchSnapshot || {},
       nameOrOptions, optOptions);
   const comparator: Comparator = mimeTypeToComparator[helper.mimeType];
@@ -282,7 +292,7 @@ export async function toHaveScreenshot(
   if (!testInfo)
     throw new Error(`toHaveScreenshot() must be called during the test`);
   const helper = new SnapshotHelper(
-      testInfo, 'png',
+      testInfo, testInfo._screenshotPath.bind(testInfo), 'png',
       testInfo.project.expect?.toHaveScreenshot || {},
       nameOrOptions, optOptions);
   const [page, locator] = pageOrLocator.constructor.name === 'Page' ? [(pageOrLocator as PageEx), undefined] : [(pageOrLocator as Locator).page() as PageEx, pageOrLocator as LocatorEx];

packages/playwright-test/src/testInfo.ts

@@ -46,6 +46,7 @@ export class TestInfoImpl implements TestInfo {
   private _currentRunnable: RunnableDescription = { type: 'test' };
   // Holds elapsed time of the "time pool" shared between fixtures, each hooks and test itself.
   private _elapsedTestTime = 0;
+  readonly _screenshotsDir: string;
 
   // ------------ TestInfo fields ------------
   readonly repeatEachIndex: number;
@@ -142,6 +143,10 @@ export class TestInfoImpl implements TestInfo {
       const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile);
       return path.join(this.project.snapshotDir, relativeTestFilePath + '-snapshots');
     })();
+    this._screenshotsDir = (() => {
+      const relativeTestFilePath = path.relative(this.project.testDir, test._requireFile);
+      return path.join(this.project.screenshotsDir, relativeTestFilePath);
+    })();
   }
 
   private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
@@ -278,6 +283,14 @@ export class TestInfoImpl implements TestInfo {
     throw new Error(`The snapshotPath is not allowed outside of the parent directory. Please fix the defined path.\n\n\tsnapshotPath: ${subPath}`);
   }
 
+  _screenshotPath(...pathSegments: string[]) {
+    const subPath = path.join(...pathSegments);
+    const screenshotPath = getContainedPath(this._screenshotsDir, subPath);
+    if (screenshotPath)
+      return screenshotPath;
+    throw new Error(`Screenshot name "${subPath}" should not point outside of the parent directory.`);
+  }
+
   skip(...args: [arg?: any, description?: string]) {
     this._modifier('skip', args);
   }

packages/playwright-test/types/test.d.ts

@@ -162,8 +162,7 @@ interface TestProject {
    */
   name?: string;
   /**
-   * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and
+   * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to
-   * `toHaveScreenshot`. Defaults to
    * [testProject.testDir](https://playwright.dev/docs/api/class-testproject#test-project-test-dir).
    *
    * The directory for each test can be accessed by
@@ -175,6 +174,41 @@ interface TestProject {
    * resolve to `snapshots/a.spec.js-snapshots`.
    */
   snapshotDir?: string;
+  /**
+   * The base directory, relative to the config file, for screenshot files created with `toHaveScreenshot`. Defaults to
+   *
+   * ```
+   * <directory-of-configuration-file>/__screenshots__/<platform name>/<project name>
+   * ```
+   *
+   * This path will serve as the base directory for each test file screenshot directory. For example, the following test
+   * structure:
+   *
+   * ```
+   * smoke-tests/
+   * └── basic.spec.ts
+   * ```
+   *
+   * will result in the following screenshots folder structure:
+   *
+   * ```
+   * __screenshots__/
+   * └── darwin/
+   *     ├── Mobile Safari/
+   *     │   └── smoke-tests/
+   *     │       └── basic.spec.ts/
+   *     │           └── screenshot-expectation.png
+   *     └── Desktop Chrome/
+   *         └── smoke-tests/
+   *             └── basic.spec.ts/
+   *                 └── screenshot-expectation.png
+   * ```
+   *
+   * where:
+   * - `darwin/` - a platform name folder
+   * - `Mobile Safari` and `Desktop Chrome` - project names
+   */
+  screenshotsDir?: string;
   /**
    * The output directory for files created during test execution. Defaults to `test-results`.
    *
@@ -717,8 +751,7 @@ interface TestConfig {
   metadata?: any;
   name?: string;
   /**
-   * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot` and
+   * The base directory, relative to the config file, for snapshot files created with `toMatchSnapshot`. Defaults to
-   * `toHaveScreenshot`. Defaults to
    * [testConfig.testDir](https://playwright.dev/docs/api/class-testconfig#test-config-test-dir).
    *
    * The directory for each test can be accessed by
@@ -730,6 +763,41 @@ interface TestConfig {
    * resolve to `snapshots/a.spec.js-snapshots`.
    */
   snapshotDir?: string;
+  /**
+   * The base directory, relative to the config file, for screenshot files created with `toHaveScreenshot`. Defaults to
+   *
+   * ```
+   * <directory-of-configuration-file>/__screenshots__/<platform name>/<project name>
+   * ```
+   *
+   * This path will serve as the base directory for each test file screenshot directory. For example, the following test
+   * structure:
+   *
+   * ```
+   * smoke-tests/
+   * └── basic.spec.ts
+   * ```
+   *
+   * will result in the following screenshots folder structure:
+   *
+   * ```
+   * __screenshots__/
+   * └── darwin/
+   *     ├── Mobile Safari/
+   *     │   └── smoke-tests/
+   *     │       └── basic.spec.ts/
+   *     │           └── screenshot-expectation.png
+   *     └── Desktop Chrome/
+   *         └── smoke-tests/
+   *             └── basic.spec.ts/
+   *                 └── screenshot-expectation.png
+   * ```
+   *
+   * where:
+   * - `darwin/` - a platform name folder
+   * - `Mobile Safari` and `Desktop Chrome` - project names
+   */
+  screenshotsDir?: string;
   /**
    * The output directory for files created during test execution. Defaults to `test-results`.
    *
@@ -1571,9 +1639,9 @@ export interface TestInfo {
   stderr: (string | Buffer)[];
   /**
    * Suffix used to differentiate snapshots between multiple test configurations. For example, if snapshots depend on the
-   * platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case both
+   * platform, you can set `testInfo.snapshotSuffix` equal to `process.platform`. In this case
-   * `expect(value).toMatchSnapshot(snapshotName)` and `expect(page).toHaveScreenshot(snapshotName)` will use different
+   * `expect(value).toMatchSnapshot(snapshotName)` will use different snapshots depending on the platform. Learn more about
-   * snapshots depending on the platform. Learn more about [snapshots](https://playwright.dev/docs/test-snapshots).
+   * [snapshots](https://playwright.dev/docs/test-snapshots).
    */
   snapshotSuffix: string;
   /**

tests/playwright-test/to-have-screenshot.spec.ts

@@ -32,24 +32,11 @@ const redImage = createImage(IMG_WIDTH, IMG_HEIGHT, 255, 0, 0);
 const greenImage = createImage(IMG_WIDTH, IMG_HEIGHT, 0, 255, 0);
 const blueImage = createImage(IMG_WIDTH, IMG_HEIGHT, 0, 0, 255);
 
-const files = {
-  'helper.ts': `
-    export const test = pwt.test.extend({
-      auto: [ async ({}, run, testInfo) => {
-        testInfo.snapshotSuffix = '';
-        await run();
-      }, { auto: true } ]
-    });
-  `
-};
-
 test('should fail to screenshot a page with infinite animation', async ({ runInlineTest }, testInfo) => {
   const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html'));
   const result = await runInlineTest({
-    ...files,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await page.goto('${infiniteAnimationURL}');
         await expect(page).toHaveScreenshot({ timeout: 2000 });
       });
@@ -63,6 +50,36 @@ test('should fail to screenshot a page with infinite animation', async ({ runInl
   expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(false);
 });
 
+test('screenshotPath should include platform and project name by default', async ({ runInlineTest }, testInfo) => {
+  const PROJECT_NAME = 'woof-woof';
+  const result = await runInlineTest({
+    ...playwrightConfig({
+      projects: [{
+        name: PROJECT_NAME,
+      }],
+    }),
+    'a.spec.js': `
+      pwt.test('is a test', async ({ page }, testInfo) => {
+        await pwt.expect(page).toHaveScreenshot('snapshot.png');
+      });
+    `,
+    'foo/b.spec.js': `
+      pwt.test('is a test', async ({ page }, testInfo) => {
+        await pwt.expect(page).toHaveScreenshot('snapshot.png');
+      });
+    `,
+    'foo/bar/baz/c.spec.js': `
+      pwt.test('is a test', async ({ page }, testInfo) => {
+        await pwt.expect(page).toHaveScreenshot('snapshot.png');
+      });
+    `,
+  }, { 'update-snapshots': true });
+  expect(result.exitCode).toBe(0);
+  expect(fs.existsSync(testInfo.outputPath('__screenshots__', process.platform, PROJECT_NAME, 'a.spec.js', 'snapshot.png'))).toBeTruthy();
+  expect(fs.existsSync(testInfo.outputPath('__screenshots__', process.platform, PROJECT_NAME, 'foo', 'b.spec.js', 'snapshot.png'))).toBeTruthy();
+  expect(fs.existsSync(testInfo.outputPath('__screenshots__', process.platform, PROJECT_NAME, 'foo', 'bar', 'baz', 'c.spec.js', 'snapshot.png'))).toBeTruthy();
+});
+
 test('should report toHaveScreenshot step with expectation name in title', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'reporter.ts': `
@@ -78,10 +95,8 @@ test('should report toHaveScreenshot step with expectation name in title', async
         reporter: './reporter',
       };
     `,
-    ...files,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         // Named expectation.
         await expect(page).toHaveScreenshot('foo.png', { timeout: 2000 });
         // Anonymous expectation.
@@ -104,11 +119,12 @@ test('should report toHaveScreenshot step with expectation name in title', async
 test('should not fail when racing with navigation', async ({ runInlineTest }, testInfo) => {
   const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html'));
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({
-    'a.spec.js-snapshots/snapshot.png': createImage(10, 10, 255, 0, 0),
+      screenshotsDir: '__screenshots__',
+    }),
+    '__screenshots__/a.spec.js/snapshot.png': createImage(10, 10, 255, 0, 0),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await Promise.all([
           page.goto('${infiniteAnimationURL}'),
           expect(page).toHaveScreenshot({
@@ -126,10 +142,9 @@ test('should not fail when racing with navigation', async ({ runInlineTest }, te
 test('should successfully screenshot a page with infinite animation with disableAnimation: true', async ({ runInlineTest }, testInfo) => {
   const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html'));
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await page.goto('${infiniteAnimationURL}');
         await expect(page).toHaveScreenshot({
           animations: "disabled",
@@ -138,16 +153,15 @@ test('should successfully screenshot a page with infinite animation with disable
     `
   }, { 'update-snapshots': true });
   expect(result.exitCode).toBe(0);
-  expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(true);
+  expect(fs.existsSync(testInfo.outputPath('__screenshots__', 'a.spec.js', 'is-a-test-1.png'))).toBe(true);
 });
 
 test('should support clip option for page', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': createImage(50, 50, 255, 255, 255),
+    '__screenshots__/a.spec.js/snapshot.png': createImage(50, 50, 255, 255, 255),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot({
           name: 'snapshot.png',
           clip: { x: 0, y: 0, width: 50, height: 50, },
@@ -160,10 +174,9 @@ test('should support clip option for page', async ({ runInlineTest }, testInfo)
 
 test('should support omitBackground option for locator', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await page.evaluate(() => {
           document.body.style.setProperty('width', '100px');
           document.body.style.setProperty('height', '100px');
@@ -176,7 +189,7 @@ test('should support omitBackground option for locator', async ({ runInlineTest
     `
   }, { 'update-snapshots': true });
   expect(result.exitCode).toBe(0);
-  const snapshotPath = testInfo.outputPath('a.spec.js-snapshots', 'snapshot.png');
+  const snapshotPath = testInfo.outputPath('__screenshots__', 'a.spec.js', 'snapshot.png');
   expect(fs.existsSync(snapshotPath)).toBe(true);
   const png = PNG.sync.read(fs.readFileSync(snapshotPath));
   expect.soft(png.width, 'image width must be 100').toBe(100);
@@ -190,10 +203,9 @@ test('should support omitBackground option for locator', async ({ runInlineTest
 test('should fail to screenshot an element with infinite animation', async ({ runInlineTest }, testInfo) => {
   const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html'));
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await page.goto('${infiniteAnimationURL}');
         await expect(page.locator('body')).toHaveScreenshot({ timeout: 2000 });
       });
@@ -204,16 +216,15 @@ test('should fail to screenshot an element with infinite animation', async ({ ru
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-actual.png'))).toBe(true);
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-expected.png'))).toBe(true);
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-diff.png'))).toBe(true);
-  expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(false);
+  expect(fs.existsSync(testInfo.outputPath('__screenshots__', 'a.spec.js', 'is-a-test-1.png'))).toBe(false);
 });
 
 test('should fail to screenshot an element that keeps moving', async ({ runInlineTest }, testInfo) => {
   const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html'));
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await page.goto('${infiniteAnimationURL}');
         await expect(page.locator('div')).toHaveScreenshot({ timeout: 2000 });
       });
@@ -225,22 +236,21 @@ test('should fail to screenshot an element that keeps moving', async ({ runInlin
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-actual.png'))).toBe(false);
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-expected.png'))).toBe(false);
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-diff.png'))).toBe(false);
-  expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(false);
+  expect(fs.existsSync(testInfo.outputPath('__screenshots__', 'a.spec.js', 'is-a-test-1.png'))).toBe(false);
 });
 
 test('should generate default name', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot();
       });
     `
   });
   expect(result.exitCode).toBe(1);
   expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-actual.png'))).toBe(true);
-  expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(true);
+  expect(fs.existsSync(testInfo.outputPath('__screenshots__', 'a.spec.js', 'is-a-test-1.png'))).toBe(true);
 });
 
 test('should compile with different option combinations', async ({ runTSC }) => {
@@ -267,11 +277,10 @@ test('should compile with different option combinations', async ({ runTSC }) =>
 
 test('should fail when screenshot is different size', async ({ runInlineTest }) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': createImage(22, 33),
+    '__screenshots__/a.spec.js/snapshot.png': createImage(22, 33),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 });
       });
     `
@@ -282,11 +291,10 @@ test('should fail when screenshot is different size', async ({ runInlineTest })
 
 test('should fail when screenshot is different pixels', async ({ runInlineTest }) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': paintBlackPixels(whiteImage, 12345),
+    '__screenshots__/a.spec.js/snapshot.png': paintBlackPixels(whiteImage, 12345),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 });
       });
     `
@@ -302,11 +310,10 @@ test('should fail when screenshot is different pixels', async ({ runInlineTest }
 
 test('doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': blueImage,
+    '__screenshots__/a.spec.js/snapshot.png': blueImage,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).not.toHaveScreenshot('snapshot.png');
       });
     `
@@ -324,11 +331,10 @@ test('doesn\'t create comparison artifacts in an output folder for passed negate
 
 test('should fail on same snapshots with negate matcher', async ({ runInlineTest }) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': whiteImage,
+    '__screenshots__/a.spec.js/snapshot.png': whiteImage,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).not.toHaveScreenshot('snapshot.png', { timeout: 2000 });
       });
     `
@@ -341,10 +347,9 @@ test('should fail on same snapshots with negate matcher', async ({ runInlineTest
 
 test('should write missing expectations locally twice and continue', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png');
         await expect(page).toHaveScreenshot('snapshot2.png');
         console.log('Here we are!');
@@ -355,52 +360,50 @@ test('should write missing expectations locally twice and continue', async ({ ru
   expect(result.exitCode).toBe(1);
   expect(result.failed).toBe(1);
 
-  const snapshot1OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshot1OutputPath = testInfo.outputPath('__screenshots__', 'a.spec.js', 'snapshot.png');
   expect(result.output).toContain(`Error: ${snapshot1OutputPath} is missing in snapshots, writing actual`);
   expect(pngComparator(fs.readFileSync(snapshot1OutputPath), whiteImage)).toBe(null);
 
-  const snapshot2OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot2.png');
+  const snapshot2OutputPath = testInfo.outputPath('__screenshots__', 'a.spec.js', 'snapshot2.png');
   expect(result.output).toContain(`Error: ${snapshot2OutputPath} is missing in snapshots, writing actual`);
   expect(pngComparator(fs.readFileSync(snapshot2OutputPath), whiteImage)).toBe(null);
 
   expect(result.output).toContain('Here we are!');
 
   const stackLines = stripAnsi(result.output).split('\n').filter(line => line.includes('    at ')).filter(line => !line.includes(testInfo.outputPath()));
-  expect(result.output).toContain('a.spec.js:8');
+  expect(result.output).toContain('a.spec.js:5');
   expect(stackLines.length).toBe(0);
 });
 
 test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).not.toHaveScreenshot('snapshot.png');
       });
     `
   });
 
   expect(result.exitCode).toBe(1);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshotOutputPath = testInfo.outputPath('__screenshots__/a.spec.js/snapshot.png');
   expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`);
   expect(fs.existsSync(snapshotOutputPath)).toBe(false);
 });
 
 test('should update snapshot with the update-snapshots flag', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': blueImage,
+    '__screenshots__/a.spec.js/snapshot.png': blueImage,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png');
       });
     `
   }, { 'update-snapshots': true });
 
   expect(result.exitCode).toBe(0);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshotOutputPath = testInfo.outputPath('__screenshots__/a.spec.js/snapshot.png');
   expect(result.output).toContain(`${snapshotOutputPath} is re-generated, writing actual.`);
   expect(pngComparator(fs.readFileSync(snapshotOutputPath), whiteImage)).toBe(null);
 });
@@ -408,34 +411,32 @@ test('should update snapshot with the update-snapshots flag', async ({ runInline
 test('shouldn\'t update snapshot with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => {
   const EXPECTED_SNAPSHOT = blueImage;
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).not.toHaveScreenshot('snapshot.png');
       });
     `
   }, { 'update-snapshots': true });
 
   expect(result.exitCode).toBe(0);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshotOutputPath = testInfo.outputPath('__screenshots__/a.spec.js/snapshot.png');
   expect(fs.readFileSync(snapshotOutputPath).equals(EXPECTED_SNAPSHOT)).toBe(true);
 });
 
 test('should silently write missing expectations locally with the update-snapshots flag', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png');
       });
     `
   }, { 'update-snapshots': true });
 
   expect(result.exitCode).toBe(0);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshotOutputPath = testInfo.outputPath('__screenshots__/a.spec.js/snapshot.png');
   expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`);
   const data = fs.readFileSync(snapshotOutputPath);
   expect(pngComparator(data, whiteImage)).toBe(null);
@@ -443,30 +444,28 @@ test('should silently write missing expectations locally with the update-snapsho
 
 test('should not write missing expectations locally with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).not.toHaveScreenshot('snapshot.png');
       });
     `
   }, { 'update-snapshots': true });
 
   expect(result.exitCode).toBe(1);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshotOutputPath = testInfo.outputPath('__screenshots__/a.spec.js/snapshot.png');
   expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`);
   expect(fs.existsSync(snapshotOutputPath)).toBe(false);
 });
 
 test('should match multiple snapshots', async ({ runInlineTest }) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/red.png': redImage,
+    '__screenshots__/a.spec.js/red.png': redImage,
-    'a.spec.js-snapshots/green.png': greenImage,
+    '__screenshots__/a.spec.js/green.png': greenImage,
-    'a.spec.js-snapshots/blue.png': blueImage,
+    '__screenshots__/a.spec.js/blue.png': blueImage,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await Promise.all([
           page.evaluate(() => document.documentElement.style.setProperty('background', '#f00')),
           expect(page).toHaveScreenshot('red.png'),
@@ -487,11 +486,10 @@ test('should match multiple snapshots', async ({ runInlineTest }) => {
 
 test('should use provided name', async ({ runInlineTest }) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/provided.png': whiteImage,
+    '__screenshots__/a.spec.js/provided.png': whiteImage,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('provided.png');
       });
     `
@@ -501,11 +499,10 @@ test('should use provided name', async ({ runInlineTest }) => {
 
 test('should use provided name via options', async ({ runInlineTest }) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/provided.png': whiteImage,
+    '__screenshots__/a.spec.js/provided.png': whiteImage,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot({ name: 'provided.png' });
       });
     `
@@ -518,22 +515,20 @@ test('should respect maxDiffPixels option', async ({ runInlineTest }) => {
   const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 });
       });
     `
   })).exitCode, 'make sure default comparison fails').toBe(1);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', {
           maxDiffPixels: ${BAD_PIXELS}
         });
@@ -542,16 +537,21 @@ test('should respect maxDiffPixels option', async ({ runInlineTest }) => {
   })).exitCode, 'make sure maxDiffPixels option is respected').toBe(0);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({
-    'playwright.config.ts': `
+      projects: [
-      module.exports = { projects: [
+        {
-        { expect: { toHaveScreenshot: { maxDiffPixels: ${BAD_PIXELS} } } },
+          screenshotsDir: '__screenshots__',
-      ]};
+          expect: {
-    `,
+            toHaveScreenshot: {
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+              maxDiffPixels: BAD_PIXELS
+            }
+          },
+        },
+      ],
+    }),
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png');
       });
     `
@@ -564,22 +564,20 @@ test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInli
   const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_COUNT);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 });
       });
     `
   })).exitCode, 'make sure default comparison fails').toBe(1);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', {
           maxDiffPixels: ${Math.floor(BAD_COUNT / 2)},
           maxDiffPixelRatio: ${BAD_RATIO},
@@ -590,11 +588,10 @@ test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInli
   })).exitCode, 'make sure it fails when maxDiffPixels < actualBadPixels < maxDiffPixelRatio').toBe(1);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', {
           maxDiffPixels: ${BAD_COUNT},
           maxDiffPixelRatio: ${BAD_RATIO / 2},
@@ -605,11 +602,10 @@ test('should satisfy both maxDiffPixelRatio and maxDiffPixels', async ({ runInli
   })).exitCode, 'make sure it fails when maxDiffPixelRatio < actualBadPixels < maxDiffPixels').toBe(1);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', {
           maxDiffPixels: ${BAD_COUNT},
           maxDiffPixelRatio: ${BAD_RATIO},
@@ -625,22 +621,20 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => {
   const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 });
       });
     `
   })).exitCode, 'make sure default comparison fails').toBe(1);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', {
           maxDiffPixelRatio: ${BAD_RATIO}
         });
@@ -649,16 +643,19 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => {
   })).exitCode, 'make sure maxDiffPixelRatio option is respected').toBe(0);
 
   expect((await runInlineTest({
-    ...files,
+    ...playwrightConfig({
-    'playwright.config.ts': `
+      projects: [{
-      module.exports = { projects: [
+        screenshotsDir: '__screenshots__',
-        { expect: { toHaveScreenshot: { maxDiffPixelRatio: ${BAD_RATIO} } } },
+        expect: {
-      ]};
+          toHaveScreenshot: {
-    `,
+            maxDiffPixelRatio: BAD_RATIO,
-    'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT,
+          },
+        },
+      }],
+    }),
+    '__screenshots__/a.spec.js/snapshot.png': EXPECTED_SNAPSHOT,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png');
       });
     `
@@ -667,10 +664,8 @@ test('should respect maxDiffPixelRatio option', async ({ runInlineTest }) => {
 
 test('should throw for invalid maxDiffPixels values', async ({ runInlineTest }) => {
   expect((await runInlineTest({
-    ...files,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot({
           maxDiffPixels: -1,
         });
@@ -681,10 +676,8 @@ test('should throw for invalid maxDiffPixels values', async ({ runInlineTest })
 
 test('should throw for invalid maxDiffPixelRatio values', async ({ runInlineTest }) => {
   expect((await runInlineTest({
-    ...files,
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot({
           maxDiffPixelRatio: 12,
         });
@@ -696,14 +689,13 @@ test('should throw for invalid maxDiffPixelRatio values', async ({ runInlineTest
 
 test('should attach expected/actual and no diff when sizes are different', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({ screenshotsDir: '__screenshots__' }),
-    'a.spec.js-snapshots/snapshot.png': createImage(2, 2),
+    '__screenshots__/a.spec.js/snapshot.png': createImage(2, 2),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test.afterEach(async ({}, testInfo) => {
-      test.afterEach(async ({}, testInfo) => {
         console.log('## ' + JSON.stringify(testInfo.attachments));
       });
-      test('is a test', async ({ page }) => {
+      pwt.test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 });
       });
     `
@@ -731,13 +723,12 @@ test('should attach expected/actual and no diff when sizes are different', async
 
 test('should fail with missing expectations and retries', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({
-    'playwright.config.ts': `
+      retries: 1,
-      module.exports = { retries: 1 };
+      screenshotsDir: '__screenshots__'
-    `,
+    }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png');
       });
     `
@@ -745,7 +736,7 @@ test('should fail with missing expectations and retries', async ({ runInlineTest
 
   expect(result.exitCode).toBe(1);
   expect(result.failed).toBe(1);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshotOutputPath = testInfo.outputPath('__screenshots__/a.spec.js/snapshot.png');
   expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`);
   const data = fs.readFileSync(snapshotOutputPath);
   expect(pngComparator(data, whiteImage)).toBe(null);
@@ -753,13 +744,12 @@ test('should fail with missing expectations and retries', async ({ runInlineTest
 
 test('should update expectations with retries', async ({ runInlineTest }, testInfo) => {
   const result = await runInlineTest({
-    ...files,
+    ...playwrightConfig({
-    'playwright.config.ts': `
+      retries: 1,
-      module.exports = { retries: 1 };
+      screenshotsDir: '__screenshots__'
-    `,
+    }),
     'a.spec.js': `
-      const { test } = require('./helper');
+      pwt.test('is a test', async ({ page }) => {
-      test('is a test', async ({ page }) => {
         await expect(page).toHaveScreenshot('snapshot.png');
       });
     `
@@ -767,9 +757,16 @@ test('should update expectations with retries', async ({ runInlineTest }, testIn
 
   expect(result.exitCode).toBe(0);
   expect(result.passed).toBe(1);
-  const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png');
+  const snapshotOutputPath = testInfo.outputPath('__screenshots__/a.spec.js/snapshot.png');
   expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`);
   const data = fs.readFileSync(snapshotOutputPath);
   expect(pngComparator(data, whiteImage)).toBe(null);
 });
 
+function playwrightConfig(obj: any) {
+  return {
+    'playwright.config.js': `
+      module.exports = ${JSON.stringify(obj, null, 2)}
+    `,
+  };
+}

utils/generate_types/overrides-test.d.ts

@@ -70,6 +70,7 @@ interface TestProject {
   metadata?: any;
   name?: string;
   snapshotDir?: string;
+  screenshotsDir?: string;
   outputDir?: string;
   repeatEach?: number;
   retries?: number;
@@ -146,6 +147,7 @@ interface TestConfig {
   metadata?: any;
   name?: string;
   snapshotDir?: string;
+  screenshotsDir?: string;
   outputDir?: string;
   repeatEach?: number;
   retries?: number;