feat: project.teardown that runs after all dependents have finished (#22696)

This replicates globalTeardown in the deps world.

Fixes #21914.

docs/src/test-api/class-testproject.md

@@ -57,7 +57,7 @@ behaves as if they were not specified.
 Using dependencies allows global setup to produce traces and other artifacts,
 see the setup steps in the test report, etc.
 
-For example:
+**Usage**
 
 ```js
 // playwright.config.ts
@@ -198,6 +198,52 @@ Use [`method: Test.describe.configure`] to change the number of retries for a sp
 
 Use [`property: TestConfig.retries`] to change this option for all projects.
 
+## property: TestProject.teardown
+* since: v1.34
+- type: ?<[string]>
+
+Name of a project that needs to run after this and any dependent projects have finished. Teardown is useful to cleanup any resources acquired by this project.
+
+Passing `--no-deps` argument ignores [`property: TestProject.teardown`] and behaves as if it was not specified.
+
+**Usage**
+
+A common pattern is a "setup" dependency that has a corresponding "teardown":
+
+```js
+// playwright.config.ts
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  projects: [
+    {
+      name: 'setup',
+      testMatch: /global.setup\.ts/,
+      teardown: 'teardown',
+    },
+    {
+      name: 'teardown',
+      testMatch: /global.teardown\.ts/,
+    },
+    {
+      name: 'chromium',
+      use: devices['Desktop Chrome'],
+      dependencies: ['setup'],
+    },
+    {
+      name: 'firefox',
+      use: devices['Desktop Firefox'],
+      dependencies: ['setup'],
+    },
+    {
+      name: 'webkit',
+      use: devices['Desktop Safari'],
+      dependencies: ['setup'],
+    },
+  ],
+});
+```
+
 ## property: TestProject.testDir
 * since: v1.10
 - type: ?<[string]>

packages/playwright-test/src/common/config.ts

@@ -152,6 +152,7 @@ export class FullProjectInternal {
   readonly snapshotPathTemplate: string;
   id = '';
   deps: FullProjectInternal[] = [];
+  teardown: FullProjectInternal | undefined;
 
   constructor(configDir: string, config: Config, fullConfig: FullConfigInternal, projectConfig: Project, configCLIOverrides: ConfigCLIOverrides, throwawayArtifactsPath: string) {
     this.fullConfig = fullConfig;
@@ -174,6 +175,7 @@ export class FullProjectInternal {
       timeout: takeFirst(configCLIOverrides.timeout, projectConfig.timeout, config.timeout, defaultTimeout),
       use: mergeObjects(config.use, projectConfig.use, configCLIOverrides.use),
       dependencies: projectConfig.dependencies || [],
+      teardown: projectConfig.teardown,
     };
     (this.project as any)[projectInternalSymbol] = this;
     this.fullyParallel = takeFirst(configCLIOverrides.fullyParallel, projectConfig.fullyParallel, config.fullyParallel, undefined);
@@ -205,6 +207,7 @@ function resolveReporters(reporters: Config['reporter'], rootDir: string): Repor
 }
 
 function resolveProjectDependencies(projects: FullProjectInternal[]) {
+  const teardownToSetup = new Map<FullProjectInternal, FullProjectInternal>();
   for (const project of projects) {
     for (const dependencyName of project.project.dependencies) {
       const dependencies = projects.filter(p => p.project.name === dependencyName);
@@ -214,6 +217,28 @@ function resolveProjectDependencies(projects: FullProjectInternal[]) {
         throw new Error(`Project dependencies should have unique names, reading ${dependencyName}`);
       project.deps.push(...dependencies);
     }
+    if (project.project.teardown) {
+      const teardowns = projects.filter(p => p.project.name === project.project.teardown);
+      if (!teardowns.length)
+        throw new Error(`Project '${project.project.name}' has unknown teardown project '${project.project.teardown}'`);
+      if (teardowns.length > 1)
+        throw new Error(`Project teardowns should have unique names, reading ${project.project.teardown}`);
+      const teardown = teardowns[0];
+      project.teardown = teardown;
+      if (teardownToSetup.has(teardown))
+        throw new Error(`Project ${teardown.project.name} can not be designated as teardown to multiple projects (${teardownToSetup.get(teardown)!.project.name} and ${project.project.name})`);
+      teardownToSetup.set(teardown, project);
+    }
+  }
+  for (const teardown of teardownToSetup.keys()) {
+    if (teardown.deps.length)
+      throw new Error(`Teardown project ${teardown.project.name} must not have dependencies`);
+  }
+  for (const project of projects) {
+    for (const dep of project.deps) {
+      if (teardownToSetup.has(dep))
+        throw new Error(`Project ${project.project.name} must not depend on a teardown project ${dep.project.name}`);
+    }
   }
 }
 

packages/playwright-test/src/common/configLoader.ts

@@ -55,8 +55,10 @@ export class ConfigLoader {
     const fullConfig = await this._loadConfig(config, path.dirname(file), file);
     setCurrentConfig(fullConfig);
     if (ignoreProjectDependencies) {
-      for (const project of fullConfig.projects)
+      for (const project of fullConfig.projects) {
         project.deps = [];
+        project.teardown = undefined;
+      }
     }
     this._fullConfig = fullConfig;
     return fullConfig;

packages/playwright-test/src/isomorphic/teleReceiver.ts

@@ -48,6 +48,7 @@ export type JsonProject = {
   repeatEach: number;
   retries: number;
   suites: JsonSuite[];
+  teardown?: string;
   testDir: string;
   testIgnore: JsonPattern[];
   testMatch: JsonPattern[];
@@ -303,6 +304,7 @@ export class TeleReporterReceiver {
       grep: parseRegexPatterns(project.grep) as RegExp[],
       grepInvert: parseRegexPatterns(project.grepInvert) as RegExp[],
       dependencies: project.dependencies,
+      teardown: project.teardown,
       snapshotDir: this._absolutePath(project.snapshotDir),
       use: {},
     };

packages/playwright-test/src/reporters/teleEmitter.ts

@@ -151,6 +151,7 @@ export class TeleReporterEmitter implements Reporter {
       grepInvert: serializeRegexPatterns(project.grepInvert || []),
       dependencies: project.dependencies,
       snapshotDir: this._relativePath(project.snapshotDir),
+      teardown: project.teardown,
     };
     return report;
   }

packages/playwright-test/src/runner/projectUtils.ts

@@ -49,6 +49,15 @@ export function filterProjects(projects: FullProjectInternal[], projectNames?: s
   return result;
 }
 
+export function buildTeardownToSetupMap(projects: FullProjectInternal[]): Map<FullProjectInternal, FullProjectInternal> {
+  const result = new Map<FullProjectInternal, FullProjectInternal>();
+  for (const project of projects) {
+    if (project.teardown)
+      result.set(project.teardown, project);
+  }
+  return result;
+}
+
 export function buildProjectsClosure(projects: FullProjectInternal[]): Map<FullProjectInternal, 'top-level' | 'dependency'> {
   const result = new Map<FullProjectInternal, 'top-level' | 'dependency'>();
   const visit = (depth: number, project: FullProjectInternal) => {
@@ -59,6 +68,8 @@ export function buildProjectsClosure(projects: FullProjectInternal[]): Map<FullP
     }
     result.set(project, depth ? 'dependency' : 'top-level');
     project.deps.map(visit.bind(undefined, depth + 1));
+    if (project.teardown)
+      visit(depth + 1, project.teardown);
   };
   for (const p of projects)
     result.set(p, 'top-level');
@@ -67,6 +78,29 @@ export function buildProjectsClosure(projects: FullProjectInternal[]): Map<FullP
   return result;
 }
 
+export function buildDependentProjects(forProject: FullProjectInternal, projects: FullProjectInternal[]): Set<FullProjectInternal> {
+  const reverseDeps = new Map<FullProjectInternal, FullProjectInternal[]>(projects.map(p => ([p, []])));
+  for (const project of projects) {
+    for (const dep of project.deps)
+      reverseDeps.get(dep)!.push(project);
+  }
+  const result = new Set<FullProjectInternal>();
+  const visit = (depth: number, project: FullProjectInternal) => {
+    if (depth > 100) {
+      const error = new Error('Circular dependency detected between projects.');
+      error.stack = '';
+      throw error;
+    }
+    result.add(project);
+    for (const reverseDep of reverseDeps.get(project)!)
+      visit(depth + 1, reverseDep);
+    if (project.teardown)
+      visit(depth + 1, project.teardown);
+  };
+  visit(0, forProject);
+  return result;
+}
+
 export async function collectFilesForProject(project: FullProjectInternal, fsCache = new Map<string, string[]>()): Promise<string[]> {
   const extensions = ['.js', '.ts', '.mjs', '.tsx', '.jsx'];
   const testFileExtension = (file: string) => extensions.includes(path.extname(file));

packages/playwright-test/src/runner/tasks.ts

@@ -28,6 +28,7 @@ import type { FullConfigInternal, FullProjectInternal } from '../common/config';
 import { collectProjectsAndTestFiles, createRootSuite, loadFileSuites, loadGlobalHook } from './loadUtils';
 import type { Matcher } from '../util';
 import type { Suite } from '../common/test';
+import { buildDependentProjects, buildTeardownToSetupMap } from './projectUtils';
 
 const removeFolderAsync = promisify(rimraf);
 const readDirAsync = promisify(fs.readdir);
@@ -182,13 +183,23 @@ function createPhasesTask(): Task<TestRun> {
 
     const processed = new Set<FullProjectInternal>();
     const projectToSuite = new Map(testRun.rootSuite!.suites.map(suite => [suite._fullProject!, suite]));
+    const allProjects = [...projectToSuite.keys()];
+    const teardownToSetup = buildTeardownToSetupMap(allProjects);
+    const teardownToSetupDependents = new Map<FullProjectInternal, FullProjectInternal[]>();
+    for (const [teardown, setup] of teardownToSetup) {
+      const closure = buildDependentProjects(setup, allProjects);
+      closure.delete(teardown);
+      teardownToSetupDependents.set(teardown, [...closure]);
+    }
+
     for (let i = 0; i < projectToSuite.size; i++) {
       // Find all projects that have all their dependencies processed by previous phases.
       const phaseProjects: FullProjectInternal[] = [];
       for (const project of projectToSuite.keys()) {
         if (processed.has(project))
           continue;
-        if (project.deps.find(p => !processed.has(p)))
+        const projectsThatShouldFinishFirst = [...project.deps, ...(teardownToSetupDependents.get(project) || [])];
+        if (projectsThatShouldFinishFirst.find(p => !processed.has(p)))
           continue;
         phaseProjects.push(project);
       }
@@ -229,6 +240,7 @@ function createRunTestsTask(): Task<TestRun> {
     const { phases } = testRun;
     const successfulProjects = new Set<FullProjectInternal>();
     const extraEnvByProjectId: EnvByProjectId = new Map();
+    const teardownToSetup = buildTeardownToSetupMap(phases.map(phase => phase.projects.map(p => p.project)).flat());
 
     for (const { dispatcher, projects } of phases) {
       // Each phase contains dispatcher and a set of test groups.
@@ -240,6 +252,9 @@ function createRunTestsTask(): Task<TestRun> {
         let extraEnv: Record<string, string | undefined> = {};
         for (const dep of project.deps)
           extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(dep.id) };
+        const setupForTeardown = teardownToSetup.get(project);
+        if (setupForTeardown)
+          extraEnv = { ...extraEnv, ...extraEnvByProjectId.get(setupForTeardown.id) };
         extraEnvByProjectId.set(project.id, extraEnv);
 
         const hasFailedDeps = project.deps.some(p => !successfulProjects.has(p));

packages/playwright-test/src/runner/uiMode.ts

@@ -44,8 +44,10 @@ class UIMode {
     process.env.PW_LIVE_TRACE_STACKS = '1';
     config.cliListOnly = false;
     config.cliPassWithNoTests = true;
-    for (const project of config.projects)
+    for (const project of config.projects) {
       project.deps = [];
+      project.teardown = undefined;
+    }
 
     for (const p of config.projects) {
       p.project.retries = 0;

packages/playwright-test/src/runner/watchMode.ts

@@ -308,6 +308,8 @@ function affectedProjectsClosure(projectClosure: FullProjectInternal[], affected
         if (result.has(dep))
           result.add(p);
       }
+      if (p.teardown && result.has(p.teardown))
+        result.add(p);
     }
   }
   return result;

packages/playwright-test/types/test.d.ts

@@ -193,7 +193,7 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
    * Using dependencies allows global setup to produce traces and other artifacts, see the setup steps in the test
    * report, etc.
    *
-   * For example:
+   * **Usage**
    *
    * ```js
    * // playwright.config.ts
@@ -284,6 +284,54 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
    * option for all projects.
    */
   retries: number;
+  /**
+   * Name of a project that needs to run after this and any dependent projects have finished. Teardown is useful to
+   * cleanup any resources acquired by this project.
+   *
+   * Passing `--no-deps` argument ignores
+   * [testProject.teardown](https://playwright.dev/docs/api/class-testproject#test-project-teardown) and behaves as if
+   * it was not specified.
+   *
+   * **Usage**
+   *
+   * A common pattern is a "setup" dependency that has a corresponding "teardown":
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   projects: [
+   *     {
+   *       name: 'setup',
+   *       testMatch: /global.setup\.ts/,
+   *       teardown: 'teardown',
+   *     },
+   *     {
+   *       name: 'teardown',
+   *       testMatch: /global.teardown\.ts/,
+   *     },
+   *     {
+   *       name: 'chromium',
+   *       use: devices['Desktop Chrome'],
+   *       dependencies: ['setup'],
+   *     },
+   *     {
+   *       name: 'firefox',
+   *       use: devices['Desktop Firefox'],
+   *       dependencies: ['setup'],
+   *     },
+   *     {
+   *       name: 'webkit',
+   *       use: devices['Desktop Safari'],
+   *       dependencies: ['setup'],
+   *     },
+   *   ],
+   * });
+   * ```
+   *
+   */
+  teardown?: string;
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *
@@ -5932,7 +5980,7 @@ interface TestProject {
    * Using dependencies allows global setup to produce traces and other artifacts, see the setup steps in the test
    * report, etc.
    *
-   * For example:
+   * **Usage**
    *
    * ```js
    * // playwright.config.ts
@@ -6245,6 +6293,55 @@ interface TestProject {
    */
   snapshotPathTemplate?: string;
 
+  /**
+   * Name of a project that needs to run after this and any dependent projects have finished. Teardown is useful to
+   * cleanup any resources acquired by this project.
+   *
+   * Passing `--no-deps` argument ignores
+   * [testProject.teardown](https://playwright.dev/docs/api/class-testproject#test-project-teardown) and behaves as if
+   * it was not specified.
+   *
+   * **Usage**
+   *
+   * A common pattern is a "setup" dependency that has a corresponding "teardown":
+   *
+   * ```js
+   * // playwright.config.ts
+   * import { defineConfig } from '@playwright/test';
+   *
+   * export default defineConfig({
+   *   projects: [
+   *     {
+   *       name: 'setup',
+   *       testMatch: /global.setup\.ts/,
+   *       teardown: 'teardown',
+   *     },
+   *     {
+   *       name: 'teardown',
+   *       testMatch: /global.teardown\.ts/,
+   *     },
+   *     {
+   *       name: 'chromium',
+   *       use: devices['Desktop Chrome'],
+   *       dependencies: ['setup'],
+   *     },
+   *     {
+   *       name: 'firefox',
+   *       use: devices['Desktop Firefox'],
+   *       dependencies: ['setup'],
+   *     },
+   *     {
+   *       name: 'webkit',
+   *       use: devices['Desktop Safari'],
+   *       dependencies: ['setup'],
+   *     },
+   *   ],
+   * });
+   * ```
+   *
+   */
+  teardown?: string;
+
   /**
    * Directory that will be recursively scanned for test files. Defaults to the directory of the configuration file.
    *

tests/playwright-test/deps.spec.ts

@@ -43,9 +43,10 @@ test('should inherit env changes from dependencies', async ({ runInlineTest }) =
     'playwright.config.ts': `
       module.exports = { projects: [
         { name: 'A', testMatch: '**/a.spec.ts' },
-        { name: 'B', testMatch: '**/b.spec.ts' },
+        { name: 'B', testMatch: '**/b.spec.ts', teardown: 'E' },
         { name: 'C', testMatch: '**/c.spec.ts', dependencies: ['A'] },
         { name: 'D', testMatch: '**/d.spec.ts', dependencies: ['B'] },
+        { name: 'E', testMatch: '**/e.spec.ts' },
       ] };
     `,
     'a.spec.ts': `
@@ -75,11 +76,17 @@ test('should inherit env changes from dependencies', async ({ runInlineTest }) =
         console.log('\\n%%D-' + process.env.SET_IN_A + '-' + process.env.SET_IN_B + '-' + process.env.SET_OUTSIDE);
       });
     `,
+    'e.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('pass', async ({}, testInfo) => {
+        console.log('\\n%%E-' + process.env.SET_IN_A + '-' + process.env.SET_IN_B + '-' + process.env.SET_OUTSIDE);
+      });
+    `,
   }, {}, { SET_OUTSIDE: 'outside' });
-  expect(result.passed).toBe(4);
+  expect(result.passed).toBe(5);
   expect(result.failed).toBe(0);
   expect(result.skipped).toBe(0);
-  expect(result.outputLines.sort()).toEqual(['A', 'B', 'C-valuea-undefined-undefined', 'D-undefined-valueb-outside']);
+  expect(result.outputLines.sort()).toEqual(['A', 'B', 'C-valuea-undefined-undefined', 'D-undefined-valueb-outside', 'E-undefined-valueb-outside']);
 });
 
 test('should not run projects with dependencies when --no-deps is passed', async ({ runInlineTest }) => {
@@ -423,3 +430,132 @@ test('should run setup project with zero tests recursively', async ({ runInlineT
   expect(result.passed).toBe(2);
   expect(result.outputLines).toEqual(['A', 'C']);
 });
+
+test('should run project with teardown', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'A', teardown: 'B' },
+          { name: 'B' },
+        ],
+      };`,
+    'test.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', async ({}, testInfo) => {
+        console.log('\\n%%' + testInfo.project.name);
+      });
+    `,
+  }, { workers: 1 }, undefined, { additionalArgs: ['--project=A'] });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+  expect(result.outputLines).toEqual(['A', 'B']);
+});
+
+test('should run teardown after depedents', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'A', teardown: 'E' },
+          { name: 'B', dependencies: ['A'] },
+          { name: 'C', dependencies: ['B'], teardown: 'D' },
+          { name: 'D' },
+          { name: 'E' },
+        ],
+      };`,
+    'test.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', async ({}, testInfo) => {
+        console.log('\\n%%' + testInfo.project.name);
+      });
+    `,
+  }, { workers: 1 }, undefined, { additionalArgs: ['--project=C'] });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(5);
+  expect(result.outputLines).toEqual(['A', 'B', 'C', 'D', 'E']);
+});
+
+test('should run teardown after failure', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'A', teardown: 'D' },
+          { name: 'B', dependencies: ['A'] },
+          { name: 'C', dependencies: ['B'] },
+          { name: 'D' },
+        ],
+      };`,
+    'test.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', async ({}, testInfo) => {
+        console.log('\\n%%' + testInfo.project.name);
+        if (testInfo.project.name === 'A')
+          throw new Error('ouch');
+      });
+    `,
+  }, { workers: 1 }, undefined, { additionalArgs: ['--project=C'] });
+  expect(result.exitCode).toBe(1);
+  expect(result.passed).toBe(1);
+  expect(result.failed).toBe(1);
+  expect(result.skipped).toBe(2);
+  expect(result.outputLines).toEqual(['A', 'D']);
+});
+
+test('should complain about teardown being a dependency', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'A', teardown: 'B' },
+          { name: 'B' },
+          { name: 'C', dependencies: ['B'] },
+        ],
+      };`,
+    'test.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', () => {});
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Project C must not depend on a teardown project B`);
+});
+
+test('should complain about teardown having a dependency', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'A', teardown: 'B' },
+          { name: 'B', dependencies: ['C'] },
+          { name: 'C' },
+        ],
+      };`,
+    'test.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', () => {});
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Teardown project B must not have dependencies`);
+});
+
+test('should complain about teardown used multiple times', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          { name: 'A', teardown: 'C' },
+          { name: 'B', teardown: 'C' },
+          { name: 'C' },
+        ],
+      };`,
+    'test.spec.ts': `
+      import { test, expect } from '@playwright/test';
+      test('test', () => {});
+    `,
+  });
+  expect(result.exitCode).toBe(1);
+  expect(result.output).toContain(`Project C can not be designated as teardown to multiple projects (A and B)`);
+});

utils/generate_types/overrides-test.d.ts

@@ -36,7 +36,7 @@ export interface Project<TestArgs = {}, WorkerArgs = {}> extends TestProject {
 
 // [internal] !!! DO NOT ADD TO THIS !!!
 // [internal] It is part of the public API and is computed from the user's config.
-// [internal] If you need new fields internally, add them to FullConfigInternal instead.
+// [internal] If you need new fields internally, add them to FullProjectInternal instead.
 export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
   grep: RegExp | RegExp[];
   grepInvert: RegExp | RegExp[] | null;
@@ -47,6 +47,7 @@ export interface FullProject<TestArgs = {}, WorkerArgs = {}> {
   outputDir: string;
   repeatEach: number;
   retries: number;
+  teardown?: string;
   testDir: string;
   testIgnore: string | RegExp | (string | RegExp)[];
   testMatch: string | RegExp | (string | RegExp)[];