test(cli): setup files are in list-files and test --list output (#18890)

packages/playwright-test/src/runner.ts

@@ -203,7 +203,7 @@ export class Runner {
     for (const [project, files] of filesByProject) {
       report.projects.push({
         ...sanitizeConfigForJSON(project, new Set()),
-        files: files,
+        files
       });
     }
     return report;

tests/playwright-test/playwright-test-fixtures.ts

@@ -30,6 +30,11 @@ import { test as base } from './stable-test-runner';
 
 const removeFolderAsync = promisify(rimraf);
 
+export type CliRunResult = {
+  exitCode: number,
+  output: string,
+};
+
 export type RunResult = {
   exitCode: number,
   output: string,
@@ -113,7 +118,7 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
   }
   const outputDir = path.join(baseDir, 'test-results');
   const reportFile = path.join(outputDir, 'report.json');
-  const args = ['node', cliEntrypoint, 'test'];
+  const args = ['test'];
   if (!options.usesCustomOutputDir)
     args.push('--output=' + outputDir);
   if (!options.usesCustomReporters)
@@ -124,12 +129,71 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
   );
   if (options.additionalArgs)
     args.push(...options.additionalArgs);
-  const cacheDir = fs.mkdtempSync(path.join(os.tmpdir(), 'playwright-test-cache-'));
+
+  const cwd = options.cwd ? path.resolve(baseDir, options.cwd) : baseDir;
+  // eslint-disable-next-line prefer-const
+  let { exitCode, output } = await runPlaywrightCommand(childProcess, cwd, args, {
+    PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile,
+    ...env,
+  }, options.sendSIGINTAfter);
+
+  const summary = (re: RegExp) => {
+    let result = 0;
+    let match = re.exec(output);
+    while (match) {
+      result += (+match[1]);
+      match = re.exec(output);
+    }
+    return result;
+  };
+  const passed = summary(/(\d+) passed/g);
+  const failed = summary(/(\d+) failed/g);
+  const flaky = summary(/(\d+) flaky/g);
+  const skipped = summary(/(\d+) skipped/g);
+  const interrupted = summary(/(\d+) interrupted/g);
+  let report;
+  try {
+    report = JSON.parse(fs.readFileSync(reportFile).toString());
+  } catch (e) {
+    output += '\n' + e.toString();
+  }
+
+  const results: JSONReportTestResult[] = [];
+  function visitSuites(suites?: JSONReportSuite[]) {
+    if (!suites)
+      return;
+    for (const suite of suites) {
+      for (const spec of suite.specs) {
+        for (const test of spec.tests)
+          results.push(...test.results);
+      }
+      visitSuites(suite.suites);
+    }
+  }
+  if (report)
+    visitSuites(report.suites);
+
+  return {
+    exitCode,
+    output,
+    passed,
+    failed,
+    flaky,
+    skipped,
+    interrupted,
+    report,
+    results,
+  };
+}
+
+async function runPlaywrightCommand(childProcess: CommonFixtures['childProcess'], cwd: string, commandWithArguments: string[], env: Env, sendSIGINTAfter?: number): Promise<CliRunResult> {
+  const command = ['node', cliEntrypoint];
+  command.push(...commandWithArguments);
+  const cacheDir = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'playwright-test-cache-'));
   const testProcess = childProcess({
-    command: args,
+    command,
     env: {
       ...process.env,
-      PLAYWRIGHT_JSON_OUTPUT_NAME: reportFile,
       PWTEST_CACHE_DIR: cacheDir,
       // BEGIN: Reserved CI
       CI: undefined,
@@ -151,11 +215,11 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
       NODE_OPTIONS: undefined,
       ...env,
     },
-    cwd: options.cwd ? path.resolve(baseDir, options.cwd) : baseDir,
+    cwd,
   });
   let didSendSigint = false;
   testProcess.onOutput = () => {
-    if (options.sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= options.sendSIGINTAfter) {
+    if (sendSIGINTAfter && !didSendSigint && countTimes(testProcess.output, '%%SEND-SIGINT%%') >= sendSIGINTAfter) {
       didSendSigint = true;
       process.kill(testProcess.process.pid!, 'SIGINT');
     }
@@ -163,56 +227,10 @@ async function runPlaywrightTest(childProcess: CommonFixtures['childProcess'], b
   const { exitCode } = await testProcess.exited;
   await removeFolderAsync(cacheDir);
 
-  const outputString = testProcess.output.toString();
+  return { exitCode, output: testProcess.output.toString() };
-  const summary = (re: RegExp) => {
-    let result = 0;
-    let match = re.exec(outputString);
-    while (match) {
-      result += (+match[1]);
-      match = re.exec(outputString);
-    }
-    return result;
-  };
-  const passed = summary(/(\d+) passed/g);
-  const failed = summary(/(\d+) failed/g);
-  const flaky = summary(/(\d+) flaky/g);
-  const skipped = summary(/(\d+) skipped/g);
-  const interrupted = summary(/(\d+) interrupted/g);
-  let report;
-  try {
-    report = JSON.parse(fs.readFileSync(reportFile).toString());
-  } catch (e) {
-    testProcess.output += '\n' + e.toString();
-  }
-
-  const results: JSONReportTestResult[] = [];
-  function visitSuites(suites?: JSONReportSuite[]) {
-    if (!suites)
-      return;
-    for (const suite of suites) {
-      for (const spec of suite.specs) {
-        for (const test of spec.tests)
-          results.push(...test.results);
-      }
-      visitSuites(suite.suites);
-    }
-  }
-  if (report)
-    visitSuites(report.suites);
-
-  return {
-    exitCode,
-    output: testProcess.output,
-    passed,
-    failed,
-    flaky,
-    skipped,
-    interrupted,
-    report,
-    results,
-  };
 }
 
+
 type RunOptions = {
   sendSIGINTAfter?: number;
   usesCustomOutputDir?: boolean;
@@ -226,6 +244,7 @@ type Fixtures = {
   runTSC: (files: Files) => Promise<TSCResult>;
   nodeVersion: { major: number, minor: number, patch: number };
   runGroups: (files: Files, params?: Params, env?: Env, options?: RunOptions) => Promise<{ timeline: { titlePath: string[], event: 'begin' | 'end' }[] } & RunResult>;
+  runCommand: (files: Files, args: string[]) => Promise<CliRunResult>;
 };
 
 export const test = base
@@ -245,6 +264,13 @@ export const test = base
         });
       },
 
+      runCommand: async ({ childProcess }, use, testInfo: TestInfo) => {
+        await use(async (files: Files, args: string[]) => {
+          const baseDir = await writeFiles(testInfo, files);
+          return await runPlaywrightCommand(childProcess, baseDir, args, { });
+        });
+      },
+
       runTSC: async ({ childProcess }, use, testInfo) => {
         await use(async files => {
           const baseDir = await writeFiles(testInfo, { 'tsconfig.json': JSON.stringify(TSCONFIG), ...files });

tests/playwright-test/project-setup.spec.ts

@@ -328,3 +328,100 @@ test('same file cannot be a setup and a test in different projects', async ({ ru
   expect(exitCode).toBe(1);
   expect(output).toContain(`a.test.ts" matches 'setup' filter in project "p1" and 'testMatch' filter in project "p2"`);
 });
+
+test('list-files should enumerate setup files in same group', async ({ runCommand }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*a..setup.ts$/,
+            testMatch: /.*a.test.ts$/,
+          },
+          {
+            name: 'p2',
+            setup: /.*b.setup.ts$/,
+            testMatch: /.*b.test.ts$/
+          },
+        ]
+      };`,
+    'a1.setup.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+    `,
+    'a2.setup.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test2', async () => { });
+    `,
+    'b.setup.ts': `
+      const { test } = pwt;
+      test('test3', async () => { });
+    `,
+    'b.test.ts': `
+      const { test } = pwt;
+      test('test4', async () => { });
+    `,
+  };
+
+  const { exitCode, output } =  await runCommand(files, ['list-files']);
+  expect(exitCode).toBe(0);
+  const json = JSON.parse(output);
+  expect(json.projects.map(p => p.name)).toEqual(['p1', 'p2']);
+  expect(json.projects[0].files.map(f => path.basename(f))).toEqual(['a.test.ts', 'a1.setup.ts', 'a2.setup.ts']);
+  expect(json.projects[1].files.map(f => path.basename(f))).toEqual(['b.setup.ts', 'b.test.ts']);
+});
+
+test('test --list should enumerate setup tests as regular ones', async ({ runCommand }, testInfo) => {
+  const files = {
+    'playwright.config.ts': `
+      module.exports = {
+        projects: [
+          {
+            name: 'p1',
+            setup: /.*a..setup.ts$/,
+            testMatch: /.*a.test.ts$/,
+          },
+          {
+            name: 'p2',
+            setup: /.*b.setup.ts$/,
+            testMatch: /.*b.test.ts$/
+          },
+        ]
+      };`,
+    'a1.setup.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+    `,
+    'a2.setup.ts': `
+      const { test } = pwt;
+      test('test1', async () => { });
+    `,
+    'a.test.ts': `
+      const { test } = pwt;
+      test('test2', async () => { });
+    `,
+    'b.setup.ts': `
+      const { test } = pwt;
+      test('test3', async () => { });
+    `,
+    'b.test.ts': `
+      const { test } = pwt;
+      test('test4', async () => { });
+    `,
+  };
+
+  const { exitCode, output } =  await runCommand(files, ['test', '--list']);
+  expect(exitCode).toBe(0);
+  expect(output).toContain(`Listing tests:
+  [p1] › a.test.ts:6:7 › test2
+  [p1] › a1.setup.ts:5:7 › test1
+  [p1] › a2.setup.ts:5:7 › test1
+  [p2] › b.setup.ts:5:7 › test3
+  [p2] › b.test.ts:6:7 › test4
+Total: 5 tests in 5 files`);
+});