feat(playwright): enhance report filename generation with unique identifiers and support split mode (#629)

* feat(playwright): enhance report filename generation with unique identifiers and support split mode

* fix(playwright): update reporter type from "single" to "merged" and adjust related configurations

* chore(playwright): improve filename sanitization in reporter

* feat(web-integration): add function to replace illegal path characters

* feat(web-integration): update path sanitization function to replace spaces and illegal characters

apps/site/docs/en/integrate-with-playwright.mdx

@@ -26,7 +26,7 @@ Update playwright.config.ts
 export default defineConfig({
   testDir: './e2e',
 + timeout: 90 * 1000,
-+ reporter: [["list"], ["@midscene/web/playwright-report"]],
++ reporter: [["list"], ["@midscene/web/playwright-report", { type: "merged" }]], // type optional, default is "merged", means multiple test cases generate one report, optional value is "separate", means one report for each test case
 });
 ```
 

apps/site/docs/zh/integrate-with-playwright.mdx

@@ -27,7 +27,7 @@ import { PackageManagerTabs } from '@theme';
 export default defineConfig({
   testDir: './e2e',
 + timeout: 90 * 1000,
-+ reporter: [["list"], ["@midscene/web/playwright-report"]],
++ reporter: [["list"], ["@midscene/web/playwright-report", { type: "merged" }]], // type 可选, 默认值为 "merged"，表示多个测试用例生成一个报告，可选值为 "separate"，表示为每个测试用例一个报告
 });
 ```
 

packages/web-integration/src/common/utils.ts

@@ -77,8 +77,10 @@ export async function parseContextFromWebPage(
 
 export function reportFileName(tag = 'web') {
   const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);
-  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss-SSS');
-  return `${reportTagName || tag}-${dateTimeInFileName}`;
+  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');
+  // ensure uniqueness at the same time
+  const uniqueId = uuid().substring(0, 8);
+  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;
 }
 
 export function printReportMsg(filepath: string) {
@@ -140,3 +142,7 @@ export function generateCacheId(fileName?: string): string {
 
 export const ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED =
   'NOT_IMPLEMENTED_AS_DESIGNED';
+
+export function replaceIllegalPathCharsAndSpace(str: string) {
+  return str.replace(/[/\\:*?"<>| ]/g, '-');
+}

packages/web-integration/src/playwright/ai-fixture.ts

@@ -1,11 +1,11 @@
 import { randomUUID } from 'node:crypto';
 import type { PageAgent, PageAgentOpt } from '@/common/agent';
+import { replaceIllegalPathCharsAndSpace } from '@/common/utils';
 import { PlaywrightAgent } from '@/playwright/index';
 import type { AgentWaitForOpt } from '@midscene/core';
 import { getDebug } from '@midscene/shared/logger';
 import { type TestInfo, type TestType, test } from '@playwright/test';
 import type { Page as OriginPlaywrightPage } from 'playwright';
-
 export type APITestType = Pick<TestType<any, any>, 'step'>;
 
 const debugPage = getDebug('web:playwright:ai-fixture');
@@ -25,7 +25,12 @@ const groupAndCaseForTest = (testInfo: TestInfo) => {
     taskTitle = 'unnamed';
     taskFile = 'unnamed';
   }
-  return { taskFile, taskTitle };
+  return {
+    taskFile,
+    taskTitle: replaceIllegalPathCharsAndSpace(
+      `${taskTitle}${testInfo.retry ? `(retry #${testInfo.retry})` : ''}`,
+    ),
+  };
 };
 
 const midsceneAgentKeyId = '_midsceneAgentId';

packages/web-integration/src/playwright/reporter/index.ts

@@ -1,4 +1,8 @@
-import { printReportMsg, reportFileName } from '@/common/utils';
+import {
+  printReportMsg,
+  replaceIllegalPathCharsAndSpace,
+  reportFileName,
+} from '@/common/utils';
 import type { ReportDumpWithAttributes } from '@midscene/core';
 import { writeDumpReport } from '@midscene/core/utils';
 import type {
@@ -17,17 +21,73 @@ function logger(...message: any[]) {
 }
 
 const testDataList: Array<ReportDumpWithAttributes> = [];
-let filename: string;
-function updateReport() {
-  const reportPath = writeDumpReport(filename, testDataList);
-  reportPath && printReportMsg(reportPath);
+let mergedFilename: string;
+const testTitleToFilename: Map<string, string> = new Map();
+
+function getStableFilename(testTitle: string): string {
+  if (!testTitleToFilename.has(testTitle)) {
+    // use reportFileName to generate the base filename
+    // only replace the illegal characters in the file system: /, \, :, *, ?, ", <, >, |
+    const baseTag = `playwright-${replaceIllegalPathCharsAndSpace(testTitle)}`;
+    const generatedFilename = reportFileName(baseTag);
+    testTitleToFilename.set(testTitle, generatedFilename);
+  }
+  return testTitleToFilename.get(testTitle)!;
+}
+
+function updateReport(mode: 'merged' | 'separate', testId?: string) {
+  if (mode === 'separate') {
+    // in separate mode, find the data for the corresponding testID and generate a separate report
+    const testData = testDataList.find(
+      (data) => data.attributes?.playwright_test_id === testId,
+    );
+
+    if (testData) {
+      // use the stable filename
+      const stableFilename = getStableFilename(
+        testData.attributes?.playwright_test_title,
+      );
+
+      const reportPath = writeDumpReport(stableFilename, [testData]);
+      reportPath && printReportMsg(reportPath);
+    }
+  } else if (mode === 'merged') {
+    // in merged mode, write all test data into one file
+    if (!mergedFilename) {
+      mergedFilename = reportFileName('playwright-merged');
+    }
+
+    const reportPath = writeDumpReport(mergedFilename, testDataList);
+    reportPath && printReportMsg(reportPath);
+  } else {
+    throw new Error(
+      `Unknown reporter type in playwright config: ${mode}, only support 'merged' or 'separate'`,
+    );
+  }
+}
+
+function getMode(reporterType: string) {
+  if (!reporterType) {
+    return 'merged';
+  }
+
+  if (reporterType !== 'merged' && reporterType !== 'separate') {
+    throw new Error(
+      `Unknown reporter type in playwright config: ${reporterType}, only support 'merged' or 'separate'`,
+    );
+  }
+
+  return reporterType;
 }
 
 class MidsceneReporter implements Reporter {
+  mode?: 'merged' | 'separate';
+
   async onBegin(config: FullConfig, suite: Suite) {
-    if (!filename) {
-      filename = reportFileName('playwright-merged');
-    }
+    const reporterType = config.reporter?.[1]?.[1]?.type;
+
+    this.mode = getMode(reporterType);
+
     // const suites = suite.allTests();
     // logger(`Starting the run with ${suites.length} tests`);
   }
@@ -41,25 +101,29 @@ class MidsceneReporter implements Reporter {
       return annotation.type === 'MIDSCENE_DUMP_ANNOTATION';
     });
     if (!dumpAnnotation?.description) return;
-    testDataList.push({
+    const retry = result.retry ? `(retry #${result.retry})` : '';
+    const testId = `${test.id}${retry}`;
+    const testData: ReportDumpWithAttributes = {
       dumpString: dumpAnnotation.description,
       attributes: {
-        playwright_test_id: test.id,
-        playwright_test_title: test.title,
+        playwright_test_id: testId,
+        playwright_test_title: `${test.title}${retry}`,
         playwright_test_status: result.status,
         playwright_test_duration: result.duration,
       },
-    });
+    };
+
+    testDataList.push(testData);
+
+    updateReport(this.mode!, testId);
 
     test.annotations = test.annotations.filter(
       (annotation) => annotation.type !== 'MIDSCENE_DUMP_ANNOTATION',
     );
-
-    updateReport();
   }
 
   onEnd(result: FullResult) {
-    updateReport();
+    updateReport(this.mode!);
 
     logger(`Finished the run: ${result.status}`);
   }