feat(core): use append instead string join (#651)

* feat(utils): optimize reportHTMLContent to handle large data with Buffer and newlines

* feat(report): enhance test data handling and filtering in PlaywrightCaseSelector

* feat(report): enhance reportHTMLContent to support file writing and improved dump data handling

* fix(core): correct file writing logic and improve dump data handling in reportHTMLContent

* feat(core): add tests for reportHTMLContent

* feat(core): add performance test for handling multiple large reports in utils

apps/report/rsbuild.config.ts

@@ -5,8 +5,19 @@ import { pluginLess } from '@rsbuild/plugin-less';
 import { pluginNodePolyfill } from '@rsbuild/plugin-node-polyfill';
 import { pluginReact } from '@rsbuild/plugin-react';
 
-const testDataPath = path.join(__dirname, 'test-data', 'swag-lab.json');
-const testData = JSON.parse(fs.readFileSync(testDataPath, 'utf-8'));
+// Read all JSON files from test-data directory
+const testDataDir = path.join(__dirname, 'test-data');
+const jsonFiles = fs
+  .readdirSync(testDataDir)
+  .filter((file) => file.endsWith('.json'));
+const allTestData = jsonFiles.map((file) => {
+  const filePath = path.join(testDataDir, file);
+  const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+  return {
+    fileName: file,
+    data,
+  };
+});
 
 const copyReportTemplate = () => ({
   name: 'copy-report-template',
@@ -34,21 +45,19 @@ export default defineConfig({
     inject: 'body',
     tags:
       process.env.NODE_ENV === 'development'
-        ? [
-            {
-              tag: 'script',
-              attrs: {
-                type: 'midscene_web_dump',
-                playwright_test_name: testData.groupName,
-                playwright_test_description: testData.groupDescription,
-                playwright_test_id: '8465e854a4d9a753cc87-1f096ece43c67754f95a',
-                playwright_test_title: 'test open new tab',
-                playwright_test_status: 'passed',
-                playwright_test_duration: '44274',
-              },
-              children: JSON.stringify(testData),
+        ? allTestData.map((item) => ({
+            tag: 'script',
+            attrs: {
+              type: 'midscene_web_dump',
+              playwright_test_name: item.data.groupName,
+              playwright_test_description: item.data.groupDescription,
+              playwright_test_id: '8465e854a4d9a753cc87-1f096ece43c67754f95a',
+              playwright_test_title: 'test open new tab',
+              playwright_test_status: 'passed',
+              playwright_test_duration: '44274',
             },
-          ]
+            children: JSON.stringify(item.data),
+          }))
         : [],
   },
   resolve: {

apps/report/src/components/PlaywrightCaseSelector.tsx

@@ -1,11 +1,21 @@
 import { DownOutlined, SearchOutlined } from '@ant-design/icons';
 import type { GroupedActionDump } from '@midscene/core';
 import { iconForStatus, timeCostStrElement } from '@midscene/visualizer';
-import { Dropdown, Input } from 'antd';
+import { Dropdown, Input, Select, Space } from 'antd';
 import type React from 'react';
 import { useMemo, useState } from 'react';
 import type { ExecutionDumpWithPlaywrightAttributes } from '../types';
 
+// define all possible test statuses
+const TEST_STATUS_OPTIONS = [
+  { value: 'all', label: 'All' },
+  { value: 'passed', label: 'Passed' },
+  { value: 'failed', label: 'Failed' },
+  { value: 'skipped', label: 'Skipped' },
+  { value: 'timedOut', label: 'Timed Out' },
+  { value: 'interrupted', label: 'Interrupted' },
+];
+
 interface PlaywrightCaseSelectorProps {
   dumps?: ExecutionDumpWithPlaywrightAttributes[];
   selected?: GroupedActionDump | null;
@@ -20,6 +30,7 @@ export function PlaywrightCaseSelector({
   if (!dumps || dumps.length <= 1) return null;
 
   const [searchText, setSearchText] = useState('');
+  const [statusFilter, setStatusFilter] = useState('all');
   const [dropdownVisible, setDropdownVisible] = useState(false);
 
   const nameForDump = (dump: GroupedActionDump) =>
@@ -49,11 +60,24 @@ export function PlaywrightCaseSelector({
   };
 
   const filteredDumps = useMemo(() => {
-    if (!searchText) return dumps || [];
-    return (dumps || []).filter((dump) =>
-      nameForDump(dump).toLowerCase().includes(searchText.toLowerCase()),
-    );
-  }, [dumps, searchText]);
+    let result = dumps || [];
+
+    // apply text filter
+    if (searchText) {
+      result = result.filter((dump) =>
+        nameForDump(dump).toLowerCase().includes(searchText.toLowerCase()),
+      );
+    }
+
+    // apply status filter
+    if (statusFilter !== 'all') {
+      result = result.filter(
+        (dump) => dump.attributes?.playwright_test_status === statusFilter,
+      );
+    }
+
+    return result;
+  }, [dumps, searchText, statusFilter]);
 
   const items = filteredDumps.map((dump, index) => {
     return {
@@ -85,17 +109,30 @@ export function PlaywrightCaseSelector({
     setSearchText(e.target.value);
   };
 
+  const handleStatusChange = (value: string) => {
+    setStatusFilter(value);
+  };
+
   const dropdownRender = (menu: React.ReactNode) => (
     <div>
       <div style={{ padding: '8px' }}>
-        <Input
-          placeholder="Search test case"
-          value={searchText}
-          onChange={handleSearchChange}
-          prefix={<SearchOutlined />}
-          allowClear
-          autoFocus
-        />
+        <Space style={{ width: '100%' }}>
+          <Input
+            placeholder="Search test case"
+            value={searchText}
+            onChange={handleSearchChange}
+            prefix={<SearchOutlined />}
+            allowClear
+            autoFocus
+            style={{ flex: 1 }}
+          />
+          <Select
+            value={statusFilter}
+            onChange={handleStatusChange}
+            style={{ width: 120 }}
+            options={TEST_STATUS_OPTIONS}
+          />
+        </Space>
       </div>
       {menu}
     </div>

packages/core/src/utils.ts

@@ -71,6 +71,7 @@ export function replaceStringWithFirstAppearance(
 
 export function reportHTMLContent(
   dumpData: string | ReportDumpWithAttributes[],
+  reportPath?: string,
 ): string {
   const tpl = getReportTpl();
   if (!tpl) {
@@ -78,46 +79,90 @@ export function reportHTMLContent(
     return '';
   }
 
-  let reportContent: string;
+  const dumpPlaceholder = '{{dump}}';
+
+  // verify the template contains the placeholder
+  if (!tpl.includes(dumpPlaceholder)) {
+    console.warn('Template does not contain {{dump}} placeholder');
+    return '';
+  }
+
+  // find the first placeholder position
+  const placeholderIndex = tpl.indexOf(dumpPlaceholder);
+
+  // split the template into two parts before and after the placeholder
+  const firstPart = tpl.substring(0, placeholderIndex);
+  const secondPart = tpl.substring(placeholderIndex + dumpPlaceholder.length);
+
+  // if reportPath is set, it means we are in write to file mode
+  const writeToFile = reportPath && !ifInBrowser;
+  let resultContent = '';
+
+  // helper function: decide to write to file or append to resultContent
+  const appendOrWrite = (content: string): void => {
+    if (writeToFile) {
+      writeFileSync(reportPath!, `${content}\n`, {
+        flag: 'a',
+      });
+    } else {
+      resultContent += `${content}\n`;
+    }
+  };
+
+  // if writeToFile is true, write the first part to file, otherwise set the first part to the initial value of resultContent
+  if (writeToFile) {
+    writeFileSync(reportPath!, firstPart, { flag: 'w' }); // use 'w' flag to overwrite the existing file
+  } else {
+    resultContent = firstPart;
+  }
+
+  // generate dump content
+  // handle empty data or undefined
   if (
     (Array.isArray(dumpData) && dumpData.length === 0) ||
     typeof dumpData === 'undefined'
   ) {
-    reportContent = replaceStringWithFirstAppearance(
-      tpl,
-      '{{dump}}',
-      '<script type="midscene_web_dump" type="application/json"></script>',
-    );
-  } else if (typeof dumpData === 'string') {
-    reportContent = replaceStringWithFirstAppearance(
-      tpl,
-      '{{dump}}',
-      // biome-ignore lint/style/useTemplate: <explanation>
+    const dumpContent =
+      '<script type="midscene_web_dump" type="application/json"></script>';
+    appendOrWrite(dumpContent);
+  }
+  // handle string type dumpData
+  else if (typeof dumpData === 'string') {
+    const dumpContent =
+      // biome-ignore lint/style/useTemplate: <explanation> do not use template string here, will cause bundle error
       '<script type="midscene_web_dump" type="application/json">\n' +
-        dumpData +
-        '\n</script>',
-    );
-  } else {
-    const dumps = dumpData.map(({ dumpString, attributes }) => {
+      dumpData +
+      '\n</script>';
+    appendOrWrite(dumpContent);
+  }
+  // handle array type dumpData
+  else {
+    // for array, handle each item
+    for (let i = 0; i < dumpData.length; i++) {
+      const { dumpString, attributes } = dumpData[i];
       const attributesArr = Object.keys(attributes || {}).map((key) => {
         return `${key}="${encodeURIComponent(attributes![key])}"`;
       });
-      return (
-        // biome-ignore lint/style/useTemplate: <explanation>
+
+      const dumpContent =
+        // biome-ignore lint/style/useTemplate: <explanation> do not use template string here, will cause bundle error
         '<script type="midscene_web_dump" type="application/json" ' +
         attributesArr.join(' ') +
         '>\n' +
         dumpString +
-        '\n</script>'
-      );
-    });
-    reportContent = replaceStringWithFirstAppearance(
-      tpl,
-      '{{dump}}',
-      dumps.join('\n'),
-    );
+        '\n</script>';
+      appendOrWrite(dumpContent);
+    }
   }
-  return reportContent;
+
+  // add the second part
+  if (writeToFile) {
+    writeFileSync(reportPath!, secondPart, { flag: 'a' });
+    return reportPath!;
+  }
+
+  resultContent += secondPart;
+  return resultContent;
 }
 
 export function writeDumpReport(
@@ -140,12 +185,9 @@ export function writeDumpReport(
     getMidsceneRunSubDir('report'),
     `${fileName}.html`,
   );
-  const reportContent = reportHTMLContent(dumpData);
-  if (!reportContent) {
-    console.warn('reportContent is empty, will not write report');
-    return null;
-  }
-  writeFileSync(reportPath, reportContent);
+
+  reportHTMLContent(dumpData, reportPath);
+
   if (process.env.MIDSCENE_DEBUG_LOG_JSON) {
     writeFileSync(
       `${reportPath}.json`,

packages/core/tests/unit-test/utils.test.ts

@@ -1,9 +1,7 @@
 import { randomUUID } from 'node:crypto';
-import { readFileSync } from 'node:fs';
+import { existsSync, readFileSync, statSync } from 'node:fs';
 import {
-  adaptBboxToRect,
   adaptDoubaoBbox,
-  adaptGeminiBbox,
   adaptQwenBbox,
   expandSearchArea,
   mergeRects,
@@ -96,6 +94,139 @@ describe('utils', () => {
       `<script type="midscene_web_dump" type="application/json">\n${content}\n</script>`,
     );
   });
+
+  it('reportHTMLContent with reportPath', () => {
+    const tmpFile = getTmpFile('html');
+    expect(tmpFile).toBeTruthy();
+
+    if (!tmpFile) {
+      return;
+    }
+
+    // test empty array
+    const reportPathA = reportHTMLContent([], tmpFile);
+    expect(reportPathA).toBe(tmpFile);
+    const fileContentA = readFileSync(tmpFile, 'utf-8');
+    expect(fileContentA).toContain(
+      '<script type="midscene_web_dump" type="application/json"></script>',
+    );
+
+    // test string content
+    const content = JSON.stringify({ test: randomUUID() });
+    const reportPathB = reportHTMLContent(content, tmpFile);
+    expect(reportPathB).toBe(tmpFile);
+    const fileContentB = readFileSync(tmpFile, 'utf-8');
+    expect(fileContentB).toContain(
+      `<script type="midscene_web_dump" type="application/json">\n${content}\n</script>`,
+    );
+
+    // test array with attributes
+    const uuid1 = randomUUID();
+    const uuid2 = randomUUID();
+    const dumpArray = [
+      {
+        dumpString: JSON.stringify({ id: uuid1 }),
+        attributes: {
+          test_attr: 'test_value',
+          another_attr: 'another_value',
+        },
+      },
+      {
+        dumpString: JSON.stringify({ id: uuid2 }),
+        attributes: {
+          test_attr2: 'test_value2',
+        },
+      },
+    ];
+
+    const reportPathC = reportHTMLContent(dumpArray, tmpFile);
+    expect(reportPathC).toBe(tmpFile);
+    const fileContentC = readFileSync(tmpFile, 'utf-8');
+
+    // verify the file content contains attributes and data
+    expect(fileContentC).toContain('test_attr="test_value"');
+    expect(fileContentC).toContain('another_attr="another_value"');
+    expect(fileContentC).toContain('test_attr2="test_value2"');
+    expect(fileContentC).toContain(uuid1);
+    expect(fileContentC).toContain(uuid2);
+  });
+
+  it('should handle multiple large reports correctly', () => {
+    const tmpFile = getTmpFile('html');
+    expect(tmpFile).toBeTruthy();
+
+    if (!tmpFile) {
+      return;
+    }
+
+    // Create a large string of approximately 100MB
+    const generateLargeString = (sizeInMB: number, identifier: string) => {
+      const approximateCharsPer1MB = 1024 * 1024; // 1MB in characters
+      const totalChars = approximateCharsPer1MB * sizeInMB;
+
+      // Create a basic JSON structure with a very large string
+      const baseObj = {
+        id: identifier,
+        timestamp: new Date().toISOString(),
+        data: 'X'.repeat(totalChars - 100), // subtract a small amount for the JSON structure
+      };
+
+      return JSON.stringify(baseObj);
+    };
+
+    // Monitor memory usage
+    const startMemory = process.memoryUsage();
+    console.log(
+      'Memory usage before test:',
+      `RSS: ${Math.round(startMemory.rss / 1024 / 1024)}MB, ` +
+        `Heap Total: ${Math.round(startMemory.heapTotal / 1024 / 1024)}MB, ` +
+        `Heap Used: ${Math.round(startMemory.heapUsed / 1024 / 1024)}MB`,
+    );
+
+    // Store start time
+    const startTime = Date.now();
+
+    // Generate 10 large reports (each ~100MB)
+    const numberOfReports = 10;
+    const dumpArray = Array.from({ length: numberOfReports }).map(
+      (_, index) => ({
+        dumpString: generateLargeString(100, `large-report-${index + 1}`),
+        attributes: {
+          report_number: `${index + 1}`,
+          report_size: '100MB',
+        },
+      }),
+    );
+
+    // Write the large reports
+    const reportPath = reportHTMLContent(dumpArray, tmpFile);
+    expect(reportPath).toBe(tmpFile);
+
+    // Calculate execution time
+    const executionTime = Date.now() - startTime;
+    console.log(`Execution time: ${executionTime}ms`);
+
+    // Check memory usage after test
+    const endMemory = process.memoryUsage();
+    console.log(
+      'Memory usage after test:',
+      `RSS: ${Math.round(endMemory.rss / 1024 / 1024)}MB, ` +
+        `Heap Total: ${Math.round(endMemory.heapTotal / 1024 / 1024)}MB, ` +
+        `Heap Used: ${Math.round(endMemory.heapUsed / 1024 / 1024)}MB`,
+    );
+
+    // Check if file exists
+    expect(existsSync(tmpFile)).toBe(true);
+
+    // Verify file size is approximately (100MB * 10) + template size
+    const stats = statSync(tmpFile);
+    const fileSizeInMB = stats.size / (1024 * 1024);
+    console.log(`File size: ${fileSizeInMB.toFixed(2)}MB`);
+
+    // We expect the file to be approximately 700MB plus template overhead
+    const expectedMinSize = 1000; // 10 reports × 100MB
+    expect(fileSizeInMB).toBeGreaterThan(expectedMinSize);
+  });
 });
 
 describe('extractJSONFromCodeBlock', () => {