refactor: switch bundle type to bundleless (#437)

.github/workflows/release.yml

@@ -38,7 +38,7 @@ jobs:
         cache: 'pnpm'
 
     - name: Cache Puppeteer
-      uses: actions/cache@v4.0.2
+      uses: actions/cache@v4
       with:
         path: ~/.cache/puppeteer
         key: ${{ runner.os }}-puppeteer-${{ hashFiles('**/package-lock.json') }}

packages/core/modern.config.ts

@@ -5,18 +5,10 @@ export default defineConfig({
   plugins: [moduleTools()],
   buildPreset: 'npm-library',
   buildConfig: {
-    format: 'cjs',
-    input: {
-      index: 'src/index.ts',
-      env: 'src/env.ts',
-      utils: 'src/utils.ts',
-      tree: 'src/tree.ts',
-      'ai-model': 'src/ai-model/index.ts',
-      evaluation: 'src/evaluation.ts',
-    },
-    outDir: 'dist/lib',
+    buildType: 'bundleless',
+    format: 'esm',
     externals: ['langsmith'],
-    target: 'es2018',
+    target: 'es2020',
     define: {
       __VERSION__: version,
     },

packages/core/package.json

@@ -5,26 +5,45 @@
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
-  "type": "commonjs",
-  "main": "./dist/lib/index.js",
-  "types": "./dist/lib/types/index.d.ts",
+  "type": "module",
+  "main": "./dist/es/index.js",
+  "types": "./dist/types/index.d.ts",
   "files": ["dist", "report", "README.md"],
   "exports": {
-    ".": "./dist/lib/index.js",
-    "./env": "./dist/lib/env.js",
-    "./utils": "./dist/lib/utils.js",
-    "./ai-model": "./dist/lib/ai-model.js",
-    "./tree": "./dist/lib/tree.js",
-    "./evaluation": "./dist/lib/evaluation.js"
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "require": "./dist/lib/index.js",
+      "import": "./dist/es/index.js"
+    },
+    "./env": {
+      "types": "./dist/types/env.d.ts",
+      "import": "./dist/es/env.js",
+      "require": "./dist/lib/env.js"
+    },
+    "./utils": {
+      "types": "./dist/types/utils.d.ts",
+      "import": "./dist/es/utils.js",
+      "require": "./dist/lib/utils.js"
+    },
+    "./ai-model": {
+      "types": "./dist/types/ai-model/index.d.ts",
+      "import": "./dist/es/ai-model/index.js",
+      "require": "./dist/lib/ai-model/index.js"
+    },
+    "./tree": {
+      "types": "./dist/types/tree.d.ts",
+      "import": "./dist/es/tree.js",
+      "require": "./dist/lib/tree.js"
+    }
   },
   "typesVersions": {
     "*": {
-      ".": ["./dist/lib/types/index.d.ts"],
-      "env": ["./dist/lib/types/env.d.ts"],
-      "utils": ["./dist/lib/types/utils.d.ts"],
-      "ai-model": ["./dist/lib/types/ai-model.d.ts"],
-      "tree": ["./dist/lib/types/tree.d.ts"],
-      "evaluation": ["./dist/lib/types/evaluation.d.ts"]
+      ".": ["./dist/types/index.d.ts"],
+      "env": ["./dist/types/env.d.ts"],
+      "utils": ["./dist/types/utils.d.ts"],
+      "ai-model": ["./dist/types/ai-model/index.d.ts"],
+      "tree": ["./dist/types/tree.d.ts"],
+      "evaluation": ["./dist/types/evaluation.d.ts"]
     }
   },
   "scripts": {

packages/core/src/ai-model/action-executor.ts

@@ -1,4 +1,3 @@
-import assert from 'node:assert';
 import { MIDSCENE_MODEL_NAME, getAIConfig } from '@/env';
 import type {
   ExecutionDump,
@@ -10,6 +9,7 @@ import type {
   ExecutorContext,
 } from '@/types';
 import { getVersion } from '@/utils';
+import { assert } from '@midscene/shared/utils';
 
 export class Executor {
   name: string;

packages/core/src/ai-model/common.ts

@@ -1,5 +1,5 @@
-import assert from 'node:assert';
 import type { AIUsageInfo, Size } from '@/types';
+import { assert } from '@midscene/shared/utils';
 
 import type {
   ChatCompletionSystemMessageParam,

packages/core/src/ai-model/inspect.ts

@@ -1,4 +1,3 @@
-import assert from 'node:assert';
 import {
   MIDSCENE_USE_QWEN_VL,
   MIDSCENE_USE_VLM_UI_TARS,
@@ -19,6 +18,7 @@ import type {
   UIContext,
 } from '@/types';
 import { paddingToMatchBlock } from '@midscene/shared/img';
+import { assert } from '@midscene/shared/utils';
 import type {
   ChatCompletionSystemMessageParam,
   ChatCompletionUserMessageParam,

packages/core/src/ai-model/llm-planning.ts

@@ -1,7 +1,7 @@
-import assert from 'node:assert';
 import { MIDSCENE_USE_QWEN_VL, getAIConfigInBoolean } from '@/env';
 import type { PlanningAIResponse, UIContext } from '@/types';
 import { paddingToMatchBlock } from '@midscene/shared/img';
+import { assert } from '@midscene/shared/utils';
 import {
   AIActionType,
   type AIArgs,

packages/core/src/ai-model/prompt/util.ts

@@ -1,9 +1,9 @@
-import assert from 'node:assert';
 import { MATCH_BY_POSITION, getAIConfigInBoolean } from '@/env';
 import { imageInfoOfBase64 } from '@/image';
 import type { BaseElement, ElementTreeNode, Size, UIContext } from '@/types';
 import { NodeType } from '@midscene/shared/constants';
 import { descriptionOfTree, treeToList } from '@midscene/shared/extractor';
+import { assert } from '@midscene/shared/utils';
 import { generateHashId } from '@midscene/shared/utils';
 
 export function describeSize(size: Size) {

packages/core/src/ai-model/service-caller/index.ts

@@ -1,10 +1,10 @@
-import assert from 'node:assert';
 import { AIResponseFormat, type AIUsageInfo } from '@/types';
 import { Anthropic } from '@anthropic-ai/sdk';
 import {
   DefaultAzureCredential,
   getBearerTokenProvider,
 } from '@azure/identity';
+import { assert } from '@midscene/shared/utils';
 import { ifInBrowser } from '@midscene/shared/utils';
 import dJSON from 'dirty-json';
 import OpenAI, { AzureOpenAI } from 'openai';

packages/core/src/ai-model/ui-tars-planning.ts

@@ -1,6 +1,6 @@
-import assert from 'node:assert';
 import type { PlanningAction } from '@/types';
 import { transformHotkeyInput } from '@midscene/shared/keyboard-layout';
+import { assert } from '@midscene/shared/utils';
 import { actionParser } from '@ui-tars/action-parser';
 import type { ChatCompletionMessageParam } from 'openai/resources';
 import { AIActionType } from './common';

packages/core/src/index.ts

@@ -12,6 +12,12 @@ export {
 
 export { getAIConfig, MIDSCENE_MODEL_NAME } from './env';
 
-export * from './types';
+export type * from './types';
 export default Insight;
 export { Executor, setLogDir, getLogDirByType, Insight, getVersion };
+
+export type {
+  MidsceneYamlScript,
+  MidsceneYamlTask,
+  MidsceneYamlFlowItem,
+} from './yaml';

packages/core/src/insight/index.ts

@@ -1,4 +1,3 @@
-import assert from 'node:assert';
 import { callAiFn } from '@/ai-model/common';
 import { AiExtractElementInfo, AiInspectElement } from '@/ai-model/index';
 import { AiAssert } from '@/ai-model/inspect';
@@ -15,6 +14,7 @@ import type {
   PartialInsightDumpFromSDK,
   UIContext,
 } from '@/types';
+import { assert } from '@midscene/shared/utils';
 import { emitInsightDump } from './utils';
 
 export interface LocateOpts {

packages/core/src/insight/utils.ts

@@ -1,4 +1,3 @@
-import assert from 'node:assert';
 import {
   MIDSCENE_MODEL_NAME,
   MIDSCENE_USE_QWEN_VL,
@@ -13,6 +12,7 @@ import type {
   PartialInsightDumpFromSDK,
 } from '@/types';
 import { getLogDir, getVersion, stringifyDumpData } from '@/utils';
+import { assert } from '@midscene/shared/utils';
 import { uuid } from '@midscene/shared/utils';
 const logContent: string[] = [];
 const logIdIndexMap: Record<string, number> = {};

packages/core/src/types.ts

@@ -3,7 +3,7 @@
 import type { NodeType } from '@midscene/shared/constants';
 import type { ChatCompletionMessageParam } from 'openai/resources';
 
-export * from './yaml.d';
+export * from './yaml';
 
 export interface Point {
   left: number;

packages/core/src/utils.ts

@@ -1,9 +1,11 @@
-import assert from 'node:assert';
 import { execSync } from 'node:child_process';
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { basename, dirname, join } from 'node:path';
+import * as path from 'node:path';
+import { dirname } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { getRunningPkgInfo } from '@midscene/shared/fs';
+import { assert } from '@midscene/shared/utils';
 import { ifInBrowser, uuid } from '@midscene/shared/utils';
 import {
   MIDSCENE_DEBUG_MODE,
@@ -13,7 +15,7 @@ import {
 } from './env';
 import type { Rect, ReportDumpWithAttributes } from './types';
 
-let logDir = join(process.cwd(), './midscene_run/');
+let logDir = path.join(process.cwd(), './midscene_run/');
 let logEnvReady = false;
 export const groupedActionDumpFileExt = 'web-dump.json';
 
@@ -26,7 +28,10 @@ export function setLogDir(dir: string) {
 }
 
 export function getLogDirByType(type: 'dump' | 'cache' | 'report' | 'tmp') {
-  const dir = join(getLogDir(), type);
+  if (ifInBrowser) {
+    return '';
+  }
+  const dir = path.join(getLogDir(), type);
   if (!existsSync(dir)) {
     mkdirSync(dir, { recursive: true });
   }
@@ -39,17 +44,15 @@ function getReportTpl() {
     if (!reportTpl && (window as any).get_midscene_report_tpl) {
       reportTpl = (window as any).get_midscene_report_tpl();
     }
-    // assert(
-    //   reportTpl,
-    //   'reportTpl should be set before writing report in browser',
-    // );
     return reportTpl;
   }
 
+  const filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(filename);
   if (!reportTpl) {
-    let reportPath = join(__dirname, '../../report/index.html');
+    let reportPath = path.join(__dirname, '../../report/index.html');
     if (!existsSync(reportPath)) {
-      reportPath = join(__dirname, '../report/index.html');
+      reportPath = path.join(__dirname, '../report/index.html');
     }
     reportTpl = readFileSync(reportPath, 'utf-8');
   }
@@ -116,13 +119,15 @@ export function writeDumpReport(
     return null;
   }
 
+  const filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(filename);
   const midscenePkgInfo = getRunningPkgInfo(__dirname);
   if (!midscenePkgInfo) {
     console.warn('midscenePkgInfo not found, will not write report');
     return null;
   }
 
-  const reportPath = join(getLogDirByType('report'), `${fileName}.html`);
+  const reportPath = path.join(getLogDirByType('report'), `${fileName}.html`);
   const reportContent = reportHTMLContent(dumpData);
   if (!reportContent) {
     console.warn('reportContent is empty, will not write report');
@@ -150,14 +155,14 @@ export function writeLogFile(opts: {
     assert(targetDir, 'logDir should be set before writing dump file');
 
     // gitIgnore in the parent directory
-    const gitIgnorePath = join(targetDir, '../../.gitignore');
+    const gitIgnorePath = path.join(targetDir, '../../.gitignore');
     let gitIgnoreContent = '';
     if (existsSync(gitIgnorePath)) {
       gitIgnoreContent = readFileSync(gitIgnorePath, 'utf-8');
     }
 
     // ignore the log folder
-    const logDirName = basename(logDir);
+    const logDirName = path.basename(logDir);
     if (!gitIgnoreContent.includes(`${logDirName}/`)) {
       writeFileSync(
         gitIgnorePath,
@@ -168,11 +173,11 @@ export function writeLogFile(opts: {
     logEnvReady = true;
   }
 
-  const filePath = join(targetDir, `${fileName}.${fileExt}`);
+  const filePath = path.join(targetDir, `${fileName}.${fileExt}`);
 
   if (type !== 'dump') {
     // do not write dump file any more
-    const outputResourceDir = dirname(filePath);
+    const outputResourceDir = path.dirname(filePath);
     if (!existsSync(outputResourceDir)) {
       mkdirSync(outputResourceDir, { recursive: true });
     }
@@ -188,17 +193,18 @@ export function writeLogFile(opts: {
 }
 
 export function getTmpDir(): string | null {
-  if (ifInBrowser) {
+  try {
+    const runningPkgInfo = getRunningPkgInfo();
+    if (!runningPkgInfo) {
+      return null;
+    }
+    const { name } = runningPkgInfo;
+    const tmpPath = path.join(tmpdir(), name);
+    mkdirSync(tmpPath, { recursive: true });
+    return tmpPath;
+  } catch (e) {
     return null;
   }
-  const runningPkgInfo = getRunningPkgInfo();
-  if (!runningPkgInfo) {
-    return null;
-  }
-  const { name } = runningPkgInfo;
-  const path = join(tmpdir(), name);
-  mkdirSync(path, { recursive: true });
-  return path;
 }
 
 export function getTmpFile(fileExtWithoutDot: string): string | null {
@@ -207,7 +213,7 @@ export function getTmpFile(fileExtWithoutDot: string): string | null {
   }
   const tmpDir = getTmpDir();
   const filename = `${uuid()}.${fileExtWithoutDot}`;
-  return join(tmpDir!, filename);
+  return path.join(tmpDir!, filename);
 }
 
 export function overlapped(container: Rect, target: Rect) {

packages/core/tests/ai/assert/assert.test.ts

@@ -1,5 +1,5 @@
 import { AiAssert } from '@/ai-model';
-import { getContextFromFixture } from '@/evaluation';
+import { getContextFromFixture } from 'tests/evaluation';
 /* eslint-disable max-lines-per-function */
 import { describe, expect, it, vi } from 'vitest';
 

packages/core/tests/ai/extract/extract.test.ts

@@ -1,5 +1,5 @@
 import { AiExtractElementInfo } from '@/ai-model';
-import { getContextFromFixture } from '@/evaluation';
+import { getContextFromFixture } from 'tests/evaluation';
 import { describe, expect, it, vi } from 'vitest';
 
 vi.setConfig({

packages/core/tests/ai/llm-inspect.test.ts

@@ -1,5 +1,5 @@
 import { AiInspectElement } from '@/ai-model';
-import { getContextFromFixture } from '@/evaluation';
+import { getContextFromFixture } from 'tests/evaluation';
 import { expect, test } from 'vitest';
 
 test(

packages/core/tests/ai/llm-planning/basic.test.ts

@@ -1,6 +1,6 @@
 import { plan } from '@/ai-model';
 import { MIDSCENE_USE_QWEN_VL, getAIConfigInBoolean } from '@/env';
-import { getContextFromFixture } from '@/evaluation';
+import { getContextFromFixture } from 'tests/evaluation';
 /* eslint-disable max-lines-per-function */
 import { describe, expect, it, vi } from 'vitest';
 

packages/core/tests/ai/llm-planning/input.test.ts

@@ -1,6 +1,6 @@
 import { plan } from '@/ai-model';
-import { getContextFromFixture } from '@/evaluation';
 import type { PlanningAction } from '@/types';
+import { getContextFromFixture } from 'tests/evaluation';
 /* eslint-disable max-lines-per-function */
 import { describe, expect, it, vi } from 'vitest';
 

packages/core/tests/ai/ui-tars-planning/plan-to-target.test.ts

@@ -1,7 +1,7 @@
 import path from 'node:path';
 import { vlmPlanning } from '@/ai-model/ui-tars-planning';
-import { getContextFromFixture } from '@/evaluation';
 import { savePositionImg } from '@midscene/shared/img';
+import { getContextFromFixture } from 'tests/evaluation';
 import { assert, describe, expect, it, test } from 'vitest';
 
 const isUiTars = process.env.MIDSCENE_USE_VLM_UI_TARS === '1';

packages/core/tests/tsconfig.json

@@ -4,6 +4,6 @@
     "baseUrl": "../",
     "rootDir": "../"
   },
-  "include": ["**/*", "../src", "../../evaluation/tests/test-analyzer.ts"],
+  "include": ["**/*", "../src", "evaluation.ts"],
   "exclude": ["**/node_modules"]
 }

packages/core/tests/unit-test/prompt/utils.test.ts

@@ -1,6 +1,6 @@
 import { describeUserPage } from '@/ai-model/prompt/util';
 import { getAIConfigInBoolean } from '@/env';
-import { getContextFromFixture } from '@/evaluation';
+import { getContextFromFixture } from 'tests/evaluation';
 import { describe, expect, it } from 'vitest';
 
 describe('prompt utils', () => {

packages/core/tsconfig.json

@@ -14,12 +14,12 @@
       "@/*": ["./src/*"]
     },
     "resolveJsonModule": true,
-    "rootDir": ".",
+    "rootDir": "./src",
     "skipLibCheck": true,
     "strict": true,
     "module": "ESNext",
     "target": "es2018"
   },
   "exclude": ["**/node_modules"],
-  "include": ["src", "tests", "report"]
+  "include": ["src", "report"]
 }

packages/evaluation/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "evaluation",
+  "name": "@midscene/evaluation",
   "private": true,
   "scripts": {
     "update-page-data:headless": "playwright test ./data-generator/generator-headless.spec.ts && npm run format",
@@ -15,6 +15,8 @@
     "evaluate:assertion": "npx vitest --run tests/assertion.test.ts",
     "format": "cd ../.. && npm run lint"
   },
+  "files": ["dist", "README.md"],
+  "type": "module",
   "dependencies": {
     "@midscene/core": "workspace:*",
     "@midscene/shared": "workspace:*",

packages/evaluation/src/test-analyzer.ts

@@ -15,7 +15,7 @@ import {
   type plan,
 } from '@midscene/core';
 import { expect } from 'vitest';
-import type { TestCase } from './util';
+import type { TestCase } from '../tests/util';
 
 type ActualResult =
   | Awaited<ReturnType<typeof AiInspectElement>>
@@ -115,13 +115,20 @@ ${errorMsg ? `Error: ${errorMsg}` : ''}
         const totalTimeCost = testLogs.reduce((acc, log) => acc + log.cost, 0);
         const averagePromptTokens =
           testLogs.reduce(
-            (acc, log) => acc + (log.actualResult.usage?.prompt_tokens || 0),
+            (acc, log) =>
+              acc +
+              (log.actualResult instanceof Error
+                ? 0
+                : log.actualResult.usage?.prompt_tokens || 0),
             0,
           ) / testLogs.length;
         const averageCompletionTokens =
           testLogs.reduce(
             (acc, log) =>
-              acc + (log.actualResult.usage?.completion_tokens || 0),
+              acc +
+              (log.actualResult instanceof Error
+                ? 0
+                : log.actualResult.usage?.completion_tokens || 0),
             0,
           ) / testLogs.length;
         return {

packages/evaluation/tests/llm-locator.test.ts

@@ -9,7 +9,7 @@ import { sleep } from '@midscene/core/utils';
 import { saveBase64Image } from '@midscene/shared/img';
 import dotenv from 'dotenv';
 import { afterAll, expect, test } from 'vitest';
-import { TestResultCollector } from './test-analyzer';
+import { TestResultCollector } from '../src/test-analyzer';
 import { annotatePoints, buildContext, getCases } from './util';
 
 dotenv.config({

packages/evaluation/tests/llm-planning.test.ts

@@ -13,7 +13,7 @@ import {
 import { sleep } from '@midscene/core/utils';
 import dotenv from 'dotenv';
 import { describe, expect, test } from 'vitest';
-import { TestResultCollector } from './test-analyzer';
+import { TestResultCollector } from '../src/test-analyzer';
 import { buildContext, getCases } from './util';
 
 dotenv.config({

packages/shared/modern.config.ts

@@ -1,41 +1,11 @@
 import { defineConfig, moduleTools } from '@modern-js/module-tools';
 
-const commonConfig = {
-  input: {
-    index: './src/index.ts',
-    img: './src/img/index.ts',
-    constants: './src/constants/index.ts',
-    extractor: './src/extractor/index.ts',
-    'extractor-debug': './src/extractor/debug.ts',
-    fs: './src/fs/index.ts',
-    utils: './src/utils.ts',
-    'us-keyboard-layout': './src/us-keyboard-layout.ts',
-  },
-};
-
 export default defineConfig({
   plugins: [moduleTools()],
   buildPreset: 'npm-library',
-  buildConfig: [
-    {
-      platform: 'node',
-      format: 'cjs',
-      ...commonConfig,
-      outDir: 'dist/lib',
-      target: 'es6',
-    },
-    {
-      platform: 'node',
-      format: 'esm',
-      ...commonConfig,
-      outDir: 'dist/es',
-      target: 'es6',
-    },
-    {
-      platform: 'browser',
-      ...commonConfig,
-      outDir: 'dist/browser',
-      target: 'es6',
-    },
-  ],
+  buildConfig: {
+    buildType: 'bundleless',
+    format: 'esm',
+    target: 'es2020',
+  },
 });

packages/shared/package.json

@@ -3,68 +3,63 @@
   "version": "0.12.4",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
-  "types": "./src/index.ts",
-  "type": "commonjs",
+  "types": "./dist/types/index.d.ts",
+  "type": "module",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
   "exports": {
     ".": {
-      "types": "./src/index.ts",
-      "require": "./dist/lib/index.js",
-      "import": "./dist/es/index.js"
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/es/index.js",
+      "require": "./dist/lib/index.js"
     },
     "./constants": {
-      "types": "./src/constants/index.ts",
-      "require": "./dist/lib/constants.js",
-      "import": "./dist/es/constants.js"
+      "types": "./dist/types/constants/index.d.ts",
+      "import": "./dist/es/constants/index.js",
+      "require": "./dist/lib/constants/index.js"
     },
     "./fs": {
-      "types": "./src/fs/index.ts",
-      "require": "./dist/lib/fs.js",
-      "import": "./dist/es/fs.js"
+      "types": "./dist/types/node/fs.d.ts",
+      "require": "./dist/lib/node/fs.js",
+      "import": "./dist/es/node/fs.js"
     },
     "./img": {
-      "types": "./src/img/index.ts",
-      "require": "./dist/lib/img.js",
-      "import": "./dist/es/img.js"
-    },
-    "./browser/img": {
-      "types": "./src/img/index.ts",
-      "require": "./dist/browser/img.js",
-      "import": "./dist/browser/img.js"
+      "types": "./dist/types/img/index.d.ts",
+      "import": "./dist/es/img/index.js",
+      "require": "./dist/lib/img/index.js"
     },
     "./utils": {
-      "types": "./src/utils.ts",
-      "require": "./dist/lib/utils.js",
-      "import": "./dist/es/utils.js"
+      "types": "./dist/types/utils.d.ts",
+      "import": "./dist/es/utils.js",
+      "require": "./dist/lib/utils.js"
     },
     "./extractor": {
-      "types": "./src/extractor/index.ts",
-      "require": "./dist/lib/extractor.js",
-      "import": "./dist/es/extractor.js"
+      "types": "./dist/types/extractor/index.d.ts",
+      "import": "./dist/es/extractor/index.js",
+      "require": "./dist/lib/extractor/index.js"
     },
     "./extractor-debug": {
-      "types": "./src/extractor/debug.ts",
-      "require": "./dist/lib/extractor-debug.js",
-      "import": "./dist/es/extractor-debug.js"
+      "types": "./dist/types/extractor/debug.d.ts",
+      "import": "./dist/es/extractor-debug/index.js",
+      "require": "./dist/lib/extractor-debug/index.js"
     },
     "./keyboard-layout": {
-      "types": "./src/us-keyboard-layout.ts",
-      "require": "./dist/lib/us-keyboard-layout.js",
-      "import": "./dist/es/us-keyboard-layout.js"
+      "types": "./dist/types/us-keyboard-layout.d.ts",
+      "import": "./dist/es/us-keyboard-layout.js",
+      "require": "./dist/lib/us-keyboard-layout.js"
     }
   },
   "typesVersions": {
     "*": {
-      ".": ["./src/index.ts"],
-      "constants": ["./src/constants/index.ts"],
-      "img": ["./src/img/index.ts"],
-      "browser/img": ["./src/img/index.ts"],
-      "fs": ["./src/fs/index.ts"],
-      "utils": ["./src/utils.ts"],
-      "extractor": ["./src/extractor/index.ts"],
-      "extractor-debug": ["./src/extractor/debug.ts"],
-      "keyboard-layout": ["./src/us-keyboard-layout.ts"]
+      ".": ["./dist/types/index.d.ts"],
+      "constants": ["./dist/types/constants/index.d.ts"],
+      "img": ["./dist/types/img/index.d.ts"],
+      "browser/img": ["./dist/types/browser/img/index.d.ts"],
+      "fs": ["./dist/types/node/fs.d.ts"],
+      "utils": ["./dist/types/utils.d.ts"],
+      "extractor": ["./dist/types/extractor/index.d.ts"],
+      "extractor-debug": ["./dist/types/extractor/debug.d.ts"],
+      "keyboard-layout": ["./dist/types/us-keyboard-layout.d.ts"]
     }
   },
   "files": ["dist", "src", "README.md"],
@@ -88,7 +83,8 @@
     "test:ai": "AITEST=true npm run test"
   },
   "dependencies": {
-    "jimp": "0.22.12"
+    "jimp": "0.22.12",
+    "js-sha256": "0.11.0"
   },
   "peerDependencies": {},
   "devDependencies": {
@@ -96,72 +92,11 @@
     "typescript": "~5.0.4",
     "@types/node": "^18.0.0",
     "rimraf": "~3.0.2",
-    "vitest": "3.0.5",
-    "js-sha256": "0.11.0"
+    "vitest": "3.0.5"
   },
   "sideEffects": [],
   "publishConfig": {
     "access": "public",
-    "registry": "https://registry.npmjs.org/",
-    "exports": {
-      ".": {
-        "types": "./dist/lib/index.d.ts",
-        "require": "./dist/lib/index.js",
-        "import": "./dist/es/index.js"
-      },
-      "./constants": {
-        "types": "./dist/lib/constants.d.ts",
-        "require": "./dist/lib/constants.js",
-        "import": "./dist/es/constants.js"
-      },
-      "./fs": {
-        "types": "./dist/lib/fs.d.ts",
-        "require": "./dist/lib/fs.js",
-        "import": "./dist/es/fs.js"
-      },
-      "./img": {
-        "types": "./dist/lib/img.d.ts",
-        "require": "./dist/lib/img.js",
-        "import": "./dist/es/img.js"
-      },
-      "./browser/img": {
-        "types": "./dist/browser/img.d.ts",
-        "require": "./dist/browser/img.js",
-        "import": "./dist/browser/img.js"
-      },
-      "./utils": {
-        "types": "./dist/lib/utils.d.ts",
-        "require": "./dist/lib/utils.js",
-        "import": "./dist/es/utils.js"
-      },
-      "./extractor": {
-        "types": "./dist/lib/extractor.d.ts",
-        "require": "./dist/lib/extractor.js",
-        "import": "./dist/es/extractor.js"
-      },
-      "./extractor-debug": {
-        "types": "./dist/lib/extractor-debug.d.ts",
-        "require": "./dist/lib/extractor-debug.js",
-        "import": "./dist/es/extractor-debug.js"
-      },
-      "./keyboard-layout": {
-        "types": "./dist/lib/us-keyboard-layout.d.ts",
-        "require": "./dist/lib/us-keyboard-layout.js",
-        "import": "./dist/es/us-keyboard-layout.js"
-      }
-    },
-    "typesVersions": {
-      "*": {
-        ".": ["./dist/lib/index.d.ts"],
-        "constants": ["./dist/lib/constants.d.ts"],
-        "img": ["./dist/lib/img.d.ts"],
-        "browser/img": ["./dist/browser/img.d.ts"],
-        "fs": ["./dist/lib/fs.d.ts"],
-        "utils": ["./dist/lib/utils.d.ts"],
-        "extractor": ["./dist/lib/extractor.d.ts"],
-        "extractor-debug": ["./dist/lib/extractor-debug.d.ts"],
-        "keyboard-layout": ["./dist/lib/us-keyboard-layout.d.ts"]
-      }
-    }
+    "registry": "https://registry.npmjs.org/"
   }
 }

packages/shared/src/img/box-select.ts

@@ -62,17 +62,17 @@ const createSvgOverlay = async (
       paddedRect.top,
       paddedRect.width,
       paddedRect.height,
-      function (x, y, idx) {
+      (x: number, y: number, idx: number): void => {
         if (
           x === paddedRect.left ||
           x === paddedRect.left + paddedRect.width - 1 ||
           y === paddedRect.top ||
           y === paddedRect.top + paddedRect.height - 1
         ) {
-          this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
-          this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
-          this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
-          this.bitmap.data[idx + 3] = color.rect & 0xff; // A
+          image.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
+          image.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
+          image.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
+          image.bitmap.data[idx + 3] = color.rect & 0xff; // A
         }
       },
     );
@@ -150,12 +150,18 @@ const createSvgOverlay = async (
     // Note: If the original left position doesn't overlap and is within bounds, we keep it as is
 
     // Draw text background
-    image.scan(rectX, rectY, rectWidth, rectHeight, function (x, y, idx) {
-      this.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
-      this.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
-      this.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
-      this.bitmap.data[idx + 3] = color.rect & 0xff; // A
-    });
+    image.scan(
+      rectX,
+      rectY,
+      rectWidth,
+      rectHeight,
+      (x: number, y: number, idx: number): void => {
+        image.bitmap.data[idx + 0] = (color.rect >> 24) & 0xff; // R
+        image.bitmap.data[idx + 1] = (color.rect >> 16) & 0xff; // G
+        image.bitmap.data[idx + 2] = (color.rect >> 8) & 0xff; // B
+        image.bitmap.data[idx + 3] = color.rect & 0xff; // A
+      },
+    );
     // Draw text (simplified, as Jimp doesn't have built-in text drawing)
     try {
       cachedFont = cachedFont || (await Jimp.loadFont(Jimp.FONT_SANS_16_WHITE));

packages/shared/src/img/draw-box.ts

@@ -25,16 +25,16 @@ export async function drawBoxOnImage(options: {
     Math.floor(centerY - radius),
     radius * 2,
     radius * 2,
-    function (x, y, idx) {
+    (x: number, y: number, idx: number) => {
       // Calculate distance from current pixel to center
       const distance = Math.sqrt((x - centerX) ** 2 + (y - centerY) ** 2);
 
       // If distance is less than radius, color the pixel
       if (distance <= radius) {
-        this.bitmap.data[idx + 0] = color.r;
-        this.bitmap.data[idx + 1] = color.g;
-        this.bitmap.data[idx + 2] = color.b;
-        this.bitmap.data[idx + 3] = color.a;
+        image.bitmap.data[idx + 0] = color.r;
+        image.bitmap.data[idx + 1] = color.g;
+        image.bitmap.data[idx + 2] = color.b;
+        image.bitmap.data[idx + 3] = color.a;
       }
     },
   );

packages/shared/src/img/get-jimp.ts

@@ -1,11 +1,14 @@
+// @ts-ignore
 import type Jimp from 'jimp/browser/lib/jimp.js';
 
 const ifInBrowser = typeof window !== 'undefined';
 export default async function getJimp(): Promise<typeof Jimp> {
   if (ifInBrowser) {
+    // @ts-ignore
     await import('jimp/browser/lib/jimp.js');
     return (window as any).Jimp;
   }
   // return Jimp;
+  // @ts-ignore
   return (await import('jimp')).default;
 }

packages/shared/src/node/fs.ts

@@ -1,5 +1,8 @@
 import { existsSync, readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { assert } from '../utils';
 
 interface PkgInfo {
   name: string;
@@ -9,6 +12,7 @@ interface PkgInfo {
 
 const pkgCacheMap: Record<string, PkgInfo> = {};
 const ifInBrowser = typeof window !== 'undefined';
+
 export function getRunningPkgInfo(dir?: string): PkgInfo | null {
   if (ifInBrowser) {
     return null;
@@ -57,3 +61,19 @@ export function findNearestPackageJson(dir: string): string | null {
 
   return findNearestPackageJson(parentDir);
 }
+
+export async function getExtraReturnLogic(tree = false) {
+  if (ifInBrowser) {
+    return null;
+  }
+  // Get __dirname equivalent in ESM
+  const filename = fileURLToPath(import.meta.url);
+  const pathDir = findNearestPackageJson(dirname(filename));
+  assert(pathDir, `can't find pathDir, with ${dirname}`);
+  const scriptPath = path.join(pathDir, './dist/script/htmlElement.js');
+  const elementInfosScriptContent = readFileSync(scriptPath, 'utf-8');
+  if (tree) {
+    return `${elementInfosScriptContent}midscene_element_inspector.webExtractNodeTree()`;
+  }
+  return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
+}

packages/shared/src/utils.ts

@@ -44,3 +44,16 @@ export function generateHashId(rect: any, content = '') {
   }
   return slicedHash;
 }
+
+/**
+ * A utility function that asserts a condition and throws an error with a message if the condition is false.
+ *
+ * @param condition - The condition to assert
+ * @param message - The error message to throw if the condition is false
+ * @throws Error with the provided message if the condition is false
+ */
+export function assert(condition: any, message?: string): asserts condition {
+  if (!condition) {
+    throw new Error(message || 'Assertion failed');
+  }
+}

packages/shared/tsconfig.json

@@ -13,7 +13,10 @@
     "resolveJsonModule": true,
     "rootDir": "src",
     "skipLibCheck": true,
-    "strict": true
+    "strict": true,
+    "module": "ES2020",
+    "target": "es2020",
+    "types": ["node"]
   },
   "exclude": ["**/node_modules"],
   "include": ["src", "playwright.config.ts"]

packages/visualizer/html/report.html

@@ -21,9 +21,9 @@
     <!-- -->
     {{js}}
 
-    <script>
+    <script type="module">
         console.log(midsceneVisualizer);
-        midsceneVisualizer.default.mount('app');
+        midsceneVisualizer.mount('app');
     </script>
 
 </body>

packages/visualizer/modern.config.ts

@@ -10,7 +10,7 @@ const commonConfig = {
   },
   autoExternal: false,
   externals: [...externals],
-  target: 'es2018',
+  target: 'es2020',
   minify: process.env.CI
     ? {
         compress: true,
@@ -33,15 +33,9 @@ export default defineConfig({
       input: {
         report: 'src/index.tsx',
       },
-      umdModuleName: (path) => {
-        // if (path.includes('playground')) {
-        //   return 'midscenePlayground';
-        // }
-        return 'midsceneVisualizer';
-      },
       platform: 'browser',
       outDir: 'dist',
-      target: 'es2018',
+      target: 'es2020',
       sourceMap: true,
     },
     {
@@ -60,7 +54,7 @@ export default defineConfig({
       },
       platform: 'browser',
       outDir: 'unpacked-extension/lib',
-      target: 'es2018',
+      target: 'es2020',
     },
   ],
   plugins: [

packages/visualizer/package.json

@@ -6,6 +6,7 @@
   "types": "./dist/types/index.d.ts",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
+  "type": "module",
   "files": ["dist", "html", "README.md"],
   "watch": {
     "build": {

packages/visualizer/scripts/build-html.ts

@@ -1,7 +1,8 @@
 import { strict as assert } from 'node:assert';
+import { execSync } from 'node:child_process';
 import { rmSync, writeFileSync } from 'node:fs';
-import { join } from 'node:path';
-import { execa } from 'execa';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
 import {
   ensureDirectoryExistence,
   fileContentOfPath,
@@ -9,6 +10,9 @@ import {
   tplReplacer,
 } from './building-utils';
 
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
 const demoData = ['demo', 'demo-mobile', 'zero-execution'];
 
 const outputExtensionUnpackedBaseDir = join(__dirname, '../unpacked-extension');
@@ -55,7 +59,7 @@ function emptyDumpReportHTML() {
   html = replaceStringWithFirstAppearance(
     html,
     '{{js}}',
-    `<script>\n${reportJS}\n</script>`,
+    `<script type="module">\n${reportJS}\n</script>`,
   );
   return html;
 }
@@ -78,7 +82,7 @@ function putReportTplIntoHTML(html: string, outsourceMode = false) {
     // in Chrome extension
     return html.replace(
       '</body>',
-      `${tplWrapper}<script src="/lib/set-report-tpl.js"></script>\n</body>`,
+      `${tplWrapper}<script type="module" src="/lib/set-report-tpl.js"></script>\n</body>`,
     );
   }
 
@@ -157,10 +161,9 @@ function buildExtension() {
     join(__dirname, '../unpacked-extension/lib/htmlElement.js'),
   );
 }
-
 async function zipDir(src: string, dest: string) {
   // console.log('cwd', dirname(src));
-  await execa('zip', ['-r', dest, '.'], {
+  execSync(`zip -r "${dest}" .`, {
     cwd: src,
   });
 }

packages/visualizer/scripts/building-utils.ts

@@ -1,5 +1,6 @@
 import { copyFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
 
 export function ensureDirectoryExistence(filePath: string) {
   const directoryPath = dirname(filePath);
@@ -22,6 +23,8 @@ export function tplReplacer(
 }
 
 export const fileContentOfPath = (path: string) => {
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = dirname(__filename);
   const filePath = join(__dirname, path);
   return readFileSync(filePath, 'utf-8');
 };

packages/visualizer/scripts/tsconfig.json

@@ -1,12 +1,11 @@
 {
   "compilerOptions": {
     "target": "ES2020",
-    "module": "commonjs",
+    "module": "ES2020",
     "strict": true,
     "esModuleInterop": true,
     "skipLibCheck": true,
-    "forceConsistentCasingInFileNames": true,
-    "resolveJsonModule": true
+    "forceConsistentCasingInFileNames": true
   },
   "include": ["./**/*.ts"],
   "exclude": ["node_modules"]

packages/visualizer/src/index.tsx

@@ -465,6 +465,20 @@ function mount(id: string) {
   root.render(<Visualizer dumps={reportDump} />);
 }
 
+declare global {
+  interface Window {
+    midsceneVisualizer: {
+      mount: (id: string) => void;
+      Visualizer: typeof Visualizer;
+    };
+  }
+}
+
+window.midsceneVisualizer = {
+  mount,
+  Visualizer,
+};
+
 export default {
   mount,
   Visualizer,

packages/visualizer/src/types.d.ts

@@ -0,0 +1,8 @@
+// 扩展 Window 接口以包含 global 和 Buffer 属性
+interface Window {
+  global: typeof globalThis;
+  Buffer: any;
+}
+
+// 版本变量
+declare const __VERSION__: string;

packages/visualizer/tsconfig.json

@@ -17,6 +17,7 @@
     "rootDir": "src",
     "skipLibCheck": true,
     "strict": true,
+    "target": "ES2022",
     "types": ["react"]
   },
   "exclude": ["**/node_modules"],

packages/web-integration/modern.config.ts

@@ -27,34 +27,12 @@ export default defineConfig({
   plugins: [moduleTools()],
   buildPreset: 'npm-library',
   buildConfig: {
-    format: 'cjs',
-    input: {
-      index: 'src/index.ts',
-      'bridge-mode': 'src/bridge-mode/index.ts',
-      'bridge-mode-browser': 'src/bridge-mode/browser.ts',
-      utils: 'src/common/utils.ts',
-      'ui-utils': 'src/common/ui-utils.ts',
-      puppeteer: 'src/puppeteer/index.ts',
-      playwright: 'src/playwright/index.ts',
-      playground: 'src/playground/index.ts',
-      'midscene-playground': 'src/playground/bin.ts',
-      appium: 'src/appium/index.ts',
-      'playwright-report': './src/playwright/reporter/index.ts',
-      'chrome-extension': 'src/chrome-extension/index.ts',
-      yaml: 'src/yaml/index.ts',
-    },
-    target: 'es2018',
-    externals: [
-      '@midscene/core',
-      '@midscene/shared',
-      'puppeteer',
-      'bufferutil',
-      'utf-8-validate',
-    ],
+    format: 'esm',
+    target: 'es6',
+    buildType: 'bundleless',
     define: {
       __VERSION__: version,
     },
     sourceMap: true,
-    // splitting: true,
   },
 });

packages/web-integration/package.json

@@ -7,95 +7,99 @@
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "types": "./dist/types/index.d.ts",
+  "type": "module",
   "bin": {
     "midscene-playground": "./bin/midscene-playground"
   },
   "exports": {
     ".": {
-      "require": "./dist/lib/index.js",
+      "types": "./dist/types/index.d.ts",
       "import": "./dist/es/index.js",
-      "types": "./dist/types/index.d.ts"
+      "require": "./dist/lib/index.js"
     },
     "./bridge-mode": {
-      "require": "./dist/lib/bridge-mode.js",
-      "import": "./dist/es/bridge-mode.js",
-      "types": "./dist/types/bridge-mode.d.ts"
+      "types": "./dist/types/bridge-mode/index.d.ts",
+      "import": "./dist/es/bridge-mode/index.js",
+      "require": "./dist/lib/bridge-mode/index.js"
     },
     "./bridge-mode-browser": {
-      "require": "./dist/lib/bridge-mode-browser.js",
-      "import": "./dist/es/bridge-mode-browser.js",
-      "types": "./dist/types/bridge-mode-browser.d.ts"
+      "types": "./dist/types/bridge-mode/browser.d.ts",
+      "import": "./dist/es/bridge-mode/browser.js",
+      "require": "./dist/lib/bridge-mode/browser.js"
     },
     "./utils": {
-      "require": "./dist/lib/utils.js",
-      "import": "./dist/es/utils.js",
-      "types": "./dist/types/utils.d.ts"
+      "types": "./dist/types/common/utils.d.ts",
+      "import": "./dist/es/common/utils.js",
+      "require": "./dist/lib/common/utils.js"
     },
     "./ui-utils": {
-      "require": "./dist/lib/ui-utils.js",
-      "import": "./dist/es/ui-utils.js",
-      "types": "./dist/types/ui-utils.d.ts"
+      "types": "./dist/types/common/ui-utils.d.ts",
+      "import": "./dist/es/common/ui-utils.js",
+      "require": "./dist/lib/common/ui-utils.js"
     },
     "./puppeteer": {
-      "require": "./dist/lib/puppeteer.js",
-      "import": "./dist/es/puppeteer.js",
-      "types": "./dist/types/puppeteer.d.ts"
+      "types": "./dist/types/puppeteer/index.d.ts",
+      "import": "./dist/es/puppeteer/index.js",
+      "require": "./dist/lib/puppeteer/index.js"
     },
     "./playwright": {
-      "require": "./dist/lib/playwright.js",
-      "import": "./dist/es/playwright.js",
-      "types": "./dist/types/playwright.d.ts"
+      "types": "./dist/types/playwright/index.d.ts",
+      "import": "./dist/es/playwright/index.js",
+      "require": "./dist/lib/playwright/index.js"
     },
     "./playwright-report": {
-      "require": "./dist/lib/playwright-report.js",
-      "types": "./dist/types/playwright-report.d.ts"
+      "types": "./dist/types/playwright/reporter/index.d.ts",
+      "import": "./dist/es/playwright/reporter/index.js",
+      "require": "./dist/lib/playwright/reporter/index.js"
     },
     "./playground": {
-      "require": "./dist/lib/playground.js",
-      "import": "./dist/es/playground.js",
-      "types": "./dist/types/playground.d.ts"
+      "types": "./dist/types/playground/index.d.ts",
+      "import": "./dist/es/playground/index.js",
+      "require": "./dist/lib/playground/index.js"
     },
-    "./constants": {
-      "require": "./dist/lib/constants.js",
-      "import": "./dist/es/constants.js",
-      "types": "./dist/types/constants.d.ts"
+    "./midscene-playground": {
+      "types": "./dist/types/playground/bin.d.ts",
+      "import": "./dist/es/playground/bin.js",
+      "require": "./dist/lib/playground/bin.js"
     },
-    "./html-element": {
-      "require": "./dist/lib/html-element/index.js",
-      "types": "./dist/types/html-element/index.d.ts"
+    "./appium": {
+      "types": "./dist/types/appium/index.d.ts",
+      "import": "./dist/es/appium/index.js",
+      "require": "./dist/lib/appium/index.js"
     },
     "./chrome-extension": {
-      "require": "./dist/lib/chrome-extension.js",
-      "import": "./dist/es/chrome-extension.js",
-      "types": "./dist/types/chrome-extension.d.ts"
+      "types": "./dist/types/chrome-extension/index.d.ts",
+      "import": "./dist/es/chrome-extension/index.js",
+      "require": "./dist/lib/chrome-extension/index.js"
     },
     "./yaml": {
-      "require": "./dist/lib/yaml.js",
-      "import": "./dist/es/yaml.js",
-      "types": "./dist/types/yaml.d.ts"
+      "types": "./dist/types/yaml/index.d.ts",
+      "import": "./dist/es/yaml/index.js",
+      "require": "./dist/lib/yaml/index.js"
     }
   },
   "typesVersions": {
     "*": {
       ".": ["./dist/types/index.d.ts"],
-      "bridge-mode": ["./dist/types/bridge-mode.d.ts"],
-      "bridge-mode-browser": ["./dist/types/bridge-mode-browser.d.ts"],
-      "utils": ["./dist/types/utils.d.ts"],
-      "ui-utils": ["./dist/types/ui-utils.d.ts"],
-      "puppeteer": ["./dist/types/puppeteer.d.ts"],
-      "playwright": ["./dist/types/playwright.d.ts"],
-      "playwright-report": ["./dist/types/playwright-report.d.ts"],
-      "playground": ["./dist/types/playground.d.ts"],
-      "constants": ["./dist/types/constants.d.ts"],
-      "html-element": ["./dist/types/html-element/index.d.ts"],
-      "chrome-extension": ["./dist/types/chrome-extension.d.ts"],
-      "yaml": ["./dist/types/yaml.d.ts"]
+      "bridge-mode": ["./dist/types/bridge-mode/index.d.ts"],
+      "bridge-mode-browser": ["./dist/types/bridge-mode/browser.d.ts"],
+      "utils": ["./dist/types/common/utils.d.ts"],
+      "ui-utils": ["./dist/types/common/ui-utils.d.ts"],
+      "puppeteer": ["./dist/types/puppeteer/index.d.ts"],
+      "playwright": ["./dist/types/playwright/index.d.ts"],
+      "playwright-report": ["./dist/types/playwright/reporter/index.d.ts"],
+      "playground": ["./dist/types/playground/index.d.ts"],
+      "midscene-playground": ["./dist/types/playground/bin.d.ts"],
+      "appium": ["./dist/types/appium/index.d.ts"],
+      "chrome-extension": ["./dist/types/chrome-extension/index.d.ts"],
+      "yaml": ["./dist/types/yaml/index.d.ts"]
     }
   },
   "scripts": {
     "dev": "modern dev",
     "dev:server": "npm run build && ./bin/midscene-playground",
     "build": "modern build -c ./modern.config.ts",
+    "postbuild": "node scripts/check-exports.js",
     "build:watch": "modern build -w -c ./modern.config.ts",
     "test": "vitest --run",
     "test:u": "vitest --run -u",
@@ -107,11 +111,11 @@
     "test:ai:native": "MIDSCENE_CACHE=true AI_TEST_TYPE=native npm run test",
     "upgrade": "modern upgrade",
     "prepublishOnly": "npm run build",
-    "e2e": "playwright test --config=playwright.config.ts",
-    "e2e:report": "MIDSCENE_REPORT=true playwright test --config=playwright.config.ts",
-    "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts",
-    "e2e:ui": "playwright test --config=playwright.config.ts --ui",
-    "e2e:ui:cache": "MIDSCENE_CACHE=true playwright test --config=playwright.config.ts --ui"
+    "e2e": "playwright test --config=tests/playwright.config.ts",
+    "e2e:report": "MIDSCENE_REPORT=true playwright test --config=tests/playwright.config.ts",
+    "e2e:cache": "MIDSCENE_CACHE=true playwright test --config=tests/playwright.config.ts",
+    "e2e:ui": "playwright test --config=tests/playwright.config.ts --ui",
+    "e2e:ui:cache": "MIDSCENE_CACHE=true playwright test --config=tests/playwright.config.ts --ui"
   },
   "files": ["static", "dist", "iife-script", "README.md", "bin"],
   "dependencies": {
@@ -169,72 +173,7 @@
   },
   "publishConfig": {
     "access": "public",
-    "registry": "https://registry.npmjs.org",
-    "exports": {
-      ".": {
-        "require": "./dist/lib/index.js",
-        "import": "./dist/es/index.js",
-        "types": "./dist/types/index.d.ts"
-      },
-      "./bridge-mode": {
-        "require": "./dist/lib/bridge-mode.js",
-        "import": "./dist/es/bridge-mode.js",
-        "types": "./dist/types/bridge-mode.d.ts"
-      },
-      "./bridge-mode-browser": {
-        "require": "./dist/lib/bridge-mode-browser.js",
-        "import": "./dist/es/bridge-mode-browser.js",
-        "types": "./dist/types/bridge-mode-browser.d.ts"
-      },
-      "./utils": {
-        "require": "./dist/lib/utils.js",
-        "import": "./dist/es/utils.js",
-        "types": "./dist/types/utils.d.ts"
-      },
-      "./ui-utils": {
-        "require": "./dist/lib/ui-utils.js",
-        "import": "./dist/es/ui-utils.js",
-        "types": "./dist/types/ui-utils.d.ts"
-      },
-      "./puppeteer": {
-        "require": "./dist/lib/puppeteer.js",
-        "import": "./dist/es/puppeteer.js",
-        "types": "./dist/types/puppeteer.d.ts"
-      },
-      "./playwright": {
-        "require": "./dist/lib/playwright.js",
-        "import": "./dist/es/playwright.js",
-        "types": "./dist/types/playwright.d.ts"
-      },
-      "./playwright-report": {
-        "require": "./dist/lib/playwright-report.js",
-        "types": "./dist/types/playwright-report.d.ts"
-      },
-      "./playground": {
-        "require": "./dist/lib/playground.js",
-        "import": "./dist/es/playground.js",
-        "types": "./dist/types/playground.d.ts"
-      },
-      "./constants": {
-        "require": "./dist/lib/constants.js",
-        "import": "./dist/es/constants.js",
-        "types": "./dist/types/constants.d.ts"
-      },
-      "./html-element": {
-        "require": "./dist/lib/html-element/index.js",
-        "types": "./dist/types/html-element/index.d.ts"
-      },
-      "./chrome-extension": {
-        "require": "./dist/lib/chrome-extension.js",
-        "import": "./dist/es/chrome-extension.js",
-        "types": "./dist/types/chrome-extension.d.ts"
-      },
-      "./yaml": {
-        "require": "./dist/lib/yaml.js",
-        "import": "./dist/es/yaml.js",
-        "types": "./dist/types/yaml.d.ts"
-      }
-    }
+    "registry": "https://registry.npmjs.org"
   },
   "license": "MIT"
 }

packages/web-integration/scripts/check-exports.js

@@ -0,0 +1,92 @@
+// scripts/check-exports.js
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+function checkConsistency() {
+  const pkgPath = path.resolve(__dirname, '../package.json');
+  const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8'));
+  const exports = pkg.exports;
+  const typesVersions = pkg.typesVersions ? pkg.typesVersions['*'] || {} : {};
+  const rootDir = path.dirname(pkgPath);
+
+  const errors = [];
+
+  // Check if each exports entry has a corresponding typesVersions entry, and check if files exist
+  for (const [key, value] of Object.entries(exports)) {
+    // Check if types file exists
+    const typesPath = value.types;
+    if (typesPath) {
+      const absoluteTypesPath = path.resolve(rootDir, typesPath);
+      if (!fs.existsSync(absoluteTypesPath)) {
+        errors.push(`File does not exist: ${typesPath}`);
+      }
+
+      const tsVersionPath = typesVersions[key.replace('./', '')];
+      if (!tsVersionPath) {
+        errors.push(`Missing typesVersions entry: ${key}`);
+      } else if (tsVersionPath[0] !== typesPath) {
+        errors.push(
+          `Path mismatch: exports[${key}].types = ${typesPath}, typesVersions[${key}] = ${tsVersionPath[0]}`,
+        );
+      }
+    }
+
+    // Check if import and require files exist
+    if (value.import) {
+      const importPath = value.import;
+      const absoluteImportPath = path.resolve(rootDir, importPath);
+      if (!fs.existsSync(absoluteImportPath)) {
+        errors.push(`File does not exist: ${importPath}`);
+      }
+    }
+
+    if (value.require) {
+      const requirePath = value.require;
+      const absoluteRequirePath = path.resolve(rootDir, requirePath);
+      if (!fs.existsSync(absoluteRequirePath)) {
+        errors.push(`File does not exist: ${requirePath}`);
+      }
+    }
+
+    // If value is a string, directly check if the file exists
+    if (typeof value === 'string') {
+      const absolutePath = path.resolve(rootDir, value);
+      if (!fs.existsSync(absolutePath)) {
+        errors.push(`File does not exist: ${value}`);
+      }
+    }
+  }
+
+  // Check if each entry in typesVersions has a corresponding entry in exports
+  for (const [key, value] of Object.entries(typesVersions)) {
+    const exportKey = key === '.' ? key : `./${key}`;
+    if (!exports[exportKey]) {
+      errors.push(`Missing exports entry: ${exportKey}`);
+    }
+
+    // Check if files in typesVersions exist
+    if (Array.isArray(value) && value.length > 0) {
+      const typePath = value[0];
+      const absoluteTypePath = path.resolve(rootDir, typePath);
+      if (!fs.existsSync(absoluteTypePath)) {
+        errors.push(`File does not exist: ${typePath}`);
+      }
+    }
+  }
+
+  if (errors.length > 0) {
+    console.error('Found issues with exports and typesVersions:');
+    errors.forEach((err) => console.error(` - ${err}`));
+    process.exit(1);
+  } else {
+    console.log(
+      'exports and typesVersions configuration is consistent and all files exist ✓',
+    );
+  }
+}
+
+checkConsistency();

packages/web-integration/src/bridge-mode/agent-cli-side.ts

@@ -1,10 +1,10 @@
-import assert from 'node:assert';
 import { PageAgent, type PageAgentOpt } from '@/common/agent';
 import type {
   ChromePageDestroyOptions,
   KeyboardAction,
   MouseAction,
 } from '@/page';
+import { assert } from '@midscene/shared/utils';
 import {
   type BridgeConnectTabOptions,
   BridgeEvent,

packages/web-integration/src/bridge-mode/io-client.ts

@@ -1,4 +1,4 @@
-import assert from 'node:assert';
+import { assert } from '@midscene/shared/utils';
 import { io as ClientIO, type Socket as ClientSocket } from 'socket.io-client';
 import {
   type BridgeCallRequest,

packages/web-integration/src/bridge-mode/page-browser-side.ts

@@ -1,9 +1,9 @@
-import assert from 'node:assert';
 import type {
   ChromePageDestroyOptions,
   KeyboardAction,
   MouseAction,
 } from '@/page';
+import { assert } from '@midscene/shared/utils';
 import ChromeExtensionProxyPage from '../chrome-extension/page';
 import {
   type BridgeConnectTabOptions,

packages/web-integration/src/chrome-extension/cdpInput.ts

@@ -6,13 +6,12 @@
  * SPDX-License-Identifier: Apache-2.0
  */
 
-import assert from 'node:assert';
-import { WebKeyInput } from '@/common/page';
 import {
   type KeyDefinition,
   type KeyInput,
   _keyDefinitions,
 } from '@midscene/shared/keyboard-layout';
+import { assert } from '@midscene/shared/utils';
 
 type KeyDescription = Required<
   Pick<KeyDefinition, 'keyCode' | 'key' | 'text' | 'code' | 'location'>

packages/web-integration/src/chrome-extension/page.ts

@@ -5,13 +5,13 @@
   The page must be active when interacting with it.
 */
 
-import assert from 'node:assert';
 import type { WebKeyInput } from '@/common/page';
 import { limitOpenNewTabScript } from '@/common/ui-utils';
 import type { AbstractPage, ChromePageDestroyOptions } from '@/page';
 import type { ElementTreeNode, Point, Size } from '@midscene/core';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { treeToList } from '@midscene/shared/extractor';
+import { assert } from '@midscene/shared/utils';
 import type { Protocol as CDPTypes } from 'devtools-protocol';
 import { CdpKeyboard } from './cdpInput';
 import {

packages/web-integration/src/common/tasks.ts

@@ -1,4 +1,3 @@
-import assert from 'node:assert';
 import type { WebPage } from '@/common/page';
 import type { PuppeteerWebPage } from '@/puppeteer';
 import {
@@ -35,6 +34,7 @@ import {
 } from '@midscene/core/ai-model';
 import { sleep } from '@midscene/core/utils';
 import type { ElementInfo } from '@midscene/shared/extractor';
+import { assert } from '@midscene/shared/utils';
 import type { WebElementInfo } from '../web-element';
 import { TaskCache } from './task-cache';
 import { getKeyCommands } from './ui-utils';

packages/web-integration/src/common/utils.ts

@@ -1,6 +1,3 @@
-import assert from 'node:assert';
-import { readFileSync } from 'node:fs';
-import path from 'node:path';
 import type { StaticPage } from '@/playground';
 import type {
   ElementTreeNode,
@@ -16,9 +13,8 @@ import { uploadTestInfoToServer } from '@midscene/core/utils';
 import { NodeType } from '@midscene/shared/constants';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { traverseTree, treeToList } from '@midscene/shared/extractor';
-import { findNearestPackageJson } from '@midscene/shared/fs';
 import { compositeElementInfoImg, resizeImgBase64 } from '@midscene/shared/img';
-import { uuid } from '@midscene/shared/utils';
+import { assert, uuid } from '@midscene/shared/utils';
 import dayjs from 'dayjs';
 import { WebElementInfo } from '../web-element';
 import type { WebPage } from './page';
@@ -108,46 +104,6 @@ export async function parseContextFromWebPage(
   };
 }
 
-export async function getExtraReturnLogic(tree = false) {
-  const pathDir = findNearestPackageJson(__dirname);
-  assert(pathDir, `can't find pathDir, with ${__dirname}`);
-  const scriptPath = path.join(pathDir, './iife-script/htmlElement.js');
-  const elementInfosScriptContent = readFileSync(scriptPath, 'utf-8');
-  if (tree) {
-    return `${elementInfosScriptContent}midscene_element_inspector.webExtractNodeTree()`;
-  }
-  return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
-}
-
-const sizeThreshold = 3;
-// async function alignElements(
-//   elements: ElementInfo[],
-//   page: WebPage,
-// ): Promise<WebElementInfo[]> {
-//   const validElements = elements.filter((item) => {
-//     return (
-//       item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold
-//     );
-//   });
-//   const textsAligned: WebElementInfo[] = [];
-//   for (const item of validElements) {
-//     const { rect, id, content, attributes, locator, indexId } = item;
-//     textsAligned.push(
-//       new WebElementInfo({
-//         rect,
-//         locator,
-//         id,
-//         content,
-//         attributes,
-//         page,
-//         indexId,
-//       }),
-//     );
-//   }
-
-//   return textsAligned;
-// }
-
 export function reportFileName(tag = 'web') {
   const reportTagName = getAIConfig(MIDSCENE_REPORT_TAG_NAME);
   const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss-SSS');

packages/web-integration/src/playground/server.ts

@@ -1,10 +1,10 @@
-import assert from 'node:assert';
 import { randomUUID } from 'node:crypto';
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import type { Server } from 'node:http';
 import { join } from 'node:path';
 import { ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED } from '@/common/utils';
 import { getTmpDir } from '@midscene/core/utils';
+import { assert } from '@midscene/shared/utils';
 import { ifInBrowser } from '@midscene/shared/utils';
 import cors from 'cors';
 import express from 'express';

packages/web-integration/src/playwright/ai-fixture.ts

@@ -197,7 +197,7 @@ export type PlayWrightAiFixtureType = {
   aiWaitFor: (assertion: string, opt?: AgentWaitForOpt) => Promise<void>;
 };
 
-async function waitForNetworkIdle(page: OriginPlaywrightPage, timeout = 20000) {
+async function waitForNetworkIdle(page: OriginPlaywrightPage, timeout = 10000) {
   try {
     await page.waitForLoadState('networkidle', { timeout });
   } catch (error: any) {

packages/web-integration/src/puppeteer/agent-launcher.ts

@@ -1,5 +1,5 @@
-import assert from 'node:assert';
 import { readFileSync } from 'node:fs';
+import { assert } from '@midscene/shared/utils';
 
 import { PuppeteerAgent } from '@/puppeteer';
 import type { MidsceneYamlScriptEnv } from '@midscene/core';

packages/web-integration/src/puppeteer/base-page.ts

@@ -2,11 +2,12 @@ import type { ElementTreeNode, Point, Size } from '@midscene/core';
 import { getTmpFile, sleep } from '@midscene/core/utils';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { treeToList } from '@midscene/shared/extractor';
+import { getExtraReturnLogic } from '@midscene/shared/fs';
 import { base64Encoded } from '@midscene/shared/img';
+import { assert } from '@midscene/shared/utils';
 import type { Page as PlaywrightPage } from 'playwright';
 import type { Page as PuppeteerPage } from 'puppeteer';
 import type { WebKeyInput } from '../common/page';
-import { getExtraReturnLogic } from '../common/utils';
 import type { AbstractPage } from '../page';
 import type { MouseButton } from '../page';
 
@@ -57,6 +58,7 @@ export class Page<
     // The page may go through opening, closing, and reopening; if the page is closed, evaluate may return undefined, which can lead to errors.
     await this.waitForNavigation();
     const scripts = await getExtraReturnLogic(true);
+    assert(scripts, 'scripts should be set before writing report in browser');
     const captureElementSnapshot = await this.evaluate(scripts);
     return captureElementSnapshot as ElementTreeNode<ElementInfo>;
   }

packages/web-integration/src/yaml/player.ts

@@ -1,6 +1,6 @@
-import assert from 'node:assert';
 import { existsSync, mkdirSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
+import { assert } from '@midscene/shared/utils';
 
 import type { PageAgent } from '@/common/agent';
 import type {

packages/web-integration/src/yaml/utils.ts

@@ -1,4 +1,4 @@
-import assert from 'node:assert';
+import { assert } from '@midscene/shared/utils';
 import yaml from 'js-yaml';
 
 import type { MidsceneYamlScript } from '@midscene/core';

packages/web-integration/tests/playwright.config.ts

@@ -1,8 +1,13 @@
 import path from 'node:path';
+import { fileURLToPath } from 'node:url';
 import { defineConfig, devices } from '@playwright/test';
 //@ts-ignore
 import dotenv from 'dotenv';
 
+// 添加这两行获取当前文件目录（ES 模块兼容方式）
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
 const MIDSCENE_REPORT = process.env.MIDSCENE_REPORT;
 
 /**
@@ -10,7 +15,7 @@ const MIDSCENE_REPORT = process.env.MIDSCENE_REPORT;
  * https://github.com/motdotla/dotenv
  */
 dotenv.config({
-  path: path.join(__dirname, '../../.env'),
+  path: path.join(__dirname, '../../../.env'),
 });
 
 /**
@@ -47,12 +52,12 @@ export default defineConfig({
     MIDSCENE_REPORT
       ? {
           name: 'report',
-          testDir: './tests/ai/web/playwright-report-test',
+          testDir: './ai/web/playwright-report-test',
           use: { ...devices['Desktop Chrome'] },
         }
       : {
           name: 'e2e',
-          testDir: './tests/ai/web/playwright',
+          testDir: './ai/web/playwright',
           use: { ...devices['Desktop Chrome'] },
         },
   ],
@@ -63,6 +68,6 @@ export default defineConfig({
     //   { outputFile: 'midscene_run/playwright-report/test-results.json' },
     // ],
     // ['html', { outputFolder: 'midscene_run/playwright-report' }],
-    ['./src/playwright/reporter/index.ts'],
+    ['../src/playwright/reporter/index.ts'],
   ],
 });

packages/web-integration/tests/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../tsconfig.json",
+  "compilerOptions": {
+    "baseUrl": "../",
+    "rootDir": "../"
+  },
+  "include": ["**/*", "../src", "playwright.config.ts"],
+  "exclude": ["**/node_modules"]
+}

packages/web-integration/tests/unit-test/task-cache.test.ts

@@ -1,7 +1,7 @@
 import { type LocateTask, type PlanTask, TaskCache } from '@/common/task-cache';
 import type { WebUIContext } from '@/common/utils';
 import type { WebElementInfo } from '@/web-element';
-import type { AIElementParseResponse } from '@midscene/core';
+import type { AIElementIdResponse } from '@midscene/core';
 import { beforeEach, describe, expect, it } from 'vitest';
 
 describe('TaskCache', () => {
@@ -19,7 +19,7 @@ describe('TaskCache', () => {
       ...pageContext,
       screenshotBase64: '',
       content: [{ id: 'element1' } as WebElementInfo], // 示例页面内容
-    };
+    } as any;
   });
 
   it('should return false if no cache is available', async () => {
@@ -41,7 +41,11 @@ describe('TaskCache', () => {
             type: 'plan',
             prompt: 'different prompt',
             pageContext,
-            response: { plans: [] },
+            response: {
+              actions: [],
+              log: '',
+              more_actions_needed_by_instruction: false,
+            },
           },
         ],
       },
@@ -67,7 +71,7 @@ describe('TaskCache', () => {
               pageContext,
               response: {
                 elements: [{ id: 'element3' }],
-              } as AIElementParseResponse,
+              } as AIElementIdResponse,
             },
           ],
         },
@@ -85,7 +89,9 @@ describe('TaskCache', () => {
   it('should return cached response if the conditions match', async () => {
     const cachedResponse = {
       plans: [{ type: 'Locate', thought: '', param: {} }],
-    } as PlanTask['response'];
+      more_actions_needed_by_instruction: false,
+      log: '',
+    };
     taskCache.cache = {
       aiTasks: [
         {
@@ -117,7 +123,11 @@ describe('TaskCache', () => {
       type: 'plan',
       prompt: 'new prompt',
       pageContext,
-      response: { plans: [{ type: 'Locate', thought: '', param: {} }] },
+      response: {
+        actions: [{ type: 'Locate', thought: '', param: {}, locate: null }],
+        more_actions_needed_by_instruction: false,
+        log: '',
+      },
     };
     cacheGroup.saveCache(newCache);
     expect(taskCache.newCache.aiTasks[0].tasks).toContain(newCache);

packages/web-integration/tests/unit-test/yaml/player.test.ts

@@ -1,5 +1,5 @@
-import assert from 'node:assert';
 import { join } from 'node:path';
+import { assert } from '@midscene/shared/utils';
 
 import { randomUUID } from 'node:crypto';
 import { existsSync } from 'node:fs';

packages/web-integration/tsconfig.json

@@ -14,11 +14,11 @@
     },
     "target": "es6",
     "resolveJsonModule": true,
-    "rootDir": "./",
+    "rootDir": "./src",
     "skipLibCheck": true,
     "strict": true,
     "module": "ESNext"
   },
   "exclude": ["node_modules"],
-  "include": ["src", "tests", "./playwright.config.ts", "./vitest.config"]
+  "include": ["src", "./vitest.config"]
 }

pnpm-lock.yaml

@@ -222,6 +222,9 @@ importers:
       jimp:
         specifier: 0.22.12
         version: 0.22.12
+      js-sha256:
+        specifier: 0.11.0
+        version: 0.11.0
     devDependencies:
       '@modern-js/module-tools':
         specifier: 2.60.6
@@ -229,9 +232,6 @@ importers:
       '@types/node':
         specifier: ^18.0.0
         version: 18.19.62
-      js-sha256:
-        specifier: 0.11.0
-        version: 0.11.0
       rimraf:
         specifier: ~3.0.2
         version: 3.0.2