feat: migrate android code to @midscene/android (#505)

packages/android/.gitignore

@@ -0,0 +1,4 @@
+
+# Midscene.js dump files
+midscene_run/report
+midscene_run/tmp

packages/android/README.md

@@ -0,0 +1,14 @@
+## Android prerequisites
+
+Android is driven by adb, so you need install adb first:
+- [CLI](https://developer.android.com/tools/adb)
+
+```bash
+npm run test:ai -- setting.test.ts
+```
+
+or 
+
+```bash
+npm run test:ai -- todo.test.ts
+```

packages/android/modern.config.ts

@@ -1,3 +1,4 @@
+import path from 'node:path';
 import { defineConfig, moduleTools } from '@modern-js/module-tools';
 
 export default defineConfig({

packages/android/package.json

@@ -1,9 +1,10 @@
 {
   "name": "@midscene/android",
   "version": "0.13.1",
+  "description": "Android automation library for Midscene",
   "main": "./dist/lib/index.js",
   "types": "./dist/types/index.d.ts",
-  "files": ["dist"],
+  "files": ["dist", "bin", "README.md"],
   "exports": {
     ".": {
       "types": "./dist/types/index.d.ts",
@@ -11,17 +12,23 @@
     }
   },
   "scripts": {
-    "build": "modern build -c ./modern.config.ts"
+    "build": "modern build -c ./modern.config.ts",
+    "test": "vitest --run",
+    "test:u": "vitest --run -u",
+    "test:ai": "MIDSCENE_CACHE=true npm run test"
   },
   "dependencies": {
-    "@midscene/web": "workspace:*"
+    "@midscene/web": "workspace:*",
+    "@midscene/core": "workspace:*",
+    "@midscene/shared": "workspace:*",
+    "appium-adb": "12.12.1"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",
     "@types/node": "^18.0.0",
-    "typescript": "^5.8.2"
+    "dotenv": "16.4.5",
+    "typescript": "^5.8.2",
+    "vitest": "3.0.5"
   },
-  "author": "",
-  "license": "ISC",
-  "description": ""
+  "license": "MIT"
 }

packages/android/src/index.ts

@@ -1 +1,2 @@
-export { AndroidAgent, AndroidPage } from '@midscene/web/android';
+export { PageAgent as AndroidAgent } from '@midscene/web';
+export { Page as AndroidPage } from './page';

packages/android/src/page/index.ts

@@ -5,10 +5,8 @@ import { getTmpFile } from '@midscene/core/utils';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { resizeImg } from '@midscene/shared/img';
 import { getDebug } from '@midscene/shared/utils';
+import type { AbstractPage } from '@midscene/web';
 import { ADB } from 'appium-adb';
-import type { KeyInput as PuppeteerKeyInput } from 'puppeteer';
-import type { AbstractPage, MouseButton } from '../page';
-type WebKeyInput = PuppeteerKeyInput;
 
 const debugPage = getDebug('android');
 
@@ -177,8 +175,7 @@ export class Page implements AbstractPage {
 
   get mouse() {
     return {
-      click: (x: number, y: number, options?: { button: MouseButton }) =>
-        this.mouseClick(x, y, options?.button || 'left'),
+      click: (x: number, y: number) => this.mouseClick(x, y),
       wheel: (deltaX: number, deltaY: number) =>
         this.mouseWheel(deltaX, deltaY),
       move: (x: number, y: number) => this.mouseMove(x, y),
@@ -192,8 +189,8 @@ export class Page implements AbstractPage {
       type: (text: string) => this.keyboardType(text),
       press: (
         action:
-          | { key: WebKeyInput; command?: string }
-          | { key: WebKeyInput; command?: string }[],
+          | { key: string; command?: string }
+          | { key: string; command?: string }[],
       ) => this.keyboardPressAction(action),
     };
   }
@@ -301,7 +298,6 @@ export class Page implements AbstractPage {
     // Push the YADB tool to the device only once
     if (!this.yadbPushed) {
       const adb = await this.getAdb();
-
       const yadbBin = path.join(__dirname, '../../bin/yadb');
       await adb.push(yadbBin, '/data/local/tmp');
       this.yadbPushed = true;
@@ -323,7 +319,7 @@ export class Page implements AbstractPage {
     await this.execYadb(text);
   }
 
-  private async keyboardPress(key: WebKeyInput): Promise<void> {
+  private async keyboardPress(key: string): Promise<void> {
     // Map web keys to Android key codes (numbers)
     const keyCodeMap: Record<string, number> = {
       Enter: 66,
@@ -357,8 +353,8 @@ export class Page implements AbstractPage {
 
   private async keyboardPressAction(
     action:
-      | { key: WebKeyInput; command?: string }
-      | { key: WebKeyInput; command?: string }[],
+      | { key: string; command?: string }
+      | { key: string; command?: string }[],
   ): Promise<void> {
     if (Array.isArray(action)) {
       for (const act of action) {
@@ -369,18 +365,7 @@ export class Page implements AbstractPage {
     }
   }
 
-  private async mouseClick(
-    x: number,
-    y: number,
-    button: MouseButton = 'left',
-  ): Promise<void> {
-    // ADB only supports left mouse button clicks
-    if (button !== 'left') {
-      console.warn(
-        `ADB only supports left mouse button clicks. Ignored request for ${button} button.`,
-      );
-    }
-
+  private async mouseClick(x: number, y: number): Promise<void> {
     await this.mouseMove(x, y);
 
     const adb = await this.getAdb();

packages/android/tests/ai/setting.test.ts

@@ -1,5 +1,5 @@
-import { AndroidAgent } from '@/android';
 import { describe, it, vi } from 'vitest';
+import { AndroidAgent } from '../../src';
 import { launchPage } from './utils';
 
 vi.setConfig({

packages/android/tests/ai/todo.test.ts

@@ -1,7 +1,6 @@
-import { AndroidAgent } from '@/android';
 import { beforeAll, describe, expect, it, vi } from 'vitest';
+import { AndroidAgent } from '../../src';
 import { launchPage } from './utils';
-import 'dotenv/config'; // read environment variables from .env file
 
 vi.setConfig({
   testTimeout: 240 * 1000,

packages/android/tests/ai/utils.ts

@@ -1,7 +1,7 @@
 import { exec } from 'node:child_process';
 import { promisify } from 'node:util';
 import type { StartAppOptions } from 'appium-adb';
-import { AndroidPage } from '../../../../src/android';
+import { Page as AndroidPage } from '../../src/page';
 const execPromise = promisify(exec);
 
 interface LaunchOptions {

packages/android/vitest.config.ts

@@ -0,0 +1,30 @@
+import path from 'node:path';
+import dotenv from 'dotenv';
+import { defineConfig } from 'vitest/config';
+import { version } from './package.json';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+dotenv.config({
+  path: path.join(__dirname, '../../.env'),
+});
+
+const testFiles = ['tests/ai/**/*.test.ts'];
+
+export default defineConfig({
+  resolve: {
+    alias: {
+      '@': path.resolve(__dirname, 'src'),
+    },
+  },
+  test: {
+    include: testFiles,
+    testTimeout: 3 * 60 * 1000, // Global timeout set to 10 seconds
+    dangerouslyIgnoreUnhandledErrors: !!process.env.CI, // showcase.test.ts is not stable
+  },
+  define: {
+    __VERSION__: `'${version}'`,
+  },
+});

packages/core/vitest.config.ts

@@ -1,5 +1,4 @@
 import path from 'node:path';
-//@ts-ignore
 import dotenv from 'dotenv';
 import { defineConfig } from 'vitest/config';
 import { version } from './package.json';

packages/evaluation/vitest.config.ts

@@ -1,5 +1,4 @@
 import path from 'node:path';
-//@ts-ignore
 import dotenv from 'dotenv';
 import { defineConfig } from 'vitest/config';
 

packages/shared/package.json

@@ -57,6 +57,7 @@
     "@modern-js/module-tools": "2.60.6",
     "@types/debug": "4.1.12",
     "@types/node": "^18.0.0",
+    "dotenv": "16.4.5",
     "rimraf": "~3.0.2",
     "typescript": "^5.8.2",
     "vitest": "3.0.5"

packages/shared/vitest.config.ts

@@ -1,5 +1,4 @@
 import path from 'node:path';
-//@ts-ignore
 import dotenv from 'dotenv';
 import { defineConfig } from 'vitest/config';
 

packages/web-integration/README.md

@@ -4,15 +4,6 @@ Automate browser actions, extract data, and perform assertions using AI. It offe
 
 See https://midscenejs.com/ for details.
 
-## Android prerequisites
-
-Android is driven by adb, so you need install adb first:
-- [CLI](https://developer.android.com/tools/adb)
-
-```bash
-npm run test:ai -- adb
-```
-
 ## License
 
 Midscene is MIT licensed.

packages/web-integration/modern.config.ts

@@ -39,7 +39,6 @@ export default defineConfig({
       playwright: 'src/playwright/index.ts',
       playground: 'src/playground/index.ts',
       'midscene-playground': 'src/playground/bin.ts',
-      android: 'src/android/index.ts',
       'playwright-report': './src/playwright/reporter/index.ts',
       'chrome-extension': 'src/chrome-extension/index.ts',
       yaml: 'src/yaml/index.ts',

packages/web-integration/package.json

@@ -51,10 +51,6 @@
       "types": "./dist/types/midscene-playground.d.ts",
       "default": "./dist/lib/midscene-playground.js"
     },
-    "./android": {
-      "types": "./dist/types/android.d.ts",
-      "default": "./dist/lib/android.js"
-    },
     "./chrome-extension": {
       "types": "./dist/types/chrome-extension.d.ts",
       "default": "./dist/lib/chrome-extension.js"
@@ -76,7 +72,6 @@
       "playwright-report": ["./dist/types/playwright-report.d.ts"],
       "playground": ["./dist/types/playground.d.ts"],
       "midscene-playground": ["./dist/types/midscene-playground.d.ts"],
-      "android": ["./dist/types/android.d.ts"],
       "chrome-extension": ["./dist/types/chrome-extension.d.ts"],
       "yaml": ["./dist/types/yaml.d.ts"]
     }
@@ -89,12 +84,10 @@
     "build:watch": "modern build -w -c ./modern.config.ts",
     "test": "vitest --run",
     "test:u": "vitest --run -u",
-    "test:ai": "AI_TEST_TYPE=web npm run test",
-    "test:ai:temp": "MIDSCENE_CACHE=true AI_TEST_TYPE=web BRIDGE_MODE=true vitest --run tests/ai/bridge/open-new-tab.test.ts",
-    "test:ai:bridge": "MIDSCENE_CACHE=true BRIDGE_MODE=true AI_TEST_TYPE=web npm run test --inspect tests/ai/bridge/temp.test.ts",
-    "test:ai:cache": "MIDSCENE_CACHE=true AI_TEST_TYPE=web npm run test",
-    "test:ai:all": "npm run test:ai:web && npm run test:ai:native",
-    "test:ai:native": "MIDSCENE_CACHE=true AI_TEST_TYPE=native npm run test",
+    "test:ai": "npm run test",
+    "test:ai:temp": "MIDSCENE_CACHE=true BRIDGE_MODE=true vitest --run tests/ai/bridge/open-new-tab.test.ts",
+    "test:ai:bridge": "MIDSCENE_CACHE=true BRIDGE_MODE=true npm run test --inspect tests/ai/bridge/temp.test.ts",
+    "test:ai:cache": "MIDSCENE_CACHE=true npm run test",
     "upgrade": "modern upgrade",
     "prepublishOnly": "npm run build",
     "e2e": "playwright test --config=tests/playwright.config.ts",
@@ -108,7 +101,6 @@
     "@midscene/core": "workspace:*",
     "@midscene/shared": "workspace:*",
     "@xmldom/xmldom": "0.8.10",
-    "appium-adb": "12.12.1",
     "cors": "2.8.5",
     "dayjs": "1.11.11",
     "devtools-protocol": "0.0.1380148",

packages/web-integration/src/android/index.ts

@@ -1,2 +0,0 @@
-export { PageAgent as AndroidAgent } from '@/common/agent';
-export { Page as AndroidPage } from './page';

packages/web-integration/src/common/page.d.ts

@@ -1,5 +1,4 @@
 import type { KeyInput } from 'puppeteer';
-import type { AndroidPage } from '../android';
 import type ChromeExtensionProxyPage from '../chrome-extension/page';
 import type { StaticPage } from '../playground';
 import type { PlaywrightWebPage } from '../playwright';
@@ -8,7 +7,6 @@ import type { PuppeteerWebPage } from '../puppeteer';
 export type WebPage =
   | PlaywrightWebPage
   | PuppeteerWebPage
-  | AndroidPage
   | StaticPage
   | ChromeExtensionProxyPage;
 export type WebKeyInput = KeyInput;

packages/web-integration/src/index.ts

@@ -1,10 +1,11 @@
 export { PlaywrightAiFixture } from './playwright';
 export type { PlayWrightAiFixtureType } from './playwright';
 export type { WebPage } from './common/page';
+export type { AbstractPage } from './page';
 
+export { PageAgent } from './common/agent';
 export { PuppeteerAgent } from './puppeteer';
 export { PlaywrightAgent } from './playwright';
-export { AndroidAgent, AndroidPage } from './android';
 export { StaticPageAgent } from './playground/agent';
 
 export { ScriptPlayer, parseYamlScript } from './yaml';

packages/web-integration/src/puppeteer/base-page.ts

@@ -1,9 +1,8 @@
 import type { ElementTreeNode, Point, Size } from '@midscene/core';
-import { getTmpFile, sleep } from '@midscene/core/utils';
+import { sleep } from '@midscene/core/utils';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { treeToList } from '@midscene/shared/extractor';
 import { getExtraReturnLogic } from '@midscene/shared/fs';
-import { base64Encoded } from '@midscene/shared/img';
 import { assert, getDebug } from '@midscene/shared/utils';
 import type { Page as PlaywrightPage } from 'playwright';
 import type { Page as PuppeteerPage } from 'puppeteer';

packages/web-integration/tests/playwright.config.ts

@@ -1,7 +1,6 @@
 import path from 'node:path';
 // import { fileURLToPath } from 'node:url';
 import { defineConfig, devices } from '@playwright/test';
-//@ts-ignore
 import dotenv from 'dotenv';
 
 // 添加这两行获取当前文件目录（ES 模块兼容方式）

packages/web-integration/tests/tsconfig.json

@@ -4,6 +4,12 @@
     "baseUrl": "../",
     "rootDir": "../"
   },
-  "include": ["**/*", "../src", "playwright.config.ts"],
+  "include": [
+    "**/*",
+    "../src",
+    "playwright.config.ts",
+    "../../android/tests/setting.test.ts",
+    "../../android/tests/todo.test.ts"
+  ],
   "exclude": ["**/node_modules"]
 }

packages/web-integration/vitest.config.ts

@@ -1,5 +1,4 @@
 import path from 'node:path';
-//@ts-ignore
 import dotenv from 'dotenv';
 import { defineConfig } from 'vitest/config';
 import { version } from './package.json';
@@ -12,24 +11,7 @@ dotenv.config({
   path: path.join(__dirname, '../../.env'),
 });
 
-const aiTestType = process.env.AI_TEST_TYPE;
-const unitTests = ['tests/unit-test/**/*.test.ts'];
-const aiWebTests = [
-  'tests/ai/web/**/*.test.ts',
-  'tests/ai/bridge/**/*.test.ts',
-];
-const aiNativeTests = ['tests/ai/native/**/*.test.ts'];
-// const aiNativeTests = ['tests/ai/native/android/dongchedi.test.ts'];
-const testFiles = (() => {
-  switch (aiTestType) {
-    case 'web':
-      return [...aiWebTests];
-    case 'native':
-      return [...aiNativeTests];
-    default:
-      return unitTests;
-  }
-})();
+const testFiles = ['tests/ai/web/**/*.test.ts', 'tests/ai/bridge/**/*.test.ts'];
 
 export default defineConfig({
   resolve: {

pnpm-lock.yaml

@@ -136,9 +136,18 @@ importers:
 
   packages/android:
     dependencies:
+      '@midscene/core':
+        specifier: workspace:*
+        version: link:../core
+      '@midscene/shared':
+        specifier: workspace:*
+        version: link:../shared
       '@midscene/web':
         specifier: workspace:*
         version: link:../web-integration
+      appium-adb:
+        specifier: 12.12.1
+        version: 12.12.1
     devDependencies:
       '@modern-js/module-tools':
         specifier: 2.60.6
@@ -146,9 +155,15 @@ importers:
       '@types/node':
         specifier: ^18.0.0
         version: 18.19.62
+      dotenv:
+        specifier: 16.4.5
+        version: 16.4.5
       typescript:
         specifier: ^5.8.2
         version: 5.8.2
+      vitest:
+        specifier: 3.0.5
+        version: 3.0.5(@types/debug@4.1.12)(@types/node@18.19.62)(jsdom@24.1.1)(less@4.2.2)(sass-embedded@1.83.4)(terser@5.36.0)
 
   packages/cli:
     dependencies:
@@ -309,6 +324,9 @@ importers:
       '@types/node':
         specifier: ^18.0.0
         version: 18.19.62
+      dotenv:
+        specifier: 16.4.5
+        version: 16.4.5
       rimraf:
         specifier: ~3.0.2
         version: 3.0.2
@@ -421,9 +439,6 @@ importers:
       '@xmldom/xmldom':
         specifier: 0.8.10
         version: 0.8.10
-      appium-adb:
-        specifier: 12.12.1
-        version: 12.12.1
       cors:
         specifier: 2.8.5
         version: 2.8.5