From bf9b4e06e7c66d593833eb2059fea7bb826f604c Mon Sep 17 00:00:00 2001 From: Leyang Date: Wed, 26 Mar 2025 18:59:50 +0800 Subject: [PATCH] feat: migrate android code to @midscene/android (#505) --- packages/android/.gitignore | 4 +++ packages/android/README.md | 14 ++++++++ .../{web-integration => android}/bin/yadb | Bin packages/android/modern.config.ts | 1 + packages/android/package.json | 21 ++++++++---- packages/android/src/index.ts | 3 +- .../page.ts => android/src/page/index.ts} | 31 +++++------------- .../tests/ai}/setting.test.ts | 2 +- .../android => android/tests/ai}/todo.test.ts | 3 +- .../android => android/tests/ai}/utils.ts | 2 +- packages/android/vitest.config.ts | 30 +++++++++++++++++ packages/core/vitest.config.ts | 1 - packages/evaluation/vitest.config.ts | 1 - packages/shared/package.json | 1 + packages/shared/vitest.config.ts | 1 - packages/web-integration/README.md | 9 ----- packages/web-integration/modern.config.ts | 1 - packages/web-integration/package.json | 16 +++------ packages/web-integration/src/android/index.ts | 2 -- packages/web-integration/src/common/page.d.ts | 2 -- packages/web-integration/src/index.ts | 3 +- .../src/puppeteer/base-page.ts | 3 +- .../tests/playwright.config.ts | 1 - packages/web-integration/tests/tsconfig.json | 8 ++++- packages/web-integration/vitest.config.ts | 20 +---------- pnpm-lock.yaml | 21 ++++++++++-- 26 files changed, 110 insertions(+), 91 deletions(-) create mode 100644 packages/android/.gitignore create mode 100644 packages/android/README.md rename packages/{web-integration => android}/bin/yadb (100%) rename packages/{web-integration/src/android/page.ts => android/src/page/index.ts} (93%) rename packages/{web-integration/tests/ai/native/android => android/tests/ai}/setting.test.ts (95%) rename packages/{web-integration/tests/ai/native/android => android/tests/ai}/todo.test.ts (93%) rename packages/{web-integration/tests/ai/native/android => android/tests/ai}/utils.ts (98%) create mode 100644 packages/android/vitest.config.ts delete mode 100644 packages/web-integration/src/android/index.ts diff --git a/packages/android/.gitignore b/packages/android/.gitignore new file mode 100644 index 000000000..46078517a --- /dev/null +++ b/packages/android/.gitignore @@ -0,0 +1,4 @@ + +# Midscene.js dump files +midscene_run/report +midscene_run/tmp diff --git a/packages/android/README.md b/packages/android/README.md new file mode 100644 index 000000000..68ec3aee3 --- /dev/null +++ b/packages/android/README.md @@ -0,0 +1,14 @@ +## Android prerequisites + +Android is driven by adb, so you need install adb first: +- [CLI](https://developer.android.com/tools/adb) + +```bash +npm run test:ai -- setting.test.ts +``` + +or + +```bash +npm run test:ai -- todo.test.ts +``` \ No newline at end of file diff --git a/packages/web-integration/bin/yadb b/packages/android/bin/yadb similarity index 100% rename from packages/web-integration/bin/yadb rename to packages/android/bin/yadb diff --git a/packages/android/modern.config.ts b/packages/android/modern.config.ts index 4cd92c1b6..18f19a23c 100644 --- a/packages/android/modern.config.ts +++ b/packages/android/modern.config.ts @@ -1,3 +1,4 @@ +import path from 'node:path'; import { defineConfig, moduleTools } from '@modern-js/module-tools'; export default defineConfig({ diff --git a/packages/android/package.json b/packages/android/package.json index 6ba7aacf2..956f9b436 100644 --- a/packages/android/package.json +++ b/packages/android/package.json @@ -1,9 +1,10 @@ { "name": "@midscene/android", "version": "0.13.1", + "description": "Android automation library for Midscene", "main": "./dist/lib/index.js", "types": "./dist/types/index.d.ts", - "files": ["dist"], + "files": ["dist", "bin", "README.md"], "exports": { ".": { "types": "./dist/types/index.d.ts", @@ -11,17 +12,23 @@ } }, "scripts": { - "build": "modern build -c ./modern.config.ts" + "build": "modern build -c ./modern.config.ts", + "test": "vitest --run", + "test:u": "vitest --run -u", + "test:ai": "MIDSCENE_CACHE=true npm run test" }, "dependencies": { - "@midscene/web": "workspace:*" + "@midscene/web": "workspace:*", + "@midscene/core": "workspace:*", + "@midscene/shared": "workspace:*", + "appium-adb": "12.12.1" }, "devDependencies": { "@modern-js/module-tools": "2.60.6", "@types/node": "^18.0.0", - "typescript": "^5.8.2" + "dotenv": "16.4.5", + "typescript": "^5.8.2", + "vitest": "3.0.5" }, - "author": "", - "license": "ISC", - "description": "" + "license": "MIT" } diff --git a/packages/android/src/index.ts b/packages/android/src/index.ts index 46c802e90..81f100e99 100644 --- a/packages/android/src/index.ts +++ b/packages/android/src/index.ts @@ -1 +1,2 @@ -export { AndroidAgent, AndroidPage } from '@midscene/web/android'; +export { PageAgent as AndroidAgent } from '@midscene/web'; +export { Page as AndroidPage } from './page'; diff --git a/packages/web-integration/src/android/page.ts b/packages/android/src/page/index.ts similarity index 93% rename from packages/web-integration/src/android/page.ts rename to packages/android/src/page/index.ts index 30061f637..115d556c6 100644 --- a/packages/web-integration/src/android/page.ts +++ b/packages/android/src/page/index.ts @@ -5,10 +5,8 @@ import { getTmpFile } from '@midscene/core/utils'; import type { ElementInfo } from '@midscene/shared/extractor'; import { resizeImg } from '@midscene/shared/img'; import { getDebug } from '@midscene/shared/utils'; +import type { AbstractPage } from '@midscene/web'; import { ADB } from 'appium-adb'; -import type { KeyInput as PuppeteerKeyInput } from 'puppeteer'; -import type { AbstractPage, MouseButton } from '../page'; -type WebKeyInput = PuppeteerKeyInput; const debugPage = getDebug('android'); @@ -177,8 +175,7 @@ export class Page implements AbstractPage { get mouse() { return { - click: (x: number, y: number, options?: { button: MouseButton }) => - this.mouseClick(x, y, options?.button || 'left'), + click: (x: number, y: number) => this.mouseClick(x, y), wheel: (deltaX: number, deltaY: number) => this.mouseWheel(deltaX, deltaY), move: (x: number, y: number) => this.mouseMove(x, y), @@ -192,8 +189,8 @@ export class Page implements AbstractPage { type: (text: string) => this.keyboardType(text), press: ( action: - | { key: WebKeyInput; command?: string } - | { key: WebKeyInput; command?: string }[], + | { key: string; command?: string } + | { key: string; command?: string }[], ) => this.keyboardPressAction(action), }; } @@ -301,7 +298,6 @@ export class Page implements AbstractPage { // Push the YADB tool to the device only once if (!this.yadbPushed) { const adb = await this.getAdb(); - const yadbBin = path.join(__dirname, '../../bin/yadb'); await adb.push(yadbBin, '/data/local/tmp'); this.yadbPushed = true; @@ -323,7 +319,7 @@ export class Page implements AbstractPage { await this.execYadb(text); } - private async keyboardPress(key: WebKeyInput): Promise { + private async keyboardPress(key: string): Promise { // Map web keys to Android key codes (numbers) const keyCodeMap: Record = { Enter: 66, @@ -357,8 +353,8 @@ export class Page implements AbstractPage { private async keyboardPressAction( action: - | { key: WebKeyInput; command?: string } - | { key: WebKeyInput; command?: string }[], + | { key: string; command?: string } + | { key: string; command?: string }[], ): Promise { if (Array.isArray(action)) { for (const act of action) { @@ -369,18 +365,7 @@ export class Page implements AbstractPage { } } - private async mouseClick( - x: number, - y: number, - button: MouseButton = 'left', - ): Promise { - // ADB only supports left mouse button clicks - if (button !== 'left') { - console.warn( - `ADB only supports left mouse button clicks. Ignored request for ${button} button.`, - ); - } - + private async mouseClick(x: number, y: number): Promise { await this.mouseMove(x, y); const adb = await this.getAdb(); diff --git a/packages/web-integration/tests/ai/native/android/setting.test.ts b/packages/android/tests/ai/setting.test.ts similarity index 95% rename from packages/web-integration/tests/ai/native/android/setting.test.ts rename to packages/android/tests/ai/setting.test.ts index 8d5a29466..aafc19b7e 100644 --- a/packages/web-integration/tests/ai/native/android/setting.test.ts +++ b/packages/android/tests/ai/setting.test.ts @@ -1,5 +1,5 @@ -import { AndroidAgent } from '@/android'; import { describe, it, vi } from 'vitest'; +import { AndroidAgent } from '../../src'; import { launchPage } from './utils'; vi.setConfig({ diff --git a/packages/web-integration/tests/ai/native/android/todo.test.ts b/packages/android/tests/ai/todo.test.ts similarity index 93% rename from packages/web-integration/tests/ai/native/android/todo.test.ts rename to packages/android/tests/ai/todo.test.ts index e43b6bbb0..87912b108 100644 --- a/packages/web-integration/tests/ai/native/android/todo.test.ts +++ b/packages/android/tests/ai/todo.test.ts @@ -1,7 +1,6 @@ -import { AndroidAgent } from '@/android'; import { beforeAll, describe, expect, it, vi } from 'vitest'; +import { AndroidAgent } from '../../src'; import { launchPage } from './utils'; -import 'dotenv/config'; // read environment variables from .env file vi.setConfig({ testTimeout: 240 * 1000, diff --git a/packages/web-integration/tests/ai/native/android/utils.ts b/packages/android/tests/ai/utils.ts similarity index 98% rename from packages/web-integration/tests/ai/native/android/utils.ts rename to packages/android/tests/ai/utils.ts index 792b7b2ab..65e186ee6 100644 --- a/packages/web-integration/tests/ai/native/android/utils.ts +++ b/packages/android/tests/ai/utils.ts @@ -1,7 +1,7 @@ import { exec } from 'node:child_process'; import { promisify } from 'node:util'; import type { StartAppOptions } from 'appium-adb'; -import { AndroidPage } from '../../../../src/android'; +import { Page as AndroidPage } from '../../src/page'; const execPromise = promisify(exec); interface LaunchOptions { diff --git a/packages/android/vitest.config.ts b/packages/android/vitest.config.ts new file mode 100644 index 000000000..3532ed3df --- /dev/null +++ b/packages/android/vitest.config.ts @@ -0,0 +1,30 @@ +import path from 'node:path'; +import dotenv from 'dotenv'; +import { defineConfig } from 'vitest/config'; +import { version } from './package.json'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +dotenv.config({ + path: path.join(__dirname, '../../.env'), +}); + +const testFiles = ['tests/ai/**/*.test.ts']; + +export default defineConfig({ + resolve: { + alias: { + '@': path.resolve(__dirname, 'src'), + }, + }, + test: { + include: testFiles, + testTimeout: 3 * 60 * 1000, // Global timeout set to 10 seconds + dangerouslyIgnoreUnhandledErrors: !!process.env.CI, // showcase.test.ts is not stable + }, + define: { + __VERSION__: `'${version}'`, + }, +}); diff --git a/packages/core/vitest.config.ts b/packages/core/vitest.config.ts index b0b33b9ae..bb531f1c0 100644 --- a/packages/core/vitest.config.ts +++ b/packages/core/vitest.config.ts @@ -1,5 +1,4 @@ import path from 'node:path'; -//@ts-ignore import dotenv from 'dotenv'; import { defineConfig } from 'vitest/config'; import { version } from './package.json'; diff --git a/packages/evaluation/vitest.config.ts b/packages/evaluation/vitest.config.ts index 64607c4ea..965c646b5 100644 --- a/packages/evaluation/vitest.config.ts +++ b/packages/evaluation/vitest.config.ts @@ -1,5 +1,4 @@ import path from 'node:path'; -//@ts-ignore import dotenv from 'dotenv'; import { defineConfig } from 'vitest/config'; diff --git a/packages/shared/package.json b/packages/shared/package.json index b538cd902..d07340fda 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -57,6 +57,7 @@ "@modern-js/module-tools": "2.60.6", "@types/debug": "4.1.12", "@types/node": "^18.0.0", + "dotenv": "16.4.5", "rimraf": "~3.0.2", "typescript": "^5.8.2", "vitest": "3.0.5" diff --git a/packages/shared/vitest.config.ts b/packages/shared/vitest.config.ts index fbfac7f66..47549fa03 100644 --- a/packages/shared/vitest.config.ts +++ b/packages/shared/vitest.config.ts @@ -1,5 +1,4 @@ import path from 'node:path'; -//@ts-ignore import dotenv from 'dotenv'; import { defineConfig } from 'vitest/config'; diff --git a/packages/web-integration/README.md b/packages/web-integration/README.md index 15fa83404..b0e721a44 100644 --- a/packages/web-integration/README.md +++ b/packages/web-integration/README.md @@ -4,15 +4,6 @@ Automate browser actions, extract data, and perform assertions using AI. It offe See https://midscenejs.com/ for details. -## Android prerequisites - -Android is driven by adb, so you need install adb first: -- [CLI](https://developer.android.com/tools/adb) - -```bash -npm run test:ai -- adb -``` - ## License Midscene is MIT licensed. \ No newline at end of file diff --git a/packages/web-integration/modern.config.ts b/packages/web-integration/modern.config.ts index 37f4ac8b6..84bfe1063 100644 --- a/packages/web-integration/modern.config.ts +++ b/packages/web-integration/modern.config.ts @@ -39,7 +39,6 @@ export default defineConfig({ playwright: 'src/playwright/index.ts', playground: 'src/playground/index.ts', 'midscene-playground': 'src/playground/bin.ts', - android: 'src/android/index.ts', 'playwright-report': './src/playwright/reporter/index.ts', 'chrome-extension': 'src/chrome-extension/index.ts', yaml: 'src/yaml/index.ts', diff --git a/packages/web-integration/package.json b/packages/web-integration/package.json index 1214b8a2c..77dfdb64f 100644 --- a/packages/web-integration/package.json +++ b/packages/web-integration/package.json @@ -51,10 +51,6 @@ "types": "./dist/types/midscene-playground.d.ts", "default": "./dist/lib/midscene-playground.js" }, - "./android": { - "types": "./dist/types/android.d.ts", - "default": "./dist/lib/android.js" - }, "./chrome-extension": { "types": "./dist/types/chrome-extension.d.ts", "default": "./dist/lib/chrome-extension.js" @@ -76,7 +72,6 @@ "playwright-report": ["./dist/types/playwright-report.d.ts"], "playground": ["./dist/types/playground.d.ts"], "midscene-playground": ["./dist/types/midscene-playground.d.ts"], - "android": ["./dist/types/android.d.ts"], "chrome-extension": ["./dist/types/chrome-extension.d.ts"], "yaml": ["./dist/types/yaml.d.ts"] } @@ -89,12 +84,10 @@ "build:watch": "modern build -w -c ./modern.config.ts", "test": "vitest --run", "test:u": "vitest --run -u", - "test:ai": "AI_TEST_TYPE=web npm run test", - "test:ai:temp": "MIDSCENE_CACHE=true AI_TEST_TYPE=web BRIDGE_MODE=true vitest --run tests/ai/bridge/open-new-tab.test.ts", - "test:ai:bridge": "MIDSCENE_CACHE=true BRIDGE_MODE=true AI_TEST_TYPE=web npm run test --inspect tests/ai/bridge/temp.test.ts", - "test:ai:cache": "MIDSCENE_CACHE=true AI_TEST_TYPE=web npm run test", - "test:ai:all": "npm run test:ai:web && npm run test:ai:native", - "test:ai:native": "MIDSCENE_CACHE=true AI_TEST_TYPE=native npm run test", + "test:ai": "npm run test", + "test:ai:temp": "MIDSCENE_CACHE=true BRIDGE_MODE=true vitest --run tests/ai/bridge/open-new-tab.test.ts", + "test:ai:bridge": "MIDSCENE_CACHE=true BRIDGE_MODE=true npm run test --inspect tests/ai/bridge/temp.test.ts", + "test:ai:cache": "MIDSCENE_CACHE=true npm run test", "upgrade": "modern upgrade", "prepublishOnly": "npm run build", "e2e": "playwright test --config=tests/playwright.config.ts", @@ -108,7 +101,6 @@ "@midscene/core": "workspace:*", "@midscene/shared": "workspace:*", "@xmldom/xmldom": "0.8.10", - "appium-adb": "12.12.1", "cors": "2.8.5", "dayjs": "1.11.11", "devtools-protocol": "0.0.1380148", diff --git a/packages/web-integration/src/android/index.ts b/packages/web-integration/src/android/index.ts deleted file mode 100644 index 36672c98f..000000000 --- a/packages/web-integration/src/android/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export { PageAgent as AndroidAgent } from '@/common/agent'; -export { Page as AndroidPage } from './page'; diff --git a/packages/web-integration/src/common/page.d.ts b/packages/web-integration/src/common/page.d.ts index 79c43a089..deba26600 100644 --- a/packages/web-integration/src/common/page.d.ts +++ b/packages/web-integration/src/common/page.d.ts @@ -1,5 +1,4 @@ import type { KeyInput } from 'puppeteer'; -import type { AndroidPage } from '../android'; import type ChromeExtensionProxyPage from '../chrome-extension/page'; import type { StaticPage } from '../playground'; import type { PlaywrightWebPage } from '../playwright'; @@ -8,7 +7,6 @@ import type { PuppeteerWebPage } from '../puppeteer'; export type WebPage = | PlaywrightWebPage | PuppeteerWebPage - | AndroidPage | StaticPage | ChromeExtensionProxyPage; export type WebKeyInput = KeyInput; diff --git a/packages/web-integration/src/index.ts b/packages/web-integration/src/index.ts index 55714d08e..c7a50e5ba 100644 --- a/packages/web-integration/src/index.ts +++ b/packages/web-integration/src/index.ts @@ -1,10 +1,11 @@ export { PlaywrightAiFixture } from './playwright'; export type { PlayWrightAiFixtureType } from './playwright'; export type { WebPage } from './common/page'; +export type { AbstractPage } from './page'; +export { PageAgent } from './common/agent'; export { PuppeteerAgent } from './puppeteer'; export { PlaywrightAgent } from './playwright'; -export { AndroidAgent, AndroidPage } from './android'; export { StaticPageAgent } from './playground/agent'; export { ScriptPlayer, parseYamlScript } from './yaml'; diff --git a/packages/web-integration/src/puppeteer/base-page.ts b/packages/web-integration/src/puppeteer/base-page.ts index ad048abba..988a3bcdc 100644 --- a/packages/web-integration/src/puppeteer/base-page.ts +++ b/packages/web-integration/src/puppeteer/base-page.ts @@ -1,9 +1,8 @@ import type { ElementTreeNode, Point, Size } from '@midscene/core'; -import { getTmpFile, sleep } from '@midscene/core/utils'; +import { sleep } from '@midscene/core/utils'; import type { ElementInfo } from '@midscene/shared/extractor'; import { treeToList } from '@midscene/shared/extractor'; import { getExtraReturnLogic } from '@midscene/shared/fs'; -import { base64Encoded } from '@midscene/shared/img'; import { assert, getDebug } from '@midscene/shared/utils'; import type { Page as PlaywrightPage } from 'playwright'; import type { Page as PuppeteerPage } from 'puppeteer'; diff --git a/packages/web-integration/tests/playwright.config.ts b/packages/web-integration/tests/playwright.config.ts index 88b44cfc7..b97f037d7 100644 --- a/packages/web-integration/tests/playwright.config.ts +++ b/packages/web-integration/tests/playwright.config.ts @@ -1,7 +1,6 @@ import path from 'node:path'; // import { fileURLToPath } from 'node:url'; import { defineConfig, devices } from '@playwright/test'; -//@ts-ignore import dotenv from 'dotenv'; // 添加这两行获取当前文件目录（ES 模块兼容方式） diff --git a/packages/web-integration/tests/tsconfig.json b/packages/web-integration/tests/tsconfig.json index 284973abc..fc04f9aa9 100644 --- a/packages/web-integration/tests/tsconfig.json +++ b/packages/web-integration/tests/tsconfig.json @@ -4,6 +4,12 @@ "baseUrl": "../", "rootDir": "../" }, - "include": ["**/*", "../src", "playwright.config.ts"], + "include": [ + "**/*", + "../src", + "playwright.config.ts", + "../../android/tests/setting.test.ts", + "../../android/tests/todo.test.ts" + ], "exclude": ["**/node_modules"] } diff --git a/packages/web-integration/vitest.config.ts b/packages/web-integration/vitest.config.ts index 8e6fcde40..a6f5dcf09 100644 --- a/packages/web-integration/vitest.config.ts +++ b/packages/web-integration/vitest.config.ts @@ -1,5 +1,4 @@ import path from 'node:path'; -//@ts-ignore import dotenv from 'dotenv'; import { defineConfig } from 'vitest/config'; import { version } from './package.json'; @@ -12,24 +11,7 @@ dotenv.config({ path: path.join(__dirname, '../../.env'), }); -const aiTestType = process.env.AI_TEST_TYPE; -const unitTests = ['tests/unit-test/**/*.test.ts']; -const aiWebTests = [ - 'tests/ai/web/**/*.test.ts', - 'tests/ai/bridge/**/*.test.ts', -]; -const aiNativeTests = ['tests/ai/native/**/*.test.ts']; -// const aiNativeTests = ['tests/ai/native/android/dongchedi.test.ts']; -const testFiles = (() => { - switch (aiTestType) { - case 'web': - return [...aiWebTests]; - case 'native': - return [...aiNativeTests]; - default: - return unitTests; - } -})(); +const testFiles = ['tests/ai/web/**/*.test.ts', 'tests/ai/bridge/**/*.test.ts']; export default defineConfig({ resolve: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 08096574d..946fc9e4d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -136,9 +136,18 @@ importers: packages/android: dependencies: + '@midscene/core': + specifier: workspace:* + version: link:../core + '@midscene/shared': + specifier: workspace:* + version: link:../shared '@midscene/web': specifier: workspace:* version: link:../web-integration + appium-adb: + specifier: 12.12.1 + version: 12.12.1 devDependencies: '@modern-js/module-tools': specifier: 2.60.6 @@ -146,9 +155,15 @@ importers: '@types/node': specifier: ^18.0.0 version: 18.19.62 + dotenv: + specifier: 16.4.5 + version: 16.4.5 typescript: specifier: ^5.8.2 version: 5.8.2 + vitest: + specifier: 3.0.5 + version: 3.0.5(@types/debug@4.1.12)(@types/node@18.19.62)(jsdom@24.1.1)(less@4.2.2)(sass-embedded@1.83.4)(terser@5.36.0) packages/cli: dependencies: @@ -309,6 +324,9 @@ importers: '@types/node': specifier: ^18.0.0 version: 18.19.62 + dotenv: + specifier: 16.4.5 + version: 16.4.5 rimraf: specifier: ~3.0.2 version: 3.0.2 @@ -421,9 +439,6 @@ importers: '@xmldom/xmldom': specifier: 0.8.10 version: 0.8.10 - appium-adb: - specifier: 12.12.1 - version: 12.12.1 cors: specifier: 2.8.5 version: 2.8.5