feat: android yaml support (#551)

* feat: android yaml support

* feat: enhance ScriptPlayer to support web and android environments

* chore: update error message

* refactor: unify environment interfaces for YAML scripts

---------

Co-authored-by: yutao <yutao.tao@bytedance.com>

.gitignore

@@ -112,3 +112,4 @@ midscene_run/report
 midscene_run/dump
 
 extension_output
+.cursor

packages/cli/package.json

@@ -22,6 +22,7 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
+    "@midscene/android": "workspace:*",
     "@midscene/core": "workspace:*",
     "@midscene/web": "workspace:*",
     "puppeteer": "24.2.0",

packages/cli/src/yaml-runner.ts

@@ -12,9 +12,11 @@ import {
 import { TTYWindowRenderer } from './tty-renderer';
 
 import assert from 'node:assert';
-import type { FreeFn } from '@midscene/core';
+import { agentFromAdbDevice } from '@midscene/android';
+import type { FreeFn, MidsceneYamlScriptWebEnv } from '@midscene/core';
 import { AgentOverChromeBridge } from '@midscene/web/bridge-mode';
 import { puppeteerAgentForTarget } from '@midscene/web/puppeteer';
+
 export const launchServer = async (
   dir: string,
 ): Promise<ReturnType<typeof createServer>> => {
@@ -50,78 +52,110 @@ export async function playYamlFiles(
     };
     const player = new ScriptPlayer(script, async (target) => {
       const freeFn: FreeFn[] = [];
+      const webTarget = script.web || script.target;
 
-      // launch local server if needed
-      let localServer: Awaited<ReturnType<typeof launchServer>> | undefined;
-      let urlToVisit: string | undefined;
-      if (target.serve) {
-        assert(typeof target.url === 'string', 'url is required in serve mode');
-        localServer = await launchServer(target.serve);
-        const serverAddress = localServer.server.address();
-        freeFn.push({
-          name: 'local_server',
-          fn: () => localServer?.server.close(),
-        });
-        if (target.url.startsWith('/')) {
-          urlToVisit = `http://${serverAddress?.address}:${serverAddress?.port}${target.url}`;
-        } else {
-          urlToVisit = `http://${serverAddress?.address}:${serverAddress?.port}/${target.url}`;
+      // handle new web config
+      if (webTarget) {
+        if (script.target) {
+          console.warn(
+            'target is deprecated, please use web instead. See https://midscenejs.com/automate-with-scripts-in-yaml for more information. Sorry for the inconvenience.',
+          );
         }
-        target.url = urlToVisit;
+
+        // launch local server if needed
+        let localServer: Awaited<ReturnType<typeof launchServer>> | undefined;
+        let urlToVisit: string | undefined;
+        if (webTarget.serve) {
+          assert(
+            typeof webTarget.url === 'string',
+            'url is required in serve mode',
+          );
+          localServer = await launchServer(webTarget.serve);
+          const serverAddress = localServer.server.address();
+          freeFn.push({
+            name: 'local_server',
+            fn: () => localServer?.server.close(),
+          });
+          if (webTarget.url.startsWith('/')) {
+            urlToVisit = `http://${serverAddress?.address}:${serverAddress?.port}${webTarget.url}`;
+          } else {
+            urlToVisit = `http://${serverAddress?.address}:${serverAddress?.port}/${webTarget.url}`;
+          }
+          webTarget.url = urlToVisit;
+        }
+
+        if (!webTarget.bridgeMode) {
+          // 使用 puppeteer
+          const { agent, freeFn: newFreeFn } = await puppeteerAgentForTarget(
+            webTarget,
+            preference,
+          );
+          freeFn.push(...newFreeFn);
+
+          return { agent, freeFn };
+        }
+        assert(
+          webTarget.bridgeMode === 'newTabWithUrl' ||
+            webTarget.bridgeMode === 'currentTab',
+          `bridgeMode config value must be either "newTabWithUrl" or "currentTab", but got ${webTarget.bridgeMode}`,
+        );
+
+        if (
+          webTarget.userAgent ||
+          webTarget.viewportWidth ||
+          webTarget.viewportHeight ||
+          webTarget.viewportScale ||
+          webTarget.waitForNetworkIdle ||
+          webTarget.cookie
+        ) {
+          console.warn(
+            'puppeteer options (userAgent, viewportWidth, viewportHeight, viewportScale, waitForNetworkIdle, cookie) are not supported in bridge mode. They will be ignored.',
+          );
+        }
+
+        const agent = new AgentOverChromeBridge({
+          closeNewTabsAfterDisconnect: webTarget.closeNewTabsAfterDisconnect,
+          cacheId: fileName,
+        });
+
+        if (webTarget.bridgeMode === 'newTabWithUrl') {
+          await agent.connectNewTabWithUrl(webTarget.url);
+        } else {
+          if (webTarget.url) {
+            console.warn(
+              'url will be ignored in bridge mode with "currentTab"',
+            );
+          }
+          await agent.connectCurrentTab();
+        }
+        freeFn.push({
+          name: 'destroy_agent_over_chrome_bridge',
+          fn: () => agent.destroy(),
+        });
+        return {
+          agent,
+          freeFn,
+        };
       }
 
-      // puppeteer
-      if (!target.bridgeMode) {
-        const { agent, freeFn: newFreeFn } = await puppeteerAgentForTarget(
-          target,
-          preference,
-        );
-        freeFn.push(...newFreeFn);
+      // handle android
+      if (script.android) {
+        const androidTarget = script.android;
+        const agent = await agentFromAdbDevice(androidTarget.deviceId);
+
+        await agent.launch(androidTarget.launch);
+
+        freeFn.push({
+          name: 'destroy_android_agent',
+          fn: () => agent.destroy(),
+        });
 
         return { agent, freeFn };
       }
 
-      // bridge mode
-      assert(
-        target.bridgeMode === 'newTabWithUrl' ||
-          target.bridgeMode === 'currentTab',
-        `bridgeMode config value must be either "newTabWithUrl" or "currentTab", but got ${target.bridgeMode}`,
+      throw new Error(
+        'No valid target configuration found in the yaml script, should be either "web" or "android"',
       );
-
-      if (
-        target.userAgent ||
-        target.viewportWidth ||
-        target.viewportHeight ||
-        target.viewportScale ||
-        target.waitForNetworkIdle ||
-        target.cookie
-      ) {
-        console.warn(
-          'puppeteer options (userAgent, viewportWidth, viewportHeight, viewportScale, waitForNetworkIdle, cookie) are not supported in bridge mode. They will be ignored.',
-        );
-      }
-
-      const agent = new AgentOverChromeBridge({
-        closeNewTabsAfterDisconnect: target.closeNewTabsAfterDisconnect,
-        cacheId: fileName,
-      });
-
-      if (target.bridgeMode === 'newTabWithUrl') {
-        await agent.connectNewTabWithUrl(target.url);
-      } else {
-        if (target.url) {
-          console.warn('url will be ignored in bridge mode with "currentTab"');
-        }
-        await agent.connectCurrentTab();
-      }
-      freeFn.push({
-        name: 'destroy_agent_over_chrome_bridge',
-        fn: () => agent.destroy(),
-      });
-      return {
-        agent,
-        freeFn,
-      };
     });
     fileContextList.push({ file, player });
   }

packages/cli/tests/midscene_scripts/online/todomvc.yaml

@@ -1,5 +1,10 @@
 target:
   url: https://todomvc.com/
+
+# android:
+#   launch: https://todomvc.com/
+#   deviceId: s4ey59ytbitot4yp
+
 tasks:
   - name: scroll to bottom
     flow:

packages/core/src/yaml.ts

@@ -16,7 +16,9 @@ export interface scrollParam {
 }
 
 export interface MidsceneYamlScript {
-  target: MidsceneYamlScriptEnv;
+  target?: MidsceneYamlScriptWebEnv;
+  web?: MidsceneYamlScriptWebEnv;
+  android?: MidsceneYamlScriptAndroidEnv;
   tasks: MidsceneYamlTask[];
 }
 
@@ -26,10 +28,12 @@ export interface MidsceneYamlTask {
   continueOnError?: boolean;
 }
 
-export interface MidsceneYamlScriptEnv {
+export interface MidsceneYamlScriptEnvBase {
   output?: string;
   aiActionContext?: string;
+}
 
+export interface MidsceneYamlScriptWebEnv extends MidsceneYamlScriptEnvBase {
   // for web only
   serve?: string;
   url: string;
@@ -52,6 +56,19 @@ export interface MidsceneYamlScriptEnv {
   closeNewTabsAfterDisconnect?: boolean;
 }
 
+export interface MidsceneYamlScriptAndroidEnv
+  extends MidsceneYamlScriptEnvBase {
+  // The Android device ID to connect to, optional, will use the first device if not specified
+  deviceId?: string;
+
+  // The URL or app package to launch, optional, will use the current screen if not specified
+  launch?: string;
+}
+
+export type MidsceneYamlScriptEnv =
+  | MidsceneYamlScriptWebEnv
+  | MidsceneYamlScriptAndroidEnv;
+
 export interface MidsceneYamlFlowItemAIAction {
   ai?: string; // this is the shortcut for aiAction
   aiAction?: string;

packages/web-integration/src/puppeteer/agent-launcher.ts

@@ -3,7 +3,7 @@ import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
 
 import { PuppeteerAgent } from '@/puppeteer/index';
-import type { MidsceneYamlScriptEnv } from '@midscene/core';
+import type { MidsceneYamlScriptWebEnv } from '@midscene/core';
 import puppeteer from 'puppeteer';
 
 export const defaultUA =
@@ -21,7 +21,7 @@ interface FreeFn {
 const launcherDebug = getDebug('puppeteer:launcher');
 
 export async function launchPuppeteerPage(
-  target: MidsceneYamlScriptEnv,
+  target: MidsceneYamlScriptWebEnv,
   preference?: {
     headed?: boolean;
     keepWindow?: boolean;
@@ -163,7 +163,7 @@ export async function launchPuppeteerPage(
 }
 
 export async function puppeteerAgentForTarget(
-  target: MidsceneYamlScriptEnv,
+  target: MidsceneYamlScriptWebEnv,
   preference?: {
     headed?: boolean;
     keepWindow?: boolean;

packages/web-integration/src/yaml/builder.ts

@@ -1,12 +1,12 @@
 import type {
   MidsceneYamlScript,
-  MidsceneYamlScriptEnv,
+  MidsceneYamlScriptWebEnv,
   MidsceneYamlTask,
 } from '@midscene/core';
 import yaml from 'js-yaml';
 
 export function buildYaml(
-  env: MidsceneYamlScriptEnv,
+  env: MidsceneYamlScriptWebEnv,
   tasks: MidsceneYamlTask[],
 ) {
   const result: MidsceneYamlScript = {

packages/web-integration/src/yaml/player.ts

@@ -16,12 +16,14 @@ import type {
   MidsceneYamlFlowItemAIWaitFor,
   MidsceneYamlFlowItemSleep,
   MidsceneYamlScript,
+  MidsceneYamlScriptAndroidEnv,
   MidsceneYamlScriptEnv,
+  MidsceneYamlScriptWebEnv,
   ScriptPlayerStatusValue,
   ScriptPlayerTaskStatus,
 } from '@midscene/core';
 
-export class ScriptPlayer {
+export class ScriptPlayer<T extends MidsceneYamlScriptEnv> {
   public currentTaskIndex?: number;
   public taskStatusList: ScriptPlayerTaskStatus[] = [];
   public status: ScriptPlayerStatusValue = 'init';
@@ -34,7 +36,7 @@ export class ScriptPlayer {
   public agentStatusTip?: string;
   constructor(
     private script: MidsceneYamlScript,
-    private setupAgent: (target: MidsceneYamlScriptEnv) => Promise<{
+    private setupAgent: (platform: T) => Promise<{
       agent: PageAgent;
       freeFn: FreeFn[];
     }>,
@@ -199,14 +201,19 @@ export class ScriptPlayer {
   }
 
   async run() {
-    const { target, tasks } = this.script;
+    const { target, web, android, tasks } = this.script;
+    const webEnv = web || target;
+    const androidEnv = android;
+    const platform = webEnv || androidEnv;
+
     this.setPlayerStatus('running');
 
     let agent: PageAgent | null = null;
     let freeFn: FreeFn[] = [];
     try {
-      const { agent: newAgent, freeFn: newFreeFn } =
-        await this.setupAgent(target);
+      const { agent: newAgent, freeFn: newFreeFn } = await this.setupAgent(
+        platform as T,
+      );
       agent = newAgent;
       agent.onTaskStartTip = (tip) => {
         if (this.status === 'running') {

packages/web-integration/src/yaml/utils.ts

@@ -2,14 +2,6 @@ import { assert } from '@midscene/shared/utils';
 import yaml from 'js-yaml';
 
 import type { MidsceneYamlScript } from '@midscene/core';
-import type {
-  MidsceneYamlFlowItem,
-  MidsceneYamlFlowItemAIAction,
-  MidsceneYamlFlowItemAIAssert,
-  MidsceneYamlFlowItemAIQuery,
-  MidsceneYamlFlowItemAIWaitFor,
-  MidsceneYamlFlowItemSleep,
-} from '@midscene/core';
 
 function interpolateEnvVars(content: string): string {
   return content.replace(/\$\{([^}]+)\}/g, (_, envVar) => {
@@ -29,16 +21,31 @@ export function parseYamlScript(
   const interpolatedContent = interpolateEnvVars(content);
   const obj = yaml.load(interpolatedContent) as MidsceneYamlScript;
   const pathTip = filePath ? `, failed to load ${filePath}` : '';
+  const web = obj.web || obj.target;
+  const android = obj.android;
+
   if (!ignoreCheckingTarget) {
+    // make sure at least one of target/web/android is provided
     assert(
-      obj.target,
-      `property "target" is required in yaml script${pathTip}`,
+      web || android,
+      `at least one of "target", "web", or "android" properties is required in yaml script${pathTip}`,
     );
+
+    // make sure only one of target/web/android is provided
     assert(
-      typeof obj.target === 'object',
-      `property "target" must be an object${pathTip}`,
+      (web && !android) || (!web && android),
+      `only one of "target", "web", or "android" properties is allowed in yaml script${pathTip}`,
     );
+
+    // make sure the config is valid
+    if (web || android) {
+      assert(
+        typeof web === 'object' || typeof android === 'object',
+        `property "target/web/android" must be an object${pathTip}`,
+      );
+    }
   }
+
   assert(obj.tasks, `property "tasks" is required in yaml script ${pathTip}`);
   assert(
     Array.isArray(obj.tasks),

packages/web-integration/tests/unit-test/yaml/player.test.ts

@@ -5,6 +5,7 @@ import { randomUUID } from 'node:crypto';
 import { existsSync } from 'node:fs';
 import { puppeteerAgentForTarget } from '@/puppeteer';
 import { ScriptPlayer, buildYaml, parseYamlScript } from '@/yaml';
+import type { MidsceneYamlScriptWebEnv } from '@midscene/core';
 import { describe, expect, test, vi } from 'vitest';
 
 const serverRoot = join(__dirname, 'server_root');
@@ -12,7 +13,7 @@ const serverRoot = join(__dirname, 'server_root');
 const runYaml = async (yamlString: string, ignoreStatusAssertion = false) => {
   const script = parseYamlScript(yamlString);
   const statusUpdate = vi.fn();
-  const player = new ScriptPlayer(
+  const player = new ScriptPlayer<MidsceneYamlScriptWebEnv>(
     script,
     puppeteerAgentForTarget,
     statusUpdate,

pnpm-lock.yaml

@@ -237,6 +237,9 @@ importers:
 
   packages/cli:
     dependencies:
+      '@midscene/android':
+        specifier: workspace:*
+        version: link:../android
       '@midscene/core':
         specifier: workspace:*
         version: link:../core