feat(web-integration): enhance timeout configurations and logging for network idle and navigation (#624)

* feat(web-integration): enhance timeout configurations and logging for network idle and navigation

* fix(web-integration): refine timeout warning messages and remove unnecessary test files

* feat(site): add network timeout customization details and additional parameters for Puppeteer

* fix(site): update default timeout values and enhance customization options for network idle in YAML

* fix(site): remove redundant timeout customization details in FAQ documentation

* fix(web-integration): enhance Playwright agent to support network idle functionality

* docs(playwright): update config docs

* docs(playwright): update config docs

* fix(web-integration): refactor network idle handling in Playwright agent

---------

Co-authored-by: yutao <yutao.tao@bytedance.com>

apps/site/docs/en/API.mdx

@@ -17,10 +17,11 @@ These Agents share some common constructor parameters:
 * `cacheId: string | undefined`: If provided, this cacheId will be used to save or match the cache. (Default: undefined, means cache feature is disabled)
 * `actionContext: string`: Some background knowledge that should be sent to the AI model when calling `agent.aiAction()`, like 'close the cookie consent dialog first if it exists' (Default: undefined)
 
-In Puppeteer, there is an additional parameter:
+In Puppeteer, there are 3 additional parameters:
 
 * `forceSameTabNavigation: boolean`: If true, page navigation is restricted to the current tab. (Default: true)
-
+* `waitForNetworkIdleTimeout: number`: The timeout for waiting for network idle between each action. (Default: 2000ms)
+* `waitForNavigationTimeout: number`: The timeout for waiting for navigation finished. (Default: 5000ms)
 
 ## Interaction Methods
 

apps/site/docs/en/automate-with-scripts-in-yaml.mdx

@@ -140,7 +140,7 @@ web:
   
   # object, the strategy to wait for network idle, optional
   waitForNetworkIdle:
-    # number, the timeout in milliseconds, 10000ms for default, optional
+    # number, the timeout in milliseconds, 2000ms for default, optional
     timeout: <ms>
     # boolean, continue on network idle error, true for default
     continueOnNetworkIdleError: <boolean>

apps/site/docs/en/faq.md

@@ -62,4 +62,18 @@ The report files are saved in `./midscene-run/report/` by default.
 
 ## How can I learn about Midscene's working process?
 
-⁠By reviewing the report file after running the script, you can gain an overview of how Midscene works. 
+⁠By reviewing the report file after running the script, you can gain an overview of how Midscene works. 
+
+## Customize the network timeout
+
+When doing interaction or navigation on web page, Midscene automatically waits for the network to be idle. It's a strategy to ensure the stability of the automation. Nothing would happen if the waiting process is timeout. 
+
+The default timeout is configured as follows:
+
+1. If it's a page navigation, the default wait timeout is 5000ms (the `waitForNavigationTimeout`)
+2. If it's a click, input, etc., the default wait timeout is 2000ms (the `waitForNetworkIdleTimeout`)
+
+You can also customize the timeout by options
+
+- Use `waitForNetworkIdleTimeout` and `waitForNavigationTimeout` parameters in [Agent](/api.html#constructors) or [PlaywrightAiFixture](/integrate-with-playwright.html#step-2-extend-the-test-instance).
+- Use `waitForNetworkIdle` parameter in [Yaml](/automate-with-scripts-in-yaml.html#the-web-part).

apps/site/docs/en/integrate-with-playwright.mdx

@@ -39,7 +39,10 @@ import { test as base } from '@playwright/test';
 import type { PlayWrightAiFixtureType } from '@midscene/web/playwright';
 import { PlaywrightAiFixture } from '@midscene/web/playwright';
 
-export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
+export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture({
+  waitForNetworkIdleTimeout: 2000, // optional, the timeout for waiting for network idle between each action, default is 2000ms
+  waitForNavigationTimeout: 5000, // optional, the timeout for waiting for navigation finished, default is 5000ms
+}));
 ```
 
 ## Step 3: write test cases

apps/site/docs/zh/API.mdx

@@ -16,9 +16,11 @@ Midscene 中每个 Agent 都有自己的构造函数。
 * `cacheId: string | undefined`: 如果配置，则使用此 cacheId 保存或匹配缓存。默认值为 undefined，也就是不启用缓存。
 * `actionContext: string`: 调用 `agent.aiAction()` 时，发送给 AI 模型的背景知识，比如 '有 cookie 对话框时先关闭它'，默认值为空。
 
-在 puppeteer 中，还有一个额外的参数：
+在 puppeteer 中，还有三个额外的参数：
 
 * `forceSameTabNavigation: boolean`: 如果为 true，则限制页面在当前 tab 打开。默认值为 true。
+* `waitForNetworkIdleTimeout: number`: 在执行每个操作后等待网络空闲的超时时间，默认值为 2000ms。
+* `waitForNavigationTimeout: number`: 在页面跳转后等待页面加载完成的超时时间，默认值为 5000ms。
 
 ## 交互方法
 

apps/site/docs/zh/automate-with-scripts-in-yaml.mdx

@@ -140,7 +140,7 @@ web:
 
   # 等待网络空闲的策略，可选
   waitForNetworkIdle:
-    # 等待超时时间，可选，默认 10000ms
+    # 等待超时时间，可选，默认 2000ms
     timeout: <ms>
     # 是否在等待超时后继续，可选，默认 true
     continueOnNetworkIdleError: <boolean>

apps/site/docs/zh/faq.md

@@ -58,4 +58,18 @@ await page.setViewport({
 
 ## 如何了解 Midscene 的运行原理？
 
-在运行脚本后，通过查看报告文件，你可以了解 Midscene 的大致运行原理。
+在运行脚本后，通过查看报告文件，你可以了解 Midscene 的大致运行原理。
+
+## 自定义网络超时
+
+当在网页上执行某个操作后，Midscene 会自动等待网络空闲。这是为了确保自动化过程的稳定性。如果等待超时，不会发生任何事情。
+
+默认的超时时间配置如下：
+
+1. 如果是页面跳转，则等待页面加载完成，默认超时时间为 5000ms
+2. 如果是点击、输入等操作，则等待网络空闲，默认超时时间为 2000ms
+
+当然，你可以通过配置参数修改默认超时时间
+
+- 使用 [Agent](/zh/api.html#%E6%9E%84%E9%80%A0%E5%99%A8) 和 [PlaywrightAiFixture](/zh/integrate-with-playwright.html#%E7%AC%AC%E4%BA%8C%E6%AD%A5%E6%89%A9%E5%B1%95-test-%E5%AE%9E%E4%BE%8B) 上的 `waitForNetworkIdleTimeout` 和 `waitForNavigationTimeout` 参数
+- 使用 [Yaml](/zh/automate-with-scripts-in-yaml.html#web-%E9%83%A8%E5%88%86) 脚本中的 `waitForNetworkIdle` 参数

apps/site/docs/zh/integrate-with-playwright.mdx

@@ -40,7 +40,10 @@ import { test as base } from '@playwright/test';
 import type { PlayWrightAiFixtureType } from '@midscene/web/playwright';
 import { PlaywrightAiFixture } from '@midscene/web/playwright';
 
-export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
+export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture({
+  waitForNetworkIdleTimeout: 2000, // 可选, 交互过程中等待网络空闲的超时时间, 默认值为 2000ms
+  waitForNavigationTimeout: 5000, // 可选, 页面跳转过程中等待页面加载完成的超时时间, 默认值为 5000ms
+}));
 ```
 
 ## 第三步：编写测试用例

packages/core/src/yaml.ts

@@ -45,7 +45,7 @@ export interface MidsceneYamlScriptWebEnv extends MidsceneYamlScriptEnvBase {
   viewportHeight?: number;
   viewportScale?: number;
   waitForNetworkIdle?: {
-    timeout?: number; // ms, 30000 for default, set to 0 to disable
+    timeout?: number;
     continueOnNetworkIdleError?: boolean; // should continue if failed to wait for network idle, true for default
   };
   cookie?: string;

packages/shared/src/constants/index.ts

@@ -16,3 +16,8 @@ export enum NodeType {
 
 export const PLAYGROUND_SERVER_PORT = 5800;
 export const SCRCPY_SERVER_PORT = 5700;
+
+export const DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT = 5000;
+export const DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT = 2000;
+export const DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME = 300;
+export const DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY = 2;

packages/web-integration/package.json

@@ -108,7 +108,7 @@
     "build:watch": "modern build -w -c ./modern.config.ts",
     "test": "vitest --run",
     "test:u": "vitest --run -u",
-    "test:ai": "npm run test",
+    "test:ai": "AI_TEST_TYPE=web npm run test",
     "test:ai:temp": "MIDSCENE_CACHE=true BRIDGE_MODE=true vitest --run tests/ai/bridge/open-new-tab.test.ts",
     "test:ai:bridge": "MIDSCENE_CACHE=true BRIDGE_MODE=true npm run test --inspect tests/ai/bridge/temp.test.ts",
     "test:ai:cache": "MIDSCENE_CACHE=true npm run test",

packages/web-integration/src/common/agent.ts

@@ -21,9 +21,14 @@ import {
   stringifyDumpData,
   writeLogFile,
 } from '@midscene/core/utils';
+import {
+  DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
+  DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT,
+} from '@midscene/shared/constants';
 import { getDebug } from '@midscene/shared/logger';
 import { assert } from '@midscene/shared/utils';
 import { PageTaskExecutor } from '../common/tasks';
+import type { PuppeteerWebPage } from '../puppeteer';
 import type { WebElementInfo } from '../web-element';
 import { buildPlans } from './plan-builder';
 import type { AiTaskCache } from './task-cache';
@@ -52,6 +57,8 @@ export interface PageAgentOpt {
   autoPrintReportMsg?: boolean;
   onTaskStartTip?: OnTaskStartTip;
   aiActionContext?: string;
+  waitForNavigationTimeout?: number;
+  waitForNetworkIdleTimeout?: number;
 }
 
 export class PageAgent<PageType extends WebPage = WebPage> {
@@ -88,6 +95,18 @@ export class PageAgent<PageType extends WebPage = WebPage> {
       opts || {},
     );
 
+    if (
+      this.page.pageType === 'puppeteer' ||
+      this.page.pageType === 'playwright'
+    ) {
+      (this.page as PuppeteerWebPage).waitForNavigationTimeout =
+        this.opts.waitForNavigationTimeout ||
+        DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
+      (this.page as PuppeteerWebPage).waitForNetworkIdleTimeout =
+        this.opts.waitForNetworkIdleTimeout ||
+        DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
+    }
+
     this.onTaskStartTip = this.opts.onTaskStartTip;
     // get the parent browser of the puppeteer page
     // const browser = (this.page as PuppeteerWebPage).browser();

packages/web-integration/src/common/tasks.ts

@@ -111,10 +111,7 @@ export class PageTaskExecutor {
               await sleep(100);
               if ((this.page as PuppeteerWebPage).waitUntilNetworkIdle) {
                 try {
-                  await (this.page as PuppeteerWebPage).waitUntilNetworkIdle({
-                    idleTime: 100,
-                    timeout: 800,
-                  });
+                  await (this.page as PuppeteerWebPage).waitUntilNetworkIdle();
                 } catch (error) {
                   // console.error('waitUntilNetworkIdle error', error);
                 }

packages/web-integration/src/playwright/ai-fixture.ts

@@ -2,11 +2,14 @@ import { randomUUID } from 'node:crypto';
 import type { PageAgent, PageAgentOpt } from '@/common/agent';
 import { PlaywrightAgent } from '@/playwright/index';
 import type { AgentWaitForOpt } from '@midscene/core';
+import { getDebug } from '@midscene/shared/logger';
 import { type TestInfo, type TestType, test } from '@playwright/test';
 import type { Page as OriginPlaywrightPage } from 'playwright';
 
 export type APITestType = Pick<TestType<any, any>, 'step'>;
 
+const debugPage = getDebug('web:playwright:ai-fixture');
+
 const groupAndCaseForTest = (testInfo: TestInfo) => {
   let taskFile: string;
   let taskTitle: string;
@@ -30,8 +33,10 @@ export const midsceneDumpAnnotationId = 'MIDSCENE_DUMP_ANNOTATION';
 
 export const PlaywrightAiFixture = (options?: {
   forceSameTabNavigation?: boolean;
+  waitForNetworkIdleTimeout?: number;
 }) => {
-  const { forceSameTabNavigation = true } = options ?? {};
+  const { forceSameTabNavigation = true, waitForNetworkIdleTimeout = 1000 } =
+    options ?? {};
   const pageAgentMap: Record<string, PageAgent> = {};
   const createOrReuseAgentForPage = (
     page: OriginPlaywrightPage,
@@ -74,11 +79,21 @@ export const PlaywrightAiFixture = (options?: {
       | 'aiWaitFor';
   }) {
     const { page, testInfo, use, aiActionType } = options;
-    const agent = createOrReuseAgentForPage(page, testInfo);
+    const agent = createOrReuseAgentForPage(page, testInfo) as PlaywrightAgent;
+
     await use(async (taskPrompt: string, ...args: any[]) => {
       return new Promise((resolve, reject) => {
         test.step(`ai-${aiActionType} - ${JSON.stringify(taskPrompt)}`, async () => {
-          await waitForNetworkIdle(page);
+          try {
+            debugPage(
+              `waitForNetworkIdle timeout: ${waitForNetworkIdleTimeout}`,
+            );
+            await agent.waitForNetworkIdle(waitForNetworkIdleTimeout);
+          } catch (error) {
+            console.warn(
+              `[Warning:Midscene] Network idle timeout: current timeout is ${waitForNetworkIdleTimeout}ms, custom timeout please check https://midscenejs.com/faq.html#customize-the-network-timeout`,
+            );
+          }
           try {
             type AgentMethod = (
               prompt: string,
@@ -282,13 +297,3 @@ export type PlayWrightAiFixtureType = {
   ) => ReturnType<PageAgent['aiAssert']>;
   aiWaitFor: (assertion: string, opt?: AgentWaitForOpt) => Promise<void>;
 };
-
-async function waitForNetworkIdle(page: OriginPlaywrightPage, timeout = 10000) {
-  try {
-    await page.waitForLoadState('networkidle', { timeout });
-  } catch (error: any) {
-    console.warn(
-      `Network idle timeout exceeded: ${error instanceof Error ? error.message : String(error)}`,
-    );
-  }
-}

packages/web-integration/src/playwright/index.ts

@@ -29,4 +29,8 @@ export class PlaywrightAgent extends PageAgent<PlaywrightWebPage> {
       });
     }
   }
+
+  async waitForNetworkIdle(timeout = 1000) {
+    await this.page.underlyingPage.waitForLoadState('networkidle', { timeout });
+  }
 }

packages/web-integration/src/puppeteer/agent-launcher.ts

@@ -4,6 +4,7 @@ import { assert } from '@midscene/shared/utils';
 
 import { PuppeteerAgent } from '@/puppeteer/index';
 import type { MidsceneYamlScriptWebEnv } from '@midscene/core';
+import { DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT } from '@midscene/shared/constants';
 import puppeteer from 'puppeteer';
 
 export const defaultUA =
@@ -11,7 +12,8 @@ export const defaultUA =
 export const defaultViewportWidth = 1440;
 export const defaultViewportHeight = 768;
 export const defaultViewportScale = process.platform === 'darwin' ? 2 : 1;
-export const defaultWaitForNetworkIdleTimeout = 6 * 1000;
+export const defaultWaitForNetworkIdleTimeout =
+  DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT;
 
 interface FreeFn {
   name: string;

packages/web-integration/src/puppeteer/base-page.ts

@@ -1,5 +1,6 @@
 import type { ElementTreeNode, Point, Size } from '@midscene/core';
 import { sleep } from '@midscene/core/utils';
+import { DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT } from '@midscene/shared/constants';
 import type { ElementInfo } from '@midscene/shared/extractor';
 import { treeToList } from '@midscene/shared/extractor';
 import { getExtraReturnLogic } from '@midscene/shared/fs';
@@ -18,8 +19,10 @@ export class Page<
   PageType extends PuppeteerPage | PlaywrightPage,
 > implements AbstractPage
 {
-  protected underlyingPage: PageType;
+  underlyingPage: PageType;
+  protected waitForNavigationTimeout: number;
   private viewportSize?: Size;
+
   pageType: AgentType;
 
   private async evaluate<R>(
@@ -43,9 +46,17 @@ export class Page<
     return result;
   }
 
-  constructor(underlyingPage: PageType, pageType: AgentType) {
+  constructor(
+    underlyingPage: PageType,
+    pageType: AgentType,
+    opts?: {
+      waitForNavigationTimeout?: number;
+    },
+  ) {
     this.underlyingPage = underlyingPage;
     this.pageType = pageType;
+    this.waitForNavigationTimeout =
+      opts?.waitForNavigationTimeout || DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT;
   }
 
   async evaluateJavaScript<T = any>(script: string): Promise<T> {
@@ -56,13 +67,17 @@ export class Page<
     // issue: https://github.com/puppeteer/puppeteer/issues/3323
     if (this.pageType === 'puppeteer' || this.pageType === 'playwright') {
       debugPage('waitForNavigation begin');
+      debugPage(`waitForNavigation timeout: ${this.waitForNavigationTimeout}`);
       try {
-        const maxWaitTime = 5000; // 5 seconds maximum wait time
         await (this.underlyingPage as PuppeteerPage).waitForSelector('html', {
-          timeout: maxWaitTime,
+          timeout: this.waitForNavigationTimeout,
         });
       } catch (error) {
         // Ignore timeout error, continue execution
+        console.warn(
+          `[Warning:Midscene] Wait for navigation timeout: current timeout is ${this.waitForNavigationTimeout}ms, custom timeout please check https://midscenejs.com/faq.html#customize-the-network-timeout`,
+          error,
+        );
       }
       debugPage('waitForNavigation end');
     }

packages/web-integration/src/puppeteer/page.ts

@@ -1,9 +1,30 @@
+import {
+  DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
+  DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
+  DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
+  DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT,
+} from '@midscene/shared/constants';
 import type { Page as PuppeteerPageType } from 'puppeteer';
 import { Page as BasePage } from './base-page';
 
 export class WebPage extends BasePage<'puppeteer', PuppeteerPageType> {
-  constructor(page: PuppeteerPageType) {
+  waitForNavigationTimeout: number;
+  waitForNetworkIdleTimeout: number;
+
+  constructor(
+    page: PuppeteerPageType,
+    opts?: {
+      waitForNavigationTimeout?: number;
+      waitForNetworkIdleTimeout?: number;
+    },
+  ) {
     super(page, 'puppeteer');
+    const {
+      waitForNavigationTimeout = DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
+      waitForNetworkIdleTimeout = DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT,
+    } = opts ?? {};
+    this.waitForNavigationTimeout = waitForNavigationTimeout;
+    this.waitForNetworkIdleTimeout = waitForNetworkIdleTimeout;
   }
 
   async waitUntilNetworkIdle(options?: {
@@ -12,9 +33,10 @@ export class WebPage extends BasePage<'puppeteer', PuppeteerPageType> {
     timeout?: number;
   }): Promise<void> {
     await this.underlyingPage.waitForNetworkIdle({
-      idleTime: options?.idleTime || 300,
-      concurrency: options?.concurrency || 2,
-      timeout: options?.timeout || 15000,
+      idleTime: options?.idleTime || DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME,
+      concurrency:
+        options?.concurrency || DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY,
+      timeout: options?.timeout || this.waitForNetworkIdleTimeout,
     });
   }
 }

packages/web-integration/tests/ai/web/playwright/fixture.ts

@@ -2,4 +2,8 @@ import type { PlayWrightAiFixtureType } from '@/playwright/ai-fixture';
 import { PlaywrightAiFixture } from '@/playwright/ai-fixture';
 import { test as base } from '@playwright/test';
 
-export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
+export const test = base.extend<PlayWrightAiFixtureType>(
+  PlaywrightAiFixture({
+    waitForNetworkIdleTimeout: 10000,
+  }),
+);