feat: show pointer position in chrome extension (#286)

---------

Co-authored-by: zhouxiao.shaw <zhouxiao.shaw@bytedance.com>

apps/site/docs/en/model-provider.md

@@ -86,6 +86,8 @@ There are some extra configs when using Azure OpenAI Service.
 
 ### Use ADT token provider
 
+This mode cannot be used in Chrome extension.
+
 ```bash
 # this is always true when using Azure OpenAI Service
 export MIDSCENE_USE_AZURE_OPENAI=1

apps/site/docs/en/quick-experience.mdx

@@ -49,5 +49,5 @@ It's mainly due to conflicts with other extensions injecting `<iframe />` or `<s
 To find the suspicious plugins:
  
 1. Open the Devtools of the page, find the `<script>` or `<iframe>` with a url like `chrome-extension://{ID-of-the-suspicious-plugin}/...`.
-2. Copy the ID from the url, open chrome://extensions/, find the plugin with the same ID, disable it.
+2. Copy the ID from the url, open `chrome://extensions/` , use cmd+f to find the plugin with the same ID, disable it.
 3. Refresh the page, try again.

apps/site/docs/zh/model-provider.md

@@ -73,7 +73,9 @@ import 'dotenv/config';
 
 ## 使用 Azure OpenAI 服务时的配置
 
-使用 ADT token provider
+### 使用 ADT token provider
+
+此种模式无法运行在浏览器插件中。
 
 ```bash
 # 使用 Azure OpenAI 服务时，配置为 1
@@ -85,7 +87,7 @@ export AZURE_OPENAI_API_VERSION="2024-05-01-preview"
 export AZURE_OPENAI_DEPLOYMENT="gpt-4o"
 ```
 
-使用 keyless 模式
+### 使用 keyless 模式
 
 ```bash
 export MIDSCENE_USE_AZURE_OPENAI=1

apps/site/docs/zh/quick-experience.mdx

@@ -47,5 +47,5 @@ OPENAI_API_KEY="sk-replace-by-your-own"
 找到可疑插件：
 
 1. 打开页面的调试器，找到被其他插件注入的 `<iframe />` 或 `<script />`，一般 URL 是 `chrome-extension://{这串就是ID}/...` 格式，复制其 ID。
-2. 打开 chrome://extensions/ ，找到相同 ID 的插件，禁用它。
+2. 打开 `chrome://extensions/` ，用 cmd+f 找到相同 ID 的插件，禁用它。
 3. 刷新页面，再次尝试。

packages/midscene/src/ai-model/prompt/llm-planning.ts

@@ -122,7 +122,7 @@ The JSON format is as follows:
 {{
   "actions": [
     {{
-      "thought": "Reasons for generating this task, and why this task is feasible on this page",
+      "thought": "Reasons for generating this task, and why this task is feasible on this page.", // Use the same language as the user's instruction.
       "type": "Tap",
       "param": null,
       "locate": {sample} | null,
@@ -130,8 +130,8 @@ The JSON format is as follows:
     // ... more actions
   ],
   "taskWillBeAccomplished": boolean,
-  "furtherPlan": {{ "whatHaveDone": string, "whatToDoNext": string }} | null,
-  "error"?: string
+  "furtherPlan": {{ "whatHaveDone": string, "whatToDoNext": string }} | null, // Use the same language as the user's instruction.
+  "error"?: string // Use the same language as the user's instruction.
 }}
 Here is an example of how to decompose a task:
 

packages/midscene/src/insight/utils.ts

@@ -1,14 +1,15 @@
 import assert from 'node:assert';
-import { randomUUID } from 'node:crypto';
 /* eslint-disable @typescript-eslint/ban-types */
 import { existsSync } from 'node:fs';
 import { join } from 'node:path';
-import { MIDSCENE_MODEL_NAME, getAIConfig } from '@/env';
+import {
+  MIDSCENE_MODEL_NAME,
+  MIDSCENE_USE_VLM_UI_TARS,
+  getAIConfig,
+} from '@/env';
 import type {
-  BaseElement,
   DumpMeta,
   DumpSubscriber,
-  ElementById,
   InsightDump,
   PartialInsightDumpFromSDK,
 } from '@/types';
@@ -40,6 +41,9 @@ export function writeInsightDump(
     sdkVersion: getVersion(),
     logTime: Date.now(),
     model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',
+    model_description: getAIConfig(MIDSCENE_USE_VLM_UI_TARS)
+      ? 'vlm-ui-tars enabled'
+      : '',
   };
   const finalData: InsightDump = {
     logId: id,

packages/midscene/src/types.ts

@@ -140,6 +140,7 @@ export interface DumpMeta {
   sdkVersion: string;
   logTime: number;
   model_name: string;
+  model_description?: string;
 }
 
 export interface ReportDumpWithAttributes {

packages/visualizer/src/component/logo.tsx

@@ -1,13 +1,13 @@
 import './logo.less';
 
+export const LogoUrl =
+  'https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png';
+
 const Logo = ({ withGithubStar = false }: { withGithubStar?: boolean }) => {
   if (withGithubStar) {
     return (
       <div className="logo logo-with-star-wrapper">
-        <img
-          alt="Midscene_logo"
-          src="https://lf3-static.bytednsdoc.com/obj/eden-cn/vhaeh7vhabf/Midscene.png"
-        />
+        <img alt="Midscene_logo" src={LogoUrl} />
         <a
           href="https://github.com/web-infra-dev/midscene"
           target="_blank"

packages/visualizer/src/component/playground-component.tsx

@@ -33,7 +33,7 @@ import type { WebUIContext } from '@midscene/web/utils';
 import type { MenuProps } from 'antd';
 import { Dropdown, Space } from 'antd';
 import { EnvConfig } from './env-config';
-import { type HistoryItem, useChromeTabInfo, useEnvConfig } from './store';
+import { type HistoryItem, useEnvConfig } from './store';
 
 import {
   ChromeExtensionProxyPage,
@@ -339,16 +339,15 @@ export function Playground({
         }
       }
     } catch (e: any) {
+      const errorMessage = e?.message || '';
       console.error(e);
-      if (typeof e === 'string') {
-        if (e.includes('of different extension')) {
-          result.error =
-            'Conflicting extension detected. Please disable the suspicious plugins and refresh the page. Guide: https://midscenejs.com/quick-experience.html#faq';
-        } else {
-          result.error = e;
-        }
-      } else if (!e.message?.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)) {
-        result.error = e.message;
+      if (errorMessage.includes('of different extension')) {
+        result.error =
+          'Conflicting extension detected. Please disable the suspicious plugins and refresh the page. Guide: https://midscenejs.com/quick-experience.html#faq';
+      } else if (
+        !errorMessage?.includes(ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED)
+      ) {
+        result.error = errorMessage;
       } else {
         result.error = 'Unknown error';
       }

packages/visualizer/src/component/replay-scripts.tsx

@@ -119,6 +119,7 @@ export interface ReplayScriptsInfo {
   height: number;
   sdkVersion?: string;
   modelName?: string;
+  modelDescription?: string;
 }
 
 export const allScriptsFromDump = (
@@ -129,6 +130,7 @@ export const allScriptsFromDump = (
   let height = 0;
   let sdkVersion = '';
   let modelName = '';
+  let modelDescription = '';
 
   dump.executions.forEach((execution) => {
     execution.tasks.forEach((task) => {
@@ -145,6 +147,10 @@ export const allScriptsFromDump = (
       if (insightTask.log?.dump?.model_name) {
         modelName = insightTask.log.dump.model_name;
       }
+
+      if (insightTask.log?.dump?.model_description) {
+        modelDescription = insightTask.log.dump.model_description;
+      }
     });
   });
 
@@ -176,6 +182,7 @@ export const allScriptsFromDump = (
     height,
     sdkVersion,
     modelName,
+    modelDescription,
   };
 };
 

packages/visualizer/src/component/sidebar.tsx

@@ -63,6 +63,7 @@ const SideItem = (props: {
 const Sidebar = (): JSX.Element => {
   const sdkVersion = useExecutionDump((store) => store.sdkVersion);
   const modelName = useExecutionDump((store) => store.modelName);
+  const modelDescription = useExecutionDump((store) => store.modelDescription);
   const groupedDump = useExecutionDump((store) => store.dump);
   const setActiveTask = useExecutionDump((store) => store.setActiveTask);
   const activeTask = useExecutionDump((store) => store.activeTask);
@@ -110,8 +111,9 @@ const Sidebar = (): JSX.Element => {
     };
   }, [currentSelectedIndex, allTasks, setActiveTask]);
 
+  const modelDescriptionText = modelDescription ? `, ${modelDescription}` : '';
   const envInfo = sdkVersion
-    ? `v${sdkVersion}, ${modelName || 'default model'}`
+    ? `v${sdkVersion}, ${modelName || 'default model'}${modelDescriptionText}`
     : '';
 
   const sideList = groupedDump ? (

packages/visualizer/src/component/store.tsx

@@ -91,6 +91,9 @@ export const useChromeTabInfo = create<{
   };
 
   Promise.resolve().then(async () => {
+    if (typeof window.chrome === 'undefined') {
+      return;
+    }
     const tab = await activeTab();
     const windowId = await currentWindowId();
     set({
@@ -206,6 +209,7 @@ export const useExecutionDump = create<{
   allExecutionAnimation: AnimationScript[] | null;
   sdkVersion: string | null;
   modelName: string | null;
+  modelDescription: string | null;
   insightWidth: number | null;
   insightHeight: number | null;
   activeExecution: ExecutionDump | null;
@@ -228,6 +232,7 @@ export const useExecutionDump = create<{
     allExecutionAnimation: null,
     sdkVersion: null,
     modelName: null,
+    modelDescription: null,
     insightWidth: null,
     insightHeight: null,
     activeTask: null,
@@ -297,6 +302,7 @@ export const useExecutionDump = create<{
           width,
           height,
           modelName,
+          modelDescription,
           sdkVersion,
         } = allScriptsInfo;
 
@@ -307,6 +313,7 @@ export const useExecutionDump = create<{
           insightWidth: width,
           insightHeight: height,
           modelName,
+          modelDescription,
           sdkVersion,
         });
       }

packages/visualizer/src/extension/scripts/stop-water-flow.ts

@@ -1,4 +1,3 @@
-const styleElements = document.querySelectorAll('[id="water-flow-animation"]');
-styleElements.forEach((element) => {
-  document.head.removeChild(element);
-});
+if (typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+  window.midsceneWaterFlowAnimation.disable();
+}

packages/visualizer/src/extension/scripts/water-flow.ts

@@ -1,11 +1,107 @@
-const waterFlowAnimation = {
+const midsceneWaterFlowAnimation = {
   styleElement: null as null | HTMLStyleElement,
 
+  mousePointerAttribute: 'data-water-flow-pointer',
+
+  lastCallTime: 0,
+
+  cleanupTimeout: null as null | number,
+
+  // call to reset the self cleaning timer
+  registerSelfCleaning() {
+    // clean up all the indicators if there is no call for 30 seconds
+    this.lastCallTime = Date.now();
+    const cleaningTimeout = 30000;
+
+    if (this.cleanupTimeout) {
+      clearTimeout(this.cleanupTimeout);
+    }
+
+    this.cleanupTimeout = window.setTimeout(() => {
+      const now = Date.now();
+      if (now - this.lastCallTime >= cleaningTimeout) {
+        this.disable();
+      }
+    }, cleaningTimeout);
+  },
+
+  showMousePointer(x: number, y: number) {
+    this.enable(); // show water flow animation
+    this.registerSelfCleaning();
+    const existingPointer = document.querySelector(
+      `div[${this.mousePointerAttribute}]`,
+    ) as HTMLDivElement | null;
+
+    // Clear any existing timeouts to prevent race conditions
+    if (existingPointer) {
+      const timeoutId = Number(existingPointer.getAttribute('data-timeout-id'));
+      if (timeoutId) clearTimeout(timeoutId);
+      const removeTimeoutId = Number(
+        existingPointer.getAttribute('data-remove-timeout-id'),
+      );
+      if (removeTimeoutId) clearTimeout(removeTimeoutId);
+    }
+
+    const size = 30;
+    const pointer =
+      existingPointer ||
+      (() => {
+        const p = document.createElement('div');
+        p.setAttribute(this.mousePointerAttribute, 'true');
+        p.style.position = 'fixed';
+        p.style.width = `${size}px`;
+        p.style.height = `${size}px`;
+        p.style.borderRadius = '50%';
+        p.style.backgroundColor = 'rgba(0, 0, 255, 0.3)';
+        p.style.border = '1px solid rgba(0, 0, 255, 0.3)';
+        p.style.zIndex = '99999';
+        p.style.transition = 'all 1s ease-in';
+        p.style.pointerEvents = 'none'; // Make pointer not clickable
+        // Start from offset position if new pointer
+        p.style.left = `${x - size / 2}px`;
+        p.style.top = `${y - size / 2}px`;
+        document.body.appendChild(p);
+        return p;
+      })();
+
+    requestAnimationFrame(() => {
+      pointer.style.left = `${x - size / 2}px`;
+      pointer.style.top = `${y - size / 2}px`;
+      pointer.style.opacity = '1';
+    });
+
+    // Set new timeouts
+    const fadeTimeoutId = setTimeout(() => {
+      pointer.style.opacity = '0';
+      const removeTimeoutId = setTimeout(() => {
+        if (pointer.parentNode) {
+          document.body.removeChild(pointer);
+        }
+      }, 500);
+      pointer.setAttribute('data-remove-timeout-id', String(removeTimeoutId));
+    }, 3000);
+    pointer.setAttribute('data-timeout-id', String(fadeTimeoutId));
+  },
+
+  hideMousePointer() {
+    this.registerSelfCleaning();
+    const pointer = document.querySelector(
+      `div[${this.mousePointerAttribute}]`,
+    ) as HTMLDivElement | null;
+    if (pointer) {
+      document.body.removeChild(pointer);
+    }
+  },
+
   enable() {
-    if (this.styleElement) return;
-    // Check if water flow animation style already exists
-    const existingStyle = document.querySelector('#water-flow-animation');
-    if (existingStyle) return;
+    this.registerSelfCleaning();
+    if (this.styleElement) {
+      // double check if styleElement is still in the dom tree
+      if (document.head.contains(this.styleElement)) {
+        return;
+      }
+      this.styleElement = null;
+    }
 
     this.styleElement = document.createElement('style');
     this.styleElement.id = 'water-flow-animation';
@@ -66,21 +162,33 @@ const waterFlowAnimation = {
   },
 
   disable() {
-    const styleElements = document.querySelectorAll(
-      '[id="water-flow-animation"]',
-    );
+    if (this.cleanupTimeout) {
+      clearTimeout(this.cleanupTimeout);
+      this.cleanupTimeout = null;
+    }
+
+    const styleElements = document.querySelectorAll('#water-flow-animation');
     styleElements.forEach((element) => {
       document.head.removeChild(element);
     });
     this.styleElement = null;
+
+    // remove all mouse pointers
+    const mousePointers = document.querySelectorAll(
+      `div[${this.mousePointerAttribute}]`,
+    );
+    mousePointers.forEach((element) => {
+      document.body.removeChild(element);
+    });
   },
 };
 
 export {};
 declare global {
   interface Window {
-    waterFlowAnimation: typeof waterFlowAnimation;
+    midsceneWaterFlowAnimation: typeof midsceneWaterFlowAnimation;
   }
 }
-(window as any).waterFlowAnimation = waterFlowAnimation;
-(window as any).waterFlowAnimation.enable();
+(window as any).midsceneWaterFlowAnimation =
+  (window as any).midsceneWaterFlowAnimation || midsceneWaterFlowAnimation;
+(window as any).midsceneWaterFlowAnimation.enable();

packages/visualizer/src/index.tsx

@@ -24,7 +24,7 @@ import logoImg from './component/assets/logo-plain.png';
 import { globalThemeConfig } from './component/color';
 import DetailPanel from './component/detail-panel';
 import GlobalHoverPreview from './component/global-hover-preview';
-import Logo from './component/logo';
+import Logo, { LogoUrl } from './component/logo';
 import { iconForStatus, timeCostStrElement } from './component/misc';
 import Player from './component/player';
 import Timeline from './component/timeline';
@@ -134,38 +134,55 @@ export function Visualizer(props: {
       </div>
     );
   } else if (!executionDump) {
+    // mainContent = (
+    //   <div className="main-right uploader-wrapper">
+    //     <Dragger className="uploader" {...uploadProps}>
+    //       <p className="ant-upload-drag-icon">
+    //         <img
+    //           alt="Midscene_logo"
+    //           style={{ width: 80, margin: 'auto' }}
+    //           src={logoImg}
+    //         />
+    //       </p>
+    //       <p className="ant-upload-text">
+    //         Click or drag the{' '}
+    //         <b>
+    //           <i>.web-dump.json</i>
+    //         </b>{' '}
+    //         {/* or{' '}
+    //         <b>
+    //           <i>.actions.json</i>
+    //         </b>{' '} */}
+    //         file into this area.
+    //       </p>
+    //       <p className="ant-upload-text">
+    //         The latest dump file is usually placed in{' '}
+    //         <b>
+    //           <i>./midscene_run/report</i>
+    //         </b>
+    //       </p>
+    //       <p className="ant-upload-text">
+    //         All data will be processed locally by the browser. No data will be
+    //         sent to the server.
+    //       </p>
+    //     </Dragger>
+    //   </div>
+    // );
+
     mainContent = (
-      <div className="main-right uploader-wrapper">
-        <Dragger className="uploader" {...uploadProps}>
-          <p className="ant-upload-drag-icon">
-            <img
-              alt="Midscene_logo"
-              style={{ width: 80, margin: 'auto' }}
-              src={logoImg}
-            />
-          </p>
-          <p className="ant-upload-text">
-            Click or drag the{' '}
-            <b>
-              <i>.web-dump.json</i>
-            </b>{' '}
-            {/* or{' '}
-            <b>
-              <i>.actions.json</i>
-            </b>{' '} */}
-            file into this area.
-          </p>
-          <p className="ant-upload-text">
-            The latest dump file is usually placed in{' '}
-            <b>
-              <i>./midscene_run/report</i>
-            </b>
-          </p>
-          <p className="ant-upload-text">
-            All data will be processed locally by the browser. No data will be
-            sent to the server.
-          </p>
-        </Dragger>
+      <div className="main-right">
+        <div
+          className="center-content"
+          style={{
+            width: '100%',
+            height: '100%',
+            display: 'flex',
+            justifyContent: 'center',
+            alignItems: 'center',
+          }}
+        >
+          <Empty image={LogoUrl} description="Loading report content..." />
+        </div>
       </div>
     );
 

packages/web-integration/src/chrome-extension/page.ts

@@ -31,6 +31,8 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
 
   private activeTabId: number | null = null;
 
+  private tabIdOfDebuggerAttached: number | null = null;
+
   private attachingDebugger: Promise<void> | null = null;
 
   private destroyed = false;
@@ -70,17 +72,38 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
 
       try {
         const currentTabId = await this.getTabId();
-        // check if debugger is already attached to the tab
-        const targets = await chrome.debugger.getTargets();
-        const target = targets.find(
-          (target) => target.tabId === currentTabId && target.attached === true,
-        );
-        if (!target) {
-          // await chrome.debugger.detach({ tabId: currentTabId });
-          await chrome.debugger.attach({ tabId: currentTabId }, '1.3');
-          // Prevent AI logic from being influenced by changes in page width and height due to the debugger banner appearing on attach.
-          await sleep(500);
+
+        if (this.tabIdOfDebuggerAttached === currentTabId) {
+          // already attached
+          return;
         }
+        if (
+          this.tabIdOfDebuggerAttached &&
+          this.tabIdOfDebuggerAttached !== currentTabId
+        ) {
+          // detach the previous tab
+          console.log(
+            'detach the previous tab',
+            this.tabIdOfDebuggerAttached,
+            '->',
+            currentTabId,
+          );
+          try {
+            await this.detachDebugger(this.tabIdOfDebuggerAttached);
+          } catch (error) {
+            console.error('Failed to detach debugger', error);
+          }
+        }
+
+        // detach any debugger attached to the tab
+        await chrome.debugger.attach({ tabId: currentTabId }, '1.3');
+        // wait util the debugger banner in Chrome appears
+        await sleep(500);
+        this.tabIdOfDebuggerAttached = currentTabId;
+
+        await this.enableWaterFlowAnimation();
+      } catch (error) {
+        console.error('Failed to attach debugger', error);
       } finally {
         this.attachingDebugger = null;
       }
@@ -89,14 +112,58 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
     await this.attachingDebugger;
   }
 
-  private async enableWaterFlowAnimation(tabId: number) {
-    const script = await injectWaterFlowAnimation();
+  private async showMousePointer(x: number, y: number) {
+    // update mouse pointer while redirecting
+    const pointerScript = `(() => {
+      if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+        window.midsceneWaterFlowAnimation.enable();
+        window.midsceneWaterFlowAnimation.showMousePointer(${x}, ${y});
+      } else {
+        console.log('midsceneWaterFlowAnimation is not defined');
+      }
+    })()`;
 
-    await chrome.debugger.sendCommand({ tabId }, 'Runtime.evaluate', {
-      expression: script,
+    await this.sendCommandToDebugger('Runtime.evaluate', {
+      expression: `${pointerScript}`,
     });
   }
 
+  private async hideMousePointer() {
+    await this.sendCommandToDebugger('Runtime.evaluate', {
+      expression: `(() => {
+        if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+          window.midsceneWaterFlowAnimation.hideMousePointer();
+        }
+      })()`,
+    });
+  }
+
+  private async detachDebugger(tabId?: number) {
+    const tabIdToDetach = tabId || this.tabIdOfDebuggerAttached;
+    if (!tabIdToDetach) {
+      console.warn('No tab id to detach');
+      return;
+    }
+
+    await this.disableWaterFlowAnimation(tabIdToDetach);
+    await sleep(200);
+    await chrome.debugger.detach({ tabId: tabIdToDetach });
+
+    this.tabIdOfDebuggerAttached = null;
+  }
+
+  private async enableWaterFlowAnimation() {
+    const script = await injectWaterFlowAnimation();
+    // we will call this function in sendCommandToDebugger, so we have to use the chrome.debugger.sendCommand
+    await chrome.debugger.sendCommand(
+      { tabId: this.tabIdOfDebuggerAttached! },
+      'Runtime.evaluate',
+      {
+        expression: script,
+      },
+    );
+  }
+
   private async disableWaterFlowAnimation(tabId: number) {
     const script = await injectStopWaterFlowAnimation();
 
@@ -105,33 +172,18 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
     });
   }
 
-  private async detachDebugger() {
-    // check if debugger is already attached to the tab
-    const targets = await chrome.debugger.getTargets();
-    const attendTabs = targets.filter(
-      (target) =>
-        target.attached === true &&
-        !target.url.startsWith('chrome-extension://'),
-    );
-    if (attendTabs.length > 0) {
-      for (const tab of attendTabs) {
-        if (tab.tabId) {
-          await this.disableWaterFlowAnimation(tab.tabId);
-          chrome.debugger.detach({ tabId: tab.tabId });
-        }
-      }
-    }
-  }
-
   private async sendCommandToDebugger<ResponseType = any, RequestType = any>(
     command: string,
     params: RequestType,
   ): Promise<ResponseType> {
     await this.attachDebugger();
-    const tabId = await this.getTabId();
-    this.enableWaterFlowAnimation(tabId);
+
+    assert(this.tabIdOfDebuggerAttached, 'Debugger is not attached');
+
+    // wo don't have to await it
+    this.enableWaterFlowAnimation();
     return (await chrome.debugger.sendCommand(
-      { tabId },
+      { tabId: this.tabIdOfDebuggerAttached! },
       command,
       params as any,
     )) as ResponseType;
@@ -204,6 +256,7 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
   }
 
   async getElementInfos() {
+    await this.hideMousePointer();
     const content = await this.getPageContentByCDP();
     if (content?.size) {
       this.viewportSize = content.size;
@@ -220,6 +273,7 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
 
   async screenshotBase64() {
     // screenshot by cdp
+    await this.hideMousePointer();
     const base64 = await this.sendCommandToDebugger('Page.captureScreenshot', {
       format: 'jpeg',
       quality: 70,
@@ -333,6 +387,7 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
 
   mouse = {
     click: async (x: number, y: number) => {
+      await this.showMousePointer(x, y);
       await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
         type: 'mousePressed',
         x,
@@ -354,15 +409,19 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
       startX?: number,
       startY?: number,
     ) => {
+      const finalX = startX || 50;
+      const finalY = startY || 50;
+      await this.showMousePointer(finalX, finalY);
       await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
         type: 'mouseWheel',
-        x: startX || 10,
-        y: startY || 10,
+        x: finalX,
+        y: finalY,
         deltaX,
         deltaY,
       });
     },
     move: async (x: number, y: number) => {
+      await this.showMousePointer(x, y);
       await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
         type: 'mouseMoved',
         x,

packages/web-integration/src/puppeteer/base-page.ts

@@ -133,68 +133,61 @@ export class Page<
     await this.keyboard.press('Backspace');
   }
 
-  async scrollUntilTop(startingPoint?: Point): Promise<void> {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
+  private async moveToPoint(point?: Point): Promise<void> {
+    if (point) {
+      await this.mouse.move(point.left, point.top);
+    } else {
+      const size = await this.size();
+      await this.mouse.move(size.width / 2, size.height / 2);
     }
+  }
+
+  async scrollUntilTop(startingPoint?: Point): Promise<void> {
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(0, -9999999);
   }
 
   async scrollUntilBottom(startingPoint?: Point): Promise<void> {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(0, 9999999);
   }
 
   async scrollUntilLeft(startingPoint?: Point): Promise<void> {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(-9999999, 0);
   }
 
   async scrollUntilRight(startingPoint?: Point): Promise<void> {
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
+    await this.moveToPoint(startingPoint);
     return this.mouse.wheel(9999999, 0);
   }
 
   async scrollUp(distance?: number, startingPoint?: Point): Promise<void> {
     const innerHeight = await this.evaluate(() => window.innerHeight);
     const scrollDistance = distance || innerHeight * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(0, -scrollDistance);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(0, -scrollDistance);
   }
 
   async scrollDown(distance?: number, startingPoint?: Point): Promise<void> {
     const innerHeight = await this.evaluate(() => window.innerHeight);
     const scrollDistance = distance || innerHeight * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(0, scrollDistance);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(0, scrollDistance);
   }
 
   async scrollLeft(distance?: number, startingPoint?: Point): Promise<void> {
     const innerWidth = await this.evaluate(() => window.innerWidth);
     const scrollDistance = distance || innerWidth * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(-scrollDistance, 0);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(-scrollDistance, 0);
   }
 
   async scrollRight(distance?: number, startingPoint?: Point): Promise<void> {
     const innerWidth = await this.evaluate(() => window.innerWidth);
     const scrollDistance = distance || innerWidth * 0.7;
-    if (startingPoint) {
-      await this.mouse.move(startingPoint.left, startingPoint.top);
-    }
-    await this.mouse.wheel(scrollDistance, 0);
+    await this.moveToPoint(startingPoint);
+    return this.mouse.wheel(scrollDistance, 0);
   }
 
   async destroy(): Promise<void> {