-
-
-
-
-
- Click or drag the{' '}
-
- .web-dump.json
- {' '}
- {/* or{' '}
-
- .actions.json
- {' '} */}
- file into this area.
-
-
- The latest dump file is usually placed in{' '}
-
- ./midscene_run/report
-
-
-
- All data will be processed locally by the browser. No data will be
- sent to the server.
-
-
+
);
diff --git a/packages/web-integration/src/chrome-extension/page.ts b/packages/web-integration/src/chrome-extension/page.ts
index 4802ac57d..0f0f16cbd 100644
--- a/packages/web-integration/src/chrome-extension/page.ts
+++ b/packages/web-integration/src/chrome-extension/page.ts
@@ -31,6 +31,8 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
private activeTabId: number | null = null;
+ private tabIdOfDebuggerAttached: number | null = null;
+
private attachingDebugger: Promise
| null = null;
private destroyed = false;
@@ -70,17 +72,38 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
try {
const currentTabId = await this.getTabId();
- // check if debugger is already attached to the tab
- const targets = await chrome.debugger.getTargets();
- const target = targets.find(
- (target) => target.tabId === currentTabId && target.attached === true,
- );
- if (!target) {
- // await chrome.debugger.detach({ tabId: currentTabId });
- await chrome.debugger.attach({ tabId: currentTabId }, '1.3');
- // Prevent AI logic from being influenced by changes in page width and height due to the debugger banner appearing on attach.
- await sleep(500);
+
+ if (this.tabIdOfDebuggerAttached === currentTabId) {
+ // already attached
+ return;
}
+ if (
+ this.tabIdOfDebuggerAttached &&
+ this.tabIdOfDebuggerAttached !== currentTabId
+ ) {
+ // detach the previous tab
+ console.log(
+ 'detach the previous tab',
+ this.tabIdOfDebuggerAttached,
+ '->',
+ currentTabId,
+ );
+ try {
+ await this.detachDebugger(this.tabIdOfDebuggerAttached);
+ } catch (error) {
+ console.error('Failed to detach debugger', error);
+ }
+ }
+
+ // detach any debugger attached to the tab
+ await chrome.debugger.attach({ tabId: currentTabId }, '1.3');
+ // wait util the debugger banner in Chrome appears
+ await sleep(500);
+ this.tabIdOfDebuggerAttached = currentTabId;
+
+ await this.enableWaterFlowAnimation();
+ } catch (error) {
+ console.error('Failed to attach debugger', error);
} finally {
this.attachingDebugger = null;
}
@@ -89,14 +112,58 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
await this.attachingDebugger;
}
- private async enableWaterFlowAnimation(tabId: number) {
- const script = await injectWaterFlowAnimation();
+ private async showMousePointer(x: number, y: number) {
+ // update mouse pointer while redirecting
+ const pointerScript = `(() => {
+ if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+ window.midsceneWaterFlowAnimation.enable();
+ window.midsceneWaterFlowAnimation.showMousePointer(${x}, ${y});
+ } else {
+ console.log('midsceneWaterFlowAnimation is not defined');
+ }
+ })()`;
- await chrome.debugger.sendCommand({ tabId }, 'Runtime.evaluate', {
- expression: script,
+ await this.sendCommandToDebugger('Runtime.evaluate', {
+ expression: `${pointerScript}`,
});
}
+ private async hideMousePointer() {
+ await this.sendCommandToDebugger('Runtime.evaluate', {
+ expression: `(() => {
+ if(typeof window.midsceneWaterFlowAnimation !== 'undefined') {
+ window.midsceneWaterFlowAnimation.hideMousePointer();
+ }
+ })()`,
+ });
+ }
+
+ private async detachDebugger(tabId?: number) {
+ const tabIdToDetach = tabId || this.tabIdOfDebuggerAttached;
+ if (!tabIdToDetach) {
+ console.warn('No tab id to detach');
+ return;
+ }
+
+ await this.disableWaterFlowAnimation(tabIdToDetach);
+ await sleep(200);
+ await chrome.debugger.detach({ tabId: tabIdToDetach });
+
+ this.tabIdOfDebuggerAttached = null;
+ }
+
+ private async enableWaterFlowAnimation() {
+ const script = await injectWaterFlowAnimation();
+ // we will call this function in sendCommandToDebugger, so we have to use the chrome.debugger.sendCommand
+ await chrome.debugger.sendCommand(
+ { tabId: this.tabIdOfDebuggerAttached! },
+ 'Runtime.evaluate',
+ {
+ expression: script,
+ },
+ );
+ }
+
private async disableWaterFlowAnimation(tabId: number) {
const script = await injectStopWaterFlowAnimation();
@@ -105,33 +172,18 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
});
}
- private async detachDebugger() {
- // check if debugger is already attached to the tab
- const targets = await chrome.debugger.getTargets();
- const attendTabs = targets.filter(
- (target) =>
- target.attached === true &&
- !target.url.startsWith('chrome-extension://'),
- );
- if (attendTabs.length > 0) {
- for (const tab of attendTabs) {
- if (tab.tabId) {
- await this.disableWaterFlowAnimation(tab.tabId);
- chrome.debugger.detach({ tabId: tab.tabId });
- }
- }
- }
- }
-
private async sendCommandToDebugger(
command: string,
params: RequestType,
): Promise {
await this.attachDebugger();
- const tabId = await this.getTabId();
- this.enableWaterFlowAnimation(tabId);
+
+ assert(this.tabIdOfDebuggerAttached, 'Debugger is not attached');
+
+ // wo don't have to await it
+ this.enableWaterFlowAnimation();
return (await chrome.debugger.sendCommand(
- { tabId },
+ { tabId: this.tabIdOfDebuggerAttached! },
command,
params as any,
)) as ResponseType;
@@ -204,6 +256,7 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
}
async getElementInfos() {
+ await this.hideMousePointer();
const content = await this.getPageContentByCDP();
if (content?.size) {
this.viewportSize = content.size;
@@ -220,6 +273,7 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
async screenshotBase64() {
// screenshot by cdp
+ await this.hideMousePointer();
const base64 = await this.sendCommandToDebugger('Page.captureScreenshot', {
format: 'jpeg',
quality: 70,
@@ -333,6 +387,7 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
mouse = {
click: async (x: number, y: number) => {
+ await this.showMousePointer(x, y);
await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
type: 'mousePressed',
x,
@@ -354,15 +409,19 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
startX?: number,
startY?: number,
) => {
+ const finalX = startX || 50;
+ const finalY = startY || 50;
+ await this.showMousePointer(finalX, finalY);
await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
type: 'mouseWheel',
- x: startX || 10,
- y: startY || 10,
+ x: finalX,
+ y: finalY,
deltaX,
deltaY,
});
},
move: async (x: number, y: number) => {
+ await this.showMousePointer(x, y);
await this.sendCommandToDebugger('Input.dispatchMouseEvent', {
type: 'mouseMoved',
x,
diff --git a/packages/web-integration/src/puppeteer/base-page.ts b/packages/web-integration/src/puppeteer/base-page.ts
index 1f66aef29..27b7b9b53 100644
--- a/packages/web-integration/src/puppeteer/base-page.ts
+++ b/packages/web-integration/src/puppeteer/base-page.ts
@@ -133,68 +133,61 @@ export class Page<
await this.keyboard.press('Backspace');
}
- async scrollUntilTop(startingPoint?: Point): Promise {
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
+ private async moveToPoint(point?: Point): Promise {
+ if (point) {
+ await this.mouse.move(point.left, point.top);
+ } else {
+ const size = await this.size();
+ await this.mouse.move(size.width / 2, size.height / 2);
}
+ }
+
+ async scrollUntilTop(startingPoint?: Point): Promise {
+ await this.moveToPoint(startingPoint);
return this.mouse.wheel(0, -9999999);
}
async scrollUntilBottom(startingPoint?: Point): Promise {
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
- }
+ await this.moveToPoint(startingPoint);
return this.mouse.wheel(0, 9999999);
}
async scrollUntilLeft(startingPoint?: Point): Promise {
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
- }
+ await this.moveToPoint(startingPoint);
return this.mouse.wheel(-9999999, 0);
}
async scrollUntilRight(startingPoint?: Point): Promise {
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
- }
+ await this.moveToPoint(startingPoint);
return this.mouse.wheel(9999999, 0);
}
async scrollUp(distance?: number, startingPoint?: Point): Promise {
const innerHeight = await this.evaluate(() => window.innerHeight);
const scrollDistance = distance || innerHeight * 0.7;
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
- }
- await this.mouse.wheel(0, -scrollDistance);
+ await this.moveToPoint(startingPoint);
+ return this.mouse.wheel(0, -scrollDistance);
}
async scrollDown(distance?: number, startingPoint?: Point): Promise {
const innerHeight = await this.evaluate(() => window.innerHeight);
const scrollDistance = distance || innerHeight * 0.7;
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
- }
- await this.mouse.wheel(0, scrollDistance);
+ await this.moveToPoint(startingPoint);
+ return this.mouse.wheel(0, scrollDistance);
}
async scrollLeft(distance?: number, startingPoint?: Point): Promise {
const innerWidth = await this.evaluate(() => window.innerWidth);
const scrollDistance = distance || innerWidth * 0.7;
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
- }
- await this.mouse.wheel(-scrollDistance, 0);
+ await this.moveToPoint(startingPoint);
+ return this.mouse.wheel(-scrollDistance, 0);
}
async scrollRight(distance?: number, startingPoint?: Point): Promise {
const innerWidth = await this.evaluate(() => window.innerWidth);
const scrollDistance = distance || innerWidth * 0.7;
- if (startingPoint) {
- await this.mouse.move(startingPoint.left, startingPoint.top);
- }
- await this.mouse.wheel(scrollDistance, 0);
+ await this.moveToPoint(startingPoint);
+ return this.mouse.wheel(scrollDistance, 0);
}
async destroy(): Promise {