implement repeat function for scrolling until actions (#713)

* feat(android): implement repeat function for scrolling until actions

* fix(shared): fix potential error in getAIConfig by ensuring trim is called correctly

* feat(android): update scrolling behavior with adjustable duration and added sleep

* feat(android): refine scrolling durations with new constants for fast and normal scroll
This commit is contained in:
Leyang 2025-05-14 18:28:10 +08:00 committed by GitHub
parent 388bbb6a34
commit b9ff80a0db
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 43 additions and 11 deletions

View File

@ -3,14 +3,21 @@ import fs from 'node:fs';
import path from 'node:path'; import path from 'node:path';
import { type Point, type Size, getAIConfig } from '@midscene/core'; import { type Point, type Size, getAIConfig } from '@midscene/core';
import type { PageType } from '@midscene/core'; import type { PageType } from '@midscene/core';
import { getTmpFile } from '@midscene/core/utils'; import { getTmpFile, sleep } from '@midscene/core/utils';
import { MIDSCENE_ADB_PATH } from '@midscene/shared/env'; import { MIDSCENE_ADB_PATH } from '@midscene/shared/env';
import type { ElementInfo } from '@midscene/shared/extractor'; import type { ElementInfo } from '@midscene/shared/extractor';
import { isValidPNGImageBuffer, resizeImg } from '@midscene/shared/img'; import { isValidPNGImageBuffer, resizeImg } from '@midscene/shared/img';
import { getDebug } from '@midscene/shared/logger'; import { getDebug } from '@midscene/shared/logger';
import { repeat } from '@midscene/shared/utils';
import type { AndroidDevicePage } from '@midscene/web'; import type { AndroidDevicePage } from '@midscene/web';
import { ADB } from 'appium-adb'; import { ADB } from 'appium-adb';
const androidScreenshotPath = '/data/local/tmp/midscene_screenshot.png'; const androidScreenshotPath = '/data/local/tmp/midscene_screenshot.png';
// only for Android, because it's impossible to scroll to the bottom, so we need to set a default scroll times
const defaultScrollUntilTimes = 10;
const defaultFastScrollDuration = 100;
const defaultNormalScrollDuration = 1000;
export const debugPage = getDebug('android:device'); export const debugPage = getDebug('android:device');
export class AndroidDevice implements AndroidDevicePage { export class AndroidDevice implements AndroidDevicePage {
@ -397,7 +404,11 @@ ${Object.keys(size)
await this.mouseDrag(start, end); await this.mouseDrag(start, end);
return; return;
} }
await this.mouseWheel(0, 9999999, 100);
await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(0, 9999999, defaultFastScrollDuration),
);
await sleep(1000);
} }
async scrollUntilBottom(startPoint?: Point): Promise<void> { async scrollUntilBottom(startPoint?: Point): Promise<void> {
@ -408,7 +419,11 @@ ${Object.keys(size)
await this.mouseDrag(start, end); await this.mouseDrag(start, end);
return; return;
} }
await this.mouseWheel(0, -9999999, 100);
await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(0, -9999999, defaultFastScrollDuration),
);
await sleep(1000);
} }
async scrollUntilLeft(startPoint?: Point): Promise<void> { async scrollUntilLeft(startPoint?: Point): Promise<void> {
@ -418,7 +433,11 @@ ${Object.keys(size)
await this.mouseDrag(start, end); await this.mouseDrag(start, end);
return; return;
} }
await this.mouseWheel(9999999, 0, 100);
await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(9999999, 0, defaultFastScrollDuration),
);
await sleep(1000);
} }
async scrollUntilRight(startPoint?: Point): Promise<void> { async scrollUntilRight(startPoint?: Point): Promise<void> {
@ -429,7 +448,11 @@ ${Object.keys(size)
await this.mouseDrag(start, end); await this.mouseDrag(start, end);
return; return;
} }
await this.mouseWheel(-9999999, 0, 100);
await repeat(defaultScrollUntilTimes, () =>
this.mouseWheel(-9999999, 0, defaultFastScrollDuration),
);
await sleep(1000);
} }
async scrollUp(distance?: number, startPoint?: Point): Promise<void> { async scrollUp(distance?: number, startPoint?: Point): Promise<void> {
@ -444,7 +467,7 @@ ${Object.keys(size)
return; return;
} }
await this.mouseWheel(0, scrollDistance, 1000); await this.mouseWheel(0, scrollDistance);
} }
async scrollDown(distance?: number, startPoint?: Point): Promise<void> { async scrollDown(distance?: number, startPoint?: Point): Promise<void> {
@ -459,7 +482,7 @@ ${Object.keys(size)
return; return;
} }
await this.mouseWheel(0, -scrollDistance, 1000); await this.mouseWheel(0, -scrollDistance);
} }
async scrollLeft(distance?: number, startPoint?: Point): Promise<void> { async scrollLeft(distance?: number, startPoint?: Point): Promise<void> {
@ -474,7 +497,7 @@ ${Object.keys(size)
return; return;
} }
await this.mouseWheel(scrollDistance, 0, 1000); await this.mouseWheel(scrollDistance, 0);
} }
async scrollRight(distance?: number, startPoint?: Point): Promise<void> { async scrollRight(distance?: number, startPoint?: Point): Promise<void> {
@ -489,7 +512,7 @@ ${Object.keys(size)
return; return;
} }
await this.mouseWheel(-scrollDistance, 0, 1000); await this.mouseWheel(-scrollDistance, 0);
} }
private async ensureYadb() { private async ensureYadb() {
@ -595,7 +618,7 @@ ${Object.keys(size)
private async mouseWheel( private async mouseWheel(
deltaX: number, deltaX: number,
deltaY: number, deltaY: number,
duration = 1000, duration = defaultNormalScrollDuration,
): Promise<void> { ): Promise<void> {
const { width, height } = await this.size(); const { width, height } = await this.size();

View File

@ -192,7 +192,7 @@ export const getAIConfig = (
); );
} }
return getGlobalConfig()[configKey]?.trim(); return getGlobalConfig()[configKey]?.trim?.();
}; };
export const getAIConfigInBoolean = ( export const getAIConfigInBoolean = (

View File

@ -87,3 +87,12 @@ export function logMsg(...message: Parameters<typeof console.log>) {
console.log(...message); console.log(...message);
} }
} }
export async function repeat(
times: number,
fn: (index: number) => Promise<void>,
) {
for (let i = 0; i < times; i++) {
await fn(i);
}
}