mirror of
https://github.com/web-infra-dev/midscene.git
synced 2025-12-05 03:19:05 +00:00
fix(cache): app should not crash when it fails to get the cache id (#93)
This commit is contained in:
parent
d481ea4201
commit
a2e4ff61b0
@ -34,7 +34,19 @@ Bad ❌: "[number, number], the [x, y] coords of the main button"
|
||||
|
||||
Use the visualization tool to debug and understand each step of Midscene. Just upload the log, and view the AI's parse results. You can find [the tool](/visualization/) on the navigation bar on this site.
|
||||
|
||||
### Remember to cross-check the result by assertion
|
||||
### Infer or assert from the interface, not the DOM properties or browser status
|
||||
|
||||
All the data sent to the LLM is in the form of screenshots and element coordinates. The DOM and the browser instance are almost invisible to the LLM. Therefore, ensure everything you expect is visible in the on the screen.
|
||||
|
||||
Good ✅: The title is blue
|
||||
|
||||
Bad ❌: The title has a `test-id-size` property
|
||||
|
||||
Bad ❌: The browser has two active tabs
|
||||
|
||||
Bad ❌: The request has finished.
|
||||
|
||||
### Cross-check the result using assertion
|
||||
|
||||
LLM could behave incorrectly. A better practice is to check its result after running.
|
||||
|
||||
@ -49,13 +61,7 @@ expect(taskList.length).toBe(1);
|
||||
expect(taskList[0]).toBe('Learning AI the day after tomorrow');
|
||||
```
|
||||
|
||||
### Infer from the UI, not the DOM properties
|
||||
|
||||
All the data sent to the LLM are the screenshots and element coordinates. The DOM is almost invisible to the LLM. So do not expect the LLM infer any information from the DOM (such as `test-id-*` properties).
|
||||
|
||||
Ensure everything you expect from the LLM is visible in the screenshot.
|
||||
|
||||
### non-English prompting is acceptable
|
||||
### Non-English prompting is acceptable
|
||||
|
||||
Since most AI models can understand many languages, feel free to write the prompt in any language you prefer. It usually works even if the prompt is in a language different from the page's language.
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
# Cache
|
||||
|
||||
Midscene.js provides AI caching capabilities to improve the stability and speed of the entire AI execution process. The cache here mainly refers to caching AI's recognition of page elements. When the page elements have not changed, the AI query results will be cached.
|
||||
Midscene.js provides AI caching features to improve the stability and speed of the entire AI execution process. The cache mainly refers to caching how AI recognizes page elements. Cached AI query results are used if page elements haven't changed.
|
||||
|
||||
## Instructions
|
||||
|
||||
@ -15,6 +15,8 @@ Currently, the caching capability is supported in all scenarios, and Midscene ca
|
||||
|
||||
**Effect**
|
||||
|
||||
After enabling the cache, the execution time is significantly reduced, for example, from 1m16s to 23s.
|
||||
|
||||
* **before**
|
||||
|
||||

|
||||
|
||||
@ -33,6 +33,18 @@
|
||||
|
||||
使用可视化工具调试和理解 Midscene 的每个步骤。只需上传日志,就可以查看 AI 的解析结果。你可以在本站导航栏上找到 [可视化工具](/visualization/)。
|
||||
|
||||
### 从界面做推断,而不是 DOM 属性或者浏览器状态
|
||||
|
||||
所有传递给 LLM 的数据都是截图和元素坐标。DOM和浏览器 对 LLM 来说几乎是不可见的。因此,务必确保你想提取的信息都在截图中有所体现且能被 LLM “看到”。
|
||||
|
||||
正确示例 ✅:标题是蓝色的
|
||||
|
||||
错误实例 ❌:标题有个 `test-id-size` 属性
|
||||
|
||||
错误实例 ❌:浏览器有两个 tab 开着
|
||||
|
||||
错误实例 ❌:异步请求已经结束了
|
||||
|
||||
### 通过断言交叉检查结果
|
||||
|
||||
LLM 可能会表现出错误的行为。更好的做法是运行操作后检查其结果。
|
||||
@ -48,12 +60,6 @@ expect(taskList.length).toBe(1);
|
||||
expect(taskList[0]).toBe('后天学习 AI');
|
||||
```
|
||||
|
||||
### 从界面而不是 DOM 属性推断信息
|
||||
|
||||
所有传递给 LLM 的数据都是截图和元素坐标。DOM 对 LLM 来说几乎是不可见的。因此,不要指望 LLM 能从 DOM 中推断任何信息(比如 `test-id-*` 属性)。
|
||||
|
||||
务必确保你想提取的信息都在截图中有所体现且能被 LLM “看到”。
|
||||
|
||||
### 中、英文提示词都是可行的
|
||||
|
||||
由于大多数 AI 模型可以理解多种语言,所以请随意用你喜欢的语言撰写提示指令。即使提示语言与页面语言不同,通常也是可行的。
|
||||
|
||||
@ -15,6 +15,8 @@ Midscene.js 提供了 AI 缓存能力,用于提升整个 AI 执行过程的稳
|
||||
|
||||
**使用效果**
|
||||
|
||||
通过引入缓存后,用例的执行时间大幅降低了,例如从1分16秒降低到了23秒。
|
||||
|
||||
* **before**
|
||||
|
||||

|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import { assert } from 'node:console';
|
||||
import type { WebPage } from '@/common/page';
|
||||
import type {
|
||||
AgentWaitForOpt,
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import assert from 'node:assert';
|
||||
import type { Buffer } from 'node:buffer';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import type { ElementInfo } from '@/extractor';
|
||||
@ -96,9 +97,9 @@ export function printReportMsg(filepath: string) {
|
||||
* Get the current execution file name
|
||||
* @returns The name of the current execution file
|
||||
*/
|
||||
export function getCurrentExecutionFile(): string {
|
||||
export function getCurrentExecutionFile(trace?: string): string | false {
|
||||
const error = new Error();
|
||||
const stackTrace = error.stack;
|
||||
const stackTrace = trace || error.stack;
|
||||
const pkgDir = process.cwd() || '';
|
||||
if (stackTrace) {
|
||||
const stackLines = stackTrace.split('\n');
|
||||
@ -120,9 +121,7 @@ export function getCurrentExecutionFile(): string {
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`Can not find current execution file: \n __dirname: ${__dirname}, \n stackTrace: ${stackTrace}`,
|
||||
);
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
@ -150,7 +149,14 @@ export function getCurrentExecutionFile(): string {
|
||||
|
||||
const testFileIndex = new Map<string, number>();
|
||||
export function generateCacheId(fileName?: string): string {
|
||||
const taskFile = fileName || getCurrentExecutionFile();
|
||||
let taskFile = fileName || getCurrentExecutionFile();
|
||||
if (!taskFile) {
|
||||
taskFile = randomUUID();
|
||||
console.warn(
|
||||
'Midscene - using random UUID for cache id. Cache may be invalid.',
|
||||
);
|
||||
}
|
||||
|
||||
if (testFileIndex.has(taskFile)) {
|
||||
const currentIndex = testFileIndex.get(taskFile);
|
||||
if (currentIndex !== undefined) {
|
||||
|
||||
@ -6,4 +6,25 @@ describe('TaskCache', () => {
|
||||
const currentExecutionFile = getCurrentExecutionFile();
|
||||
expect(currentExecutionFile).toBe('/tests/unit-test/util.test.ts');
|
||||
});
|
||||
|
||||
it('should return false if trace is not a valid file', () => {
|
||||
const trace =
|
||||
'at Function.Module._resolveFilename (node:internal/modules/cjs/loader:1138:15)';
|
||||
const currentExecutionFile = getCurrentExecutionFile(trace);
|
||||
expect(currentExecutionFile).toBe(false);
|
||||
});
|
||||
|
||||
it('extract trace from puppeteer', () => {
|
||||
const trace = `
|
||||
at getCurrentExecutionFile (/Users/user/workspace/midscene-example/puppeteer-demo/node_modules/@midscene/web/dist/es/puppeteer.js:561:11)
|
||||
at generateCacheId (/Users/user/workspace/midscene-example/puppeteer-demo/node_modules/@midscene/web/dist/es/puppeteer.js:568:32)
|
||||
at TaskCache (/Users/user/workspace/midscene-example/puppeteer-demo/node_modules/@midscene/web/dist/es/puppeteer.js:590:24)
|
||||
at PageTaskExecutor (/Users/user/workspace/midscene-example/puppeteer-demo/node_modules/@midscene/web/dist/es/puppeteer.js:711:26)
|
||||
at PageAgent (/Users/user/workspace/midscene-example/puppeteer-demo/node_modules/@midscene/web/dist/es/puppeteer.js:1147:29)
|
||||
at PuppeteerAgent (/Users/user/workspace/midscene-example/puppeteer-demo/node_modules/@midscene/web/dist/es/puppeteer.js:1352:9)
|
||||
at <anonymous> (/Users/user/workspace/midscene-example/puppeteer-demo/demo.ts:24:17)
|
||||
`;
|
||||
const currentExecutionFile = getCurrentExecutionFile(trace);
|
||||
expect(currentExecutionFile).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user