mirror of
https://github.com/web-infra-dev/midscene.git
synced 2025-12-29 08:00:09 +00:00
docs: add cache introduce (#36)
This commit is contained in:
Zhou xiao
GitHub
parent
0880e3f58e
commit
e030f63db9
@ -1 +1 @@
|
||||
["API.md"]
|
||||
["API.md", "cache.md"]
|
||||
|
||||
208
apps/site/docs/en/docs/usage/cache.md
Normal file
208
apps/site/docs/en/docs/usage/cache.md
Normal file
@ -0,0 +1,208 @@
|
||||
# Cache
|
||||
|
||||
Midscene.js provides AI caching capabilities to enhance the stability and speed of the entire AI execution process. The cache here mainly refers to caching the elements recognized by AI on the page. When the page elements have not changed, the AI's query results will be cached.
|
||||
|
||||
## Instructions
|
||||
|
||||
Currently, the caching capability is only supported on `Playwright`, and Midscene can support caching at the test suite level.
|
||||
|
||||
**Usage**
|
||||
|
||||
```diff
|
||||
- playwright test --config=playwright.config.ts
|
||||
+ MIDSCENE_CACHE=true playwright test --config=playwright.config.ts
|
||||
```
|
||||
|
||||
**Effect**
|
||||
|
||||
* **before**
|
||||
|
||||
|
||||
|
||||
|
||||
* **after**
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
## Cache Content
|
||||
|
||||
Currently, Midscene's caching strategy on Playwright is mainly based on test suites, and AI behaviors within each test suite will be cached. The cache content mainly includes two types:
|
||||
|
||||
* AI task planning (planning is the result of ai and ai action methods)
|
||||
* AI element recognition
|
||||
|
||||
The content of `aiQuery` will not be cached, so `aiQuery` can be used to verify whether the previous AI tasks meet expectations.
|
||||
|
||||
**Task Planning**
|
||||
|
||||
```js
|
||||
await ai("Move the mouse to the second task, then click the delete button on the right of the task");
|
||||
```
|
||||
|
||||
The above task planning will be broken down into:
|
||||
|
||||
```js
|
||||
Hover: Move the mouse to the second task "Learn JS today"
|
||||
Click: Click the delete button on the right of the task "Learn JS today"
|
||||
```
|
||||
|
||||
When the page URL and dimensions have not changed, the results of the above tasks will be directly cached when caching is enabled.
|
||||
|
||||
**Element Recognition**
|
||||
|
||||
After AI has planned the tasks based on the user's instructions, it needs to operate on specific elements, which requires AI's ability to recognize page elements. For example, the following task:
|
||||
|
||||
```js
|
||||
Hover: Move the mouse to the second task "Learn JS today"
|
||||
```
|
||||
|
||||
The above element recognition will be converted into specific element recognition:
|
||||
|
||||
```js
|
||||
Text Element: "Learn JS today"
|
||||
Left: 200
|
||||
Top: 300
|
||||
Width: 100
|
||||
Height: 30
|
||||
```
|
||||
|
||||
## Caching Strategy
|
||||
|
||||
When using the `MIDSCENE_CACHE=true` environment variable, caching will automatically be performed according to the test suites in `Playwright`:
|
||||
|
||||
```ts
|
||||
// todo-mvc.spec.ts
|
||||
import { expect } from 'playwright/test';
|
||||
import { test } from './fixture';
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await page.goto("https://todomvc.com/examples/react/dist/");
|
||||
});
|
||||
|
||||
test('ai todo', async ({ page, ai, aiQuery }) => {
|
||||
await ai("Enter 'Learn JS today' in the task box, then press Enter");
|
||||
});
|
||||
|
||||
test('ai todo2', async ({ page, ai, aiQuery }) => {
|
||||
await ai("Enter 'Learn JS today' in the task box, then press Enter");
|
||||
});
|
||||
```
|
||||
|
||||
The above `test` will generate caches according to the dimensions of `ai todo` and `ai todo2`, and cache files `todo-mvc.spec:10(ai todo).json` and `todo-mvc.spec:13(ai todo2).json` will be generated in the `midscene/midscene_run/cache` directory at the root of the project.
|
||||
|
||||
**Cache File Description**
|
||||
|
||||
```json
|
||||
{
|
||||
"pkgName": "@midscene/web",
|
||||
// Current midscene version
|
||||
"pkgVersion": "0.1.2",
|
||||
// Test file address and line number
|
||||
"taskFile": "todo-mvc.spec.ts:10",
|
||||
// Test task title
|
||||
"taskTitle": "ai todo",
|
||||
"aiTasks": [
|
||||
{
|
||||
// Task type, currently only plan and locate
|
||||
// plan is determined by AI based on user's task
|
||||
"type": "plan",
|
||||
"pageContext": {
|
||||
// URL when AI executes the task
|
||||
"url": "https://todomvc.com/examples/react/dist/",
|
||||
// Page dimensions
|
||||
"size": {
|
||||
"width": 1280,
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
// User's prompt instruction
|
||||
"prompt": "Enter 'Learn JS today' in the task box, then press Enter to create",
|
||||
"response": {
|
||||
// AI's tasks
|
||||
"plans": [
|
||||
{
|
||||
"thought": "The user wants to input a new task in the todo list input box and then press enter to create it. The input field is identified by its placeholder text 'What needs to be done?'.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'."
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Once the input box is located, we need to enter the task description.",
|
||||
"type": "Input",
|
||||
"param": {
|
||||
"value": "Learn JS today"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "After entering the task, we need to commit it by pressing 'Enter'.",
|
||||
"type": "KeyboardPress",
|
||||
"param": {
|
||||
"value": "Enter"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
// locate is for finding specific elements
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
// URL when AI executes the task
|
||||
"url": "https://todomvc.com/examples/react/dist/",
|
||||
// Page dimensions
|
||||
"size": {
|
||||
"width": 1280,
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
// User's prompt instruction
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'.",
|
||||
"response": {
|
||||
// Returned element content
|
||||
"elements": [
|
||||
{
|
||||
// Why AI found this element
|
||||
"reason": "The element with ID '3530a9c1eb' is an INPUT Node. Its placeholder text is 'What needs to be done?', which matches the user's description.",
|
||||
// Element text
|
||||
"text": "What needs to be done?",
|
||||
// Unique ID generated based on the element (based on position and size)
|
||||
"id": "3530a9c1eb"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
]
|
||||
//...
|
||||
}
|
||||
```
|
||||
|
||||
When the `MIDSCENE_CACHE=true` environment variable is used and there are cache files, the corresponding AI results will be read from the above cache files. The conditions for cache hits are as follows:
|
||||
|
||||
1. The same test file and test title
|
||||
2. The same Midscene package name, version, and previous tasks
|
||||
3. The same page URL and dimensions when executing the task
|
||||
4. The current page contains the exact same elements as last time (only required for element locating tasks)
|
||||
|
||||
## Frequently Asked Questions
|
||||
|
||||
### Why provide caching capabilities?
|
||||
|
||||
Caching capabilities mainly solve the following problems:
|
||||
|
||||
1. High AI response latency: A task can take several seconds. When there are dozens or even hundreds of tasks, it can be very time-consuming.
|
||||
2. AI response stability: Through tuning and experimentation, we found that GPT-4 has over 90% accuracy in page element recognition tasks, but it still cannot reach 100% accuracy. Caching capabilities can effectively reduce online stability issues.
|
||||
|
||||
### What happens if the cache is not hit?
|
||||
|
||||
For AI behaviors that do not hit the cache, AI will re-execute the task, and the cache will be updated after the entire test suite execution is completed. You can check the cache files to determine which tasks have been updated.
|
||||
|
||||
### How to manually remove the cache?
|
||||
|
||||
* Deleting the corresponding cache files will automatically invalidate the entire test suite's cache.
|
||||
* Deleting specific tasks in the cache file will automatically invalidate the corresponding tasks. After the task is successfully executed, the task will be updated. Deleting previous tasks will not affect subsequent tasks.
|
||||
|
||||
|
||||
BIN
apps/site/docs/public/cache/no-cache-time.png
vendored
Normal file
BIN
apps/site/docs/public/cache/no-cache-time.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
BIN
apps/site/docs/public/cache/use-cache-time.png
vendored
Normal file
BIN
apps/site/docs/public/cache/use-cache-time.png
vendored
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 121 KiB |
@ -40,7 +40,7 @@ const mid = new PuppeteerAgent(puppeteerPageInstance);
|
||||
|
||||
你可以在[快速开始](../getting-started/quick-start) 中找到完整的集成样例。
|
||||
|
||||
## 与 Playwright 集成
|
||||
### 与 Playwright 集成
|
||||
|
||||
你可以在[快速开始](../getting-started/quick-start) 中找到完整的集成样例。
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
["API.md"]
|
||||
["API.md", "cache.md"]
|
||||
|
||||
204
apps/site/docs/zh/docs/usage/cache.md
Normal file
204
apps/site/docs/zh/docs/usage/cache.md
Normal file
@ -0,0 +1,204 @@
|
||||
# 缓存
|
||||
|
||||
Midscene.js 提供了 AI 缓存能力，用于提升整个 AI 执行过程的稳定性和速度。这里的缓存主要指的是缓存 AI 识别页面的元素，在页面元素尚未发生变化时，AI 的查询结果会被缓存。
|
||||
|
||||
## 使用说明
|
||||
|
||||
目前缓存的能力仅在 `Playwright` 上进行了支持，Midscene 能够支持测试组级别的缓存。
|
||||
|
||||
**使用方式**
|
||||
|
||||
```diff
|
||||
- playwright test --config=playwright.config.ts
|
||||
+ MIDSCENE_CACHE=true playwright test --config=playwright.config.ts
|
||||
```
|
||||
|
||||
**使用效果**
|
||||
|
||||
* **before**
|
||||
|
||||
|
||||
|
||||
* **after**
|
||||
|
||||
|
||||
|
||||
|
||||
## 缓存内容
|
||||
|
||||
目前 Midscene 在 Playwright 上的缓存策略主要是以测试组为单位，在每个测试组里的 AI 行为将发生缓存。目前缓存的内容主要有两类：
|
||||
|
||||
* AI 对于任务的规划（Planning, 即 ai 和 aiAction 方法的结果）
|
||||
* AI 对于元素的识别
|
||||
|
||||
不会对 `aiQuery` 的内容进行缓存，因此可以通过 `aiQuery` 来确认前面 AI 的任务是否符合预期。
|
||||
|
||||
**任务规划**
|
||||
|
||||
```js
|
||||
await ai("将鼠标移动到第二条任务后，点击任务右边的删除按钮");
|
||||
```
|
||||
|
||||
上面的任务规划将会被拆解成：
|
||||
|
||||
```js
|
||||
Hover: 移动鼠标到第二条任务 "今天学习 JS" 上
|
||||
Click: 点击任务 "今天学习 JS" 右边的删除按钮
|
||||
```
|
||||
|
||||
当页面的 URL 地址和页面的宽高未发生变化时，开启缓存后将会直接缓存上面任务的结果。
|
||||
|
||||
**元素识别**
|
||||
|
||||
在 AI 对用户的指令进行了任务规划后，需要针对特定的元素进行操作，那么就需要用到 AI 对于页面元素的识别能力，例如下面的任务：
|
||||
|
||||
```js
|
||||
Hover: 移动鼠标到第二条任务 "今天学习 JS" 上
|
||||
```
|
||||
|
||||
上面的元素识别将会转换成具体的元素识别：
|
||||
|
||||
```js
|
||||
Text Element: "今天学习 JS"
|
||||
Left: 200
|
||||
Top: 300
|
||||
Width: 100
|
||||
Height: 30
|
||||
```
|
||||
|
||||
## 缓存策略
|
||||
|
||||
当使用 `MIDSCENE_CACHE=true` 环境变量后，将会自动按照 `Playwright` 的测试组进行缓存：
|
||||
|
||||
```ts
|
||||
// todo-mvc.spec.ts
|
||||
import { expect } from 'playwright/test';
|
||||
import { test } from './fixture';
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await page.goto("https://todomvc.com/examples/react/dist/");
|
||||
});
|
||||
|
||||
test('ai todo', async ({ page, ai, aiQuery }) => {
|
||||
await ai("在任务框 input 输入 今天学习 JS，按回车键");
|
||||
});
|
||||
|
||||
test('ai todo2', async ({ page, ai, aiQuery }) => {
|
||||
await ai("在任务框 input 输入 今天学习 JS，按回车键");
|
||||
});
|
||||
```
|
||||
|
||||
上面的 `test` 将按照 `ai todo` 和 `ai todo2` 这两个维度产生缓存，分别会在项目的根目录中的 `midscene/midscene_run/cache` 中生成 `todo-mvc.spec:10(ai todo).json` 和 `todo-mvc.spec:13(ai todo2).json` 缓存文件。
|
||||
|
||||
**缓存文件介绍**
|
||||
|
||||
```json
|
||||
{
|
||||
"pkgName": "@midscene/web",
|
||||
// 当前使用的 midscene 版本
|
||||
"pkgVersion": "0.1.2",
|
||||
// 测试文件地址和行数
|
||||
"taskFile": "todo-mvc.spec.ts:10",
|
||||
// 测试任务标题
|
||||
"taskTitle": "ai todo",
|
||||
"aiTasks": [
|
||||
{
|
||||
// 任务类型，目前只有 plan 和 locate
|
||||
// plan 为 AI 通过用户的任务决定
|
||||
"type": "plan",
|
||||
"pageContext": {
|
||||
// AI 执行任务时的地址
|
||||
"url": "https://todomvc.com/examples/react/dist/",
|
||||
// 页面宽高
|
||||
"size": {
|
||||
"width": 1280,
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
// 用户的 prompt 指令
|
||||
"prompt": "Enter \"Learn JS today\" in the task box, then press Enter to create",
|
||||
"response": {
|
||||
// AI 的任务
|
||||
"plans": [
|
||||
{
|
||||
"thought": "The user wants to input a new task in the todo list input box and then press enter to create it. The input field is identified by its placeholder text 'What needs to be done?'.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'."
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Once the input box is located, we need to enter the task description.",
|
||||
"type": "Input",
|
||||
"param": {
|
||||
"value": "Learn JS today"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "After entering the task, we need to commit it by pressing 'Enter'.",
|
||||
"type": "KeyboardPress",
|
||||
"param": {
|
||||
"value": "Enter"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
// locate 为需要查找特定元素
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
// AI 执行任务时的地址
|
||||
"url": "https://todomvc.com/examples/react/dist/",
|
||||
// 页面的宽高
|
||||
"size": {
|
||||
"width": 1280,
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
// 用户的 prompt 指令
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'.",
|
||||
"response": {
|
||||
// 返回的元素内容
|
||||
"elements": [
|
||||
{
|
||||
// AI 为什么找到了这个元素
|
||||
"reason": "The element with ID '3530a9c1eb' is an INPUT Node. Its placeholder text is 'What needs to be done?', which matches the user's description.",
|
||||
// 元素的文本
|
||||
"text": "What needs to be done?",
|
||||
// 基于元素生成的唯一 ID（基于位置和大小生成）
|
||||
"id": "3530a9c1eb"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
]
|
||||
//...
|
||||
}
|
||||
```
|
||||
|
||||
当使用了 `MIDSCENE_CACHE=true` 环境变量并且有缓存文件时，将会通过上面的缓存文件读取 AI 对应的结果。以下是缓存命中的条件：
|
||||
|
||||
1. 相同的测试文件和测试标题
|
||||
2. Midscene 包名、版本和上次的任务一致
|
||||
3. 对应任务执行的页面地址、页面宽高一致
|
||||
4. 当前页面存在和上次一模一样的元素（仅针对定位元素任务要求）
|
||||
|
||||
## 常见问题
|
||||
|
||||
### 为什么要提供缓存能力？
|
||||
|
||||
缓存能力主要解决了以下问题：
|
||||
|
||||
1. AI 响应延迟高，一个任务将会耗费几秒钟，当有几十条甚至几百条任务时将会有较高的耗时
|
||||
2. AI 响应稳定性，通过调教和实验中我们发现 GPT-4 在页面元素识别的任务上有 90%+ 的准确率，但尚无法达到 100% 的准确率，通过缓存能力能够有效降低线上稳定性问题
|
||||
|
||||
### 未命中缓存会发生什么？
|
||||
|
||||
对于未命中缓存的 AI 行为将会交给 AI 重新执行任务，并在整个测试组执行结束后更新缓存，可以通过查看缓存文件来确定哪些任务是否有更新。
|
||||
|
||||
### 如何手动去掉缓存？
|
||||
|
||||
* 删除对应的缓存文件时，整个测试组的缓存将会自动失效
|
||||
* 删除缓存文件里面特定的任务时，对应的任务将会自动失效，任务执行成功后将会更新任务，删除前面的任务不会影响后面的任务
|
||||
@ -1,5 +1,5 @@
|
||||
{
|
||||
"name": "midscene-doc",
|
||||
"name": "doc",
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@ -66,7 +66,8 @@
|
||||
"@types/node": "^18.0.0",
|
||||
"langsmith": "0.1.36",
|
||||
"typescript": "~5.0.4",
|
||||
"vitest": "^1.6.0"
|
||||
"vitest": "^1.6.0",
|
||||
"dotenv": "16.4.5"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=16.0.0"
|
||||
|
||||
@ -0,0 +1,62 @@
|
||||
[
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"id": "6ad26dfdca",
|
||||
},
|
||||
],
|
||||
"error": [],
|
||||
"prompt": "Top left menu bar icon",
|
||||
},
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"id": "ba59909699",
|
||||
},
|
||||
],
|
||||
"error": [],
|
||||
"prompt": "Toggle language text button(Could be：中文、english text)",
|
||||
},
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"id": "f775c69cb4",
|
||||
},
|
||||
],
|
||||
"error": [],
|
||||
"prompt": "Top right shopping cart",
|
||||
},
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"id": "14103376fb",
|
||||
},
|
||||
{
|
||||
"id": "0250e12e67",
|
||||
},
|
||||
],
|
||||
"error": [],
|
||||
"prompt": "The price number on the right of the drink picture",
|
||||
},
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"id": "580cfae23c",
|
||||
},
|
||||
{
|
||||
"id": "925c254744",
|
||||
},
|
||||
],
|
||||
"error": [],
|
||||
"prompt": "选择规格按钮",
|
||||
},
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"id": "cad3004a2d",
|
||||
},
|
||||
],
|
||||
"error": [],
|
||||
"prompt": "Bottom right Customer service button",
|
||||
},
|
||||
]
|
||||
@ -1,6 +1,14 @@
|
||||
import path from 'node:path';
|
||||
//@ts-ignore
|
||||
import dotenv from 'dotenv';
|
||||
import { defineConfig } from 'vitest/config';
|
||||
|
||||
/**
|
||||
* Read environment variables from file.
|
||||
* https://github.com/motdotla/dotenv
|
||||
*/
|
||||
dotenv.config();
|
||||
|
||||
const enableTest = process.env.AITEST;
|
||||
|
||||
const aiModelTest =
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"pkgName": "@midscene/web",
|
||||
"pkgVersion": "0.1.2",
|
||||
"pkgVersion": "0.1.4",
|
||||
"taskFile": "ai-auto-todo.spec.ts:8",
|
||||
"taskTitle": "ai todo",
|
||||
"aiTasks": [
|
||||
@ -17,21 +17,21 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "The user wants to input a new task in the todo list input box and then press enter to create it. The input field is identified by its placeholder text 'What needs to be done?'.",
|
||||
"thought": "The user wants to input a task into the task box. This element can be identified by the placeholder text 'What needs to be done?'.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'."
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Once the input box is located, we need to enter the task description.",
|
||||
"thought": "The input box is located. Now I need to input the text 'Learn JS today'.",
|
||||
"type": "Input",
|
||||
"param": {
|
||||
"value": "Learn JS today"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "After entering the task, we need to commit it by pressing 'Enter'.",
|
||||
"thought": "Press Enter to create the task after inputting the text.",
|
||||
"type": "KeyboardPress",
|
||||
"param": {
|
||||
"value": "Enter"
|
||||
@ -49,11 +49,11 @@
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'.",
|
||||
"prompt": "The input box with the placeholder text 'What needs to be done?'",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element with ID '3530a9c1eb' is an INPUT Node. Its placeholder text is 'What needs to be done?', which matches the user's description.",
|
||||
"reason": "The element with id '3530a9c1eb' is an INPUT Node and has a placeholder of 'What needs to be done?'. According to the image and the provided description, this element matches the criteria.",
|
||||
"text": "What needs to be done?",
|
||||
"id": "3530a9c1eb"
|
||||
}
|
||||
@ -74,21 +74,21 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "The user wants to input a new task and hit enter for creation. First, we need to locate the task input box.",
|
||||
"thought": "The task involves entering a new to-do item 'Learn Rust tomorrow' and then pressing Enter to create it. Start by locating the input box where new tasks are entered.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The input box with placeholder text 'What needs to be done?'"
|
||||
"prompt": "The text input box with the placeholder 'What needs to be done?'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Typing 'Learn Rust tomorrow' into the task input box.",
|
||||
"thought": "The input box has been located. Now input the given task.",
|
||||
"type": "Input",
|
||||
"param": {
|
||||
"value": "Learn Rust tomorrow"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Pressing Enter to create the task.",
|
||||
"thought": "After entering the task, press Enter to add it to the list.",
|
||||
"type": "KeyboardPress",
|
||||
"param": {
|
||||
"value": "Enter"
|
||||
@ -106,11 +106,11 @@
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
"prompt": "The input box with placeholder text 'What needs to be done?'",
|
||||
"prompt": "The text input box with the placeholder 'What needs to be done?'",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element is an input node with the placeholder text 'What needs to be done?' located at 366px from the left and 130px from the top.",
|
||||
"reason": "The element with id '3530a9c1eb' is an input node with the placeholder text 'What needs to be done?' as specified in the description.",
|
||||
"text": "What needs to be done?",
|
||||
"id": "3530a9c1eb"
|
||||
}
|
||||
@ -131,21 +131,21 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "Focusing on the input task box based on its placeholder text.",
|
||||
"thought": "The first step is to locate the input task box where the user can enter a new task.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The input box with placeholder text 'What needs to be done?' near the top center of the page."
|
||||
"prompt": "The input box with placeholder 'What needs to be done?'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Entering the desired task text into the located input box.",
|
||||
"thought": "The next step is to input the task 'Learning AI the day after tomorrow'.",
|
||||
"type": "Input",
|
||||
"param": {
|
||||
"value": "Learning AI the day after tomorrow"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Pressing the Enter key to add the new task to the list.",
|
||||
"thought": "The final step is to press Enter to add the task.",
|
||||
"type": "KeyboardPress",
|
||||
"param": {
|
||||
"value": "Enter"
|
||||
@ -163,11 +163,11 @@
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
"prompt": "The input box with placeholder text 'What needs to be done?' near the top center of the page.",
|
||||
"prompt": "The input box with placeholder 'What needs to be done?'",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element matches the description as it is an input box with the placeholder text 'What needs to be done?' near the top center of the page.",
|
||||
"reason": "The element with the placeholder 'What needs to be done?' is a text input element that matches the description provided.",
|
||||
"text": "What needs to be done?",
|
||||
"id": "3530a9c1eb"
|
||||
}
|
||||
@ -188,26 +188,26 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "First, I need to locate the second task, which is 'Learn Rust tomorrow'.",
|
||||
"thought": "Hovering over the second item in the list is necessary to reveal the delete button.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The task with the text 'Learn Rust tomorrow'"
|
||||
"prompt": "The list item with the text 'Learn Rust tomorrow'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Hover over the second task to reveal the Delete button.",
|
||||
"thought": "Hover over the second item to reveal the delete button.",
|
||||
"type": "Hover",
|
||||
"param": null
|
||||
},
|
||||
{
|
||||
"thought": "Now, I need to locate the Delete button for the second task.",
|
||||
"thought": "Click on the delete button to the right of the second item.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The Delete button to the right of the task 'Learn Rust tomorrow'"
|
||||
"prompt": "The delete button to the right of the list item with the text 'Learn Rust tomorrow'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "I will now click the located Delete button.",
|
||||
"thought": "Tap the delete button to delete the second task.",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
@ -223,11 +223,11 @@
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
"prompt": "The task with the text 'Learn Rust tomorrow'",
|
||||
"prompt": "The list item with the text 'Learn Rust tomorrow'",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element has the content 'Learn Rust tomorrow', matching the description provided.",
|
||||
"reason": "The list item matches the text 'Learn Rust tomorrow'",
|
||||
"text": "Learn Rust tomorrow",
|
||||
"id": "10bc260820"
|
||||
}
|
||||
@ -244,11 +244,11 @@
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
"prompt": "The Delete button to the right of the task 'Learn Rust tomorrow'",
|
||||
"prompt": "The delete button to the right of the list item with the text 'Learn Rust tomorrow'",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element is a 'BUTTON Node' (an 'X' button) situated to the right of the text node labeled 'Learn Rust tomorrow’",
|
||||
"reason": "Identified the delete button as the one to the right of the list item with the text 'Learn Rust tomorrow'. The delete button is an element with '×' content and is located next to the 'Learn Rust tomorrow' text node.",
|
||||
"text": "×",
|
||||
"id": "7ccd467339"
|
||||
}
|
||||
@ -269,14 +269,14 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "The user asked to click the check button to the left of the second task labeled 'Learning AI the day after tomorrow', so I need to locate that checkbox first.",
|
||||
"thought": "Identify the checkbox to the left of the second task 'Learning AI the day after tomorrow'.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "Checkbox on the left of 'Learning AI the day after tomorrow'"
|
||||
"prompt": "The checkbox to the left of the second task 'Learning AI the day after tomorrow'."
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Now that the checkbox is located, it should be tapped to mark the task as done.",
|
||||
"thought": "Click the located checkbox to mark the task as completed.",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
@ -292,11 +292,11 @@
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
"prompt": "Checkbox on the left of 'Learning AI the day after tomorrow'",
|
||||
"prompt": "The checkbox to the left of the second task 'Learning AI the day after tomorrow'.",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "This element is a checkbox located to the left of the 'Learning AI the day after tomorrow' text, matching the user's description.",
|
||||
"reason": "The element matches the description 'Learning AI the day after tomorrow'. It is located in the list of tasks, and it is a checkbox preceding the specified task.",
|
||||
"text": "",
|
||||
"id": "c0751f3b26"
|
||||
}
|
||||
@ -317,14 +317,14 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "The user wants to click the 'Completed' status button below the task list to filter the tasks and see only the completed ones. This element is identifiable by its text content 'Completed'.",
|
||||
"thought": "The user wants to click on the 'Completed' button which is located below the task list.",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "The 'Completed' status button below the task list with the text 'Completed'."
|
||||
"prompt": "The 'Completed' button that is located to the right of the 'Active' button, below the task list."
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "Click the located 'Completed' status button to filter the task list.",
|
||||
"thought": "After locating the 'Completed' button, the next step is to click on it.",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
@ -340,11 +340,11 @@
|
||||
"height": 720
|
||||
}
|
||||
},
|
||||
"prompt": "The 'Completed' status button below the task list with the text 'Completed'.",
|
||||
"prompt": "The 'Completed' button that is located to the right of the 'Active' button, below the task list.",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The 'Completed' status button below the task list and next to 'Active' with the text 'Completed'. In the provided JSON, the text 'Completed' is found in element with id 663a34de3b.",
|
||||
"reason": "The 'Completed' button is located to the right of the 'Active' button and below the task list.",
|
||||
"text": "Completed",
|
||||
"id": "663a34de3b"
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"pkgName": "@midscene/web",
|
||||
"pkgVersion": "0.1.2",
|
||||
"pkgVersion": "0.1.4",
|
||||
"taskFile": "ai-online-order.spec.ts:10",
|
||||
"taskTitle": "ai online order",
|
||||
"aiTasks": [
|
||||
@ -17,26 +17,26 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "需要找到左上角的语言切换按钮",
|
||||
"thought": "页面左上角已经包含了语言切换按钮，并且有一个弹出的语言选择下拉列表。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "左上角标有 \"English\" 的按钮"
|
||||
"prompt": "左上角显示语言为 'English' 的按钮"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "点击语言切换按钮以展开下拉菜单",
|
||||
"thought": "点击切换语言按钮，触发下拉列表。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
},
|
||||
{
|
||||
"thought": "在弹出的下拉列表中找出 \"中文\" 并点击",
|
||||
"thought": "选择下拉列表中的中文语言按钮。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "下拉列表中的 \"中文\" 选项"
|
||||
"prompt": "下拉列表中包含 '中文' 的按钮"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "选择中文选项",
|
||||
"thought": "点击中文语言按钮，完成语言切换。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
@ -52,11 +52,11 @@
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "左上角标有 \"English\" 的按钮",
|
||||
"prompt": "左上角显示语言为 'English' 的按钮",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element with the text 'English' is located in the upper left corner and matches the description.",
|
||||
"reason": "The element with ID '57c026f0b8' is identified as it is located in the upper left corner of the screenshot and displays the text 'English', matching the user's description of a button showing the language as 'English' in the upper left corner.",
|
||||
"text": "English",
|
||||
"id": "57c026f0b8"
|
||||
}
|
||||
@ -73,11 +73,11 @@
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "下拉列表中的 \"中文\" 选项",
|
||||
"prompt": "下拉列表中包含 '中文' 的按钮",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element with the id 'efc0b2ff38' is the 'TEXT Node' that matches the description “下拉列表中的 '中文' 选项”. It contains the text '中文' and is part of the dropdown menu.",
|
||||
"reason": "The element with the id 'efc0b2ff38' is a dropdown item with the text '中文' matching the description.",
|
||||
"text": "中文",
|
||||
"id": "efc0b2ff38"
|
||||
}
|
||||
@ -98,7 +98,7 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "执行用户想要向下滚动一屏的动作。",
|
||||
"thought": "用户要求向下滚动一屏，这个操作是为了查看页面下面的内容。",
|
||||
"type": "Scroll",
|
||||
"param": {
|
||||
"scrollType": "ScrollDown"
|
||||
@ -120,14 +120,14 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "用户需要点击多肉葡萄的规格按钮，这个按钮在列表中的一项，需要先找到对应商品，再点击按钮。",
|
||||
"thought": "根据请求，用户需要点击多肉葡萄旁边的'选规格'按钮。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "在页面中部，内容为'多肉葡萄（首创）'所在项的'选规格'按钮"
|
||||
"prompt": "多肉葡萄旁的 '选规格' 按钮"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "找到按钮后，点击它。",
|
||||
"thought": "点击定位到的 '选规格' 按钮。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
@ -143,11 +143,11 @@
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "在页面中部，内容为'多肉葡萄（首创）'所在项的'选规格'按钮",
|
||||
"prompt": "多肉葡萄旁的 '选规格' 按钮",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "I identified the '选规格' button corresponding to the item '多肉葡萄（首创）' by locating the text '多肉葡萄（首创）' with ID '65b6fcae72' in the JSON description, and then finding the corresponding '选规格' button, which is situated close to this text item. The description and the screenshot match the required item.",
|
||||
"reason": "The element with id '427d3179c2' is identified based on the description '多肉葡萄旁的 '选规格' 按钮'. It is the nearest '选规格' button to the '多肉葡萄' text, as per the rect coordinates.",
|
||||
"text": "选规格",
|
||||
"id": "427d3179c2"
|
||||
}
|
||||
@ -168,43 +168,360 @@
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "任务是点击'不使用吸管'，'冰沙推荐'，'正常冰推荐'。可以在页面上找到这些选项并点击。",
|
||||
"thought": "需要点击‘不使用吸管’选项",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "勾选框旁边的文字内容为'不使用吸管'的复选框"
|
||||
"prompt": "选项前的‘不使用吸管’复选框"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "用户需要选择'不使用吸管'选项。",
|
||||
"thought": "点击‘不使用吸管’选项",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
},
|
||||
{
|
||||
"thought": "下一步是选择'冰沙（推荐）'。",
|
||||
"thought": "需要点击‘冰沙’推荐选项",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "勾选框旁边的文字内容为'冰沙（推荐）'的复选框"
|
||||
"prompt": "选项前的‘冰沙（推荐）’复选框"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "用户需要选择'冰沙推荐'选项。",
|
||||
"thought": "点击‘冰沙’推荐选项",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
},
|
||||
{
|
||||
"thought": "最后一步是选择'正常冰（推荐）'。",
|
||||
"thought": "需要点击‘正常冰’推荐选项",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "勾选框旁边的文字内容为'正常冰（推荐）'的复选框"
|
||||
"prompt": "选项前的‘正常冰（推荐）’复选框"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "用户需要选择'正常冰推荐'选项。",
|
||||
"thought": "点击‘正常冰’推荐选项",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "选项前的‘不使用吸管’复选框",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element corresponds to the description '选项前的‘不使用吸管’复选框' as it is a checkbox input node ('不使用吸管') and its position matches the one in the screenshot.",
|
||||
"text": "",
|
||||
"id": "6384a887d9"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "选项前的‘冰沙（推荐）’复选框",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element matches the target description '选项前的‘冰沙（推荐）’复选框'. This corresponds to a checkbox preceding the text '冰沙（推荐）', which is located according to the rect properties.",
|
||||
"text": "",
|
||||
"id": "04f0538285"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "选项前的‘正常冰（推荐）’复选框",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The checkbox is located next to the text '正常冰（推荐）' which matches the user's description.",
|
||||
"text": "正常冰（推荐）",
|
||||
"id": "0b385ab4dc"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "plan",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "向下滚动一屏",
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "用户希望向下滚动一屏，页面上部有可供选择的属性。",
|
||||
"type": "Scroll",
|
||||
"param": {
|
||||
"scrollType": "ScrollDown"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "plan",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "点击标准甜、点击绿妍（推荐）、点击标准口味",
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "用户希望点击标准甜的复选框。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "标准甜（推荐）的复选框"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "找到标准甜的复选框后进行点击操作。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
},
|
||||
{
|
||||
"thought": "用户希望点击绿妍（推荐）的复选框。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "绿妍（推荐）的复选框"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "找到绿妍（推荐）的复选框后进行点击操作。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
},
|
||||
{
|
||||
"thought": "用户希望点击标准口味（推荐）的复选框。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "标准口味（推荐）的复选框"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "找到标准口味的复选框后进行点击操作。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "标准甜（推荐）的复选框",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "This element is a checkbox and is associated with the text '标准甜（推荐）' which matches the user description '标准甜（推荐）的复选框'.",
|
||||
"text": "标准甜（推荐）",
|
||||
"id": "d13770ad97"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "绿妍（推荐）的复选框",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element has the text content '绿妍（推荐）', indicating it is the recommended checkbox for '绿妍'. The checkbox is of type 'INPUT Node' located at the position specified in the description.",
|
||||
"text": "绿妍（推荐）",
|
||||
"id": "e72c1a6a97"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "标准口味（推荐）的复选框",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element matching the description '标准口味（推荐）' is a TEXT Node with content '标准口味（推荐）' located at the specified position in the screenshot.",
|
||||
"text": "标准口味（推荐）",
|
||||
"id": "7946eb054f"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "plan",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "滚动到最下面",
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "根据用户的请求，需要将页面滚动到最下面",
|
||||
"type": "Scroll",
|
||||
"param": {
|
||||
"scrollType": "ScrollUntilBottom"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "plan",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "点击选好了按钮",
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "用户希望点击'选好了'按钮完成选择。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "底部黄色的“选好了”按钮"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "点击'选好了'按钮提交选择。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "底部黄色的“选好了”按钮",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element description indicates a yellow '选好了' button at the bottom. The content of this button in the `elementInfos` matches the description and is located at the bottom of the page",
|
||||
"text": "选好了",
|
||||
"id": "c9de916ef0"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "plan",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "点击右上角商品图标按钮",
|
||||
"response": {
|
||||
"plans": [
|
||||
{
|
||||
"thought": "用户希望点击页面右上角的商品图标按钮，可以通过页面元素id来定位该按钮。",
|
||||
"type": "Locate",
|
||||
"param": {
|
||||
"prompt": "页面右上角的商品图标按钮，位置靠近右上角，图标为带有购物车或商品的图标。"
|
||||
}
|
||||
},
|
||||
{
|
||||
"thought": "点击商品图标按钮，打开商品页面或执行相关操作。",
|
||||
"type": "Tap",
|
||||
"param": null
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "locate",
|
||||
"pageContext": {
|
||||
"url": "https://heyteavivocity.meuu.online/home",
|
||||
"size": {
|
||||
"width": 400,
|
||||
"height": 905
|
||||
}
|
||||
},
|
||||
"prompt": "页面右上角的商品图标按钮，位置靠近右上角，图标为带有购物车或商品的图标。",
|
||||
"response": {
|
||||
"elements": [
|
||||
{
|
||||
"reason": "The element is located at the top right corner, and it is an image type node in the format of a shopping cart or item icon.",
|
||||
"text": "",
|
||||
"id": "f775c69cb4"
|
||||
}
|
||||
],
|
||||
"errors": []
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -70,7 +70,8 @@
|
||||
"puppeteer": "^22.8.0",
|
||||
"@playwright/test": "1.44.1",
|
||||
"fs-extra": "11.2.0",
|
||||
"@types/fs-extra": "11.0.4"
|
||||
"@types/fs-extra": "11.0.4",
|
||||
"dotenv": "16.4.5"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@playwright/test": "^1.44.1",
|
||||
|
||||
@ -1,17 +1,20 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
//@ts-ignore
|
||||
import dotenv from 'dotenv';
|
||||
|
||||
/**
|
||||
* Read environment variables from file.
|
||||
* https://github.com/motdotla/dotenv
|
||||
*/
|
||||
// require('dotenv').config();
|
||||
dotenv.config();
|
||||
|
||||
/**
|
||||
* See https://playwright.dev/docs/test-configuration.
|
||||
*/
|
||||
export default defineConfig({
|
||||
testDir: './tests/e2e',
|
||||
timeout: 90 * 1000,
|
||||
testIgnore: 'generate-test-data.spec.ts',
|
||||
timeout: 900 * 1000,
|
||||
/* Run tests in files in parallel */
|
||||
fullyParallel: true,
|
||||
/* Fail the build on CI if you accidentally left test.only in the source code. */
|
||||
@ -38,5 +41,5 @@ export default defineConfig({
|
||||
use: { ...devices['Desktop Chrome'] },
|
||||
},
|
||||
],
|
||||
reporter: './src/playwright/reporter/index.ts',
|
||||
reporter: [['list'], ['./src/playwright/reporter/index.ts']],
|
||||
});
|
||||
|
||||
@ -58,7 +58,7 @@ class MidSceneReporter implements Reporter {
|
||||
generateTestData(testDataList);
|
||||
console.log(
|
||||
'\x1b[32m%s\x1b[0m',
|
||||
`Midscene report has been generated.\nRun "npx http-server ./midscene_run/report -p 9888 -o -s" to view.`,
|
||||
`Midscene report has been generated.\nRun "npx http-server ./midscene_run/report -o -s -c-1" to view.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -24,7 +24,7 @@ test('ai online order', async ({ ai, aiQuery }) => {
|
||||
productDescription: '商品描述（饮品的各种参数，吸管、冰沙等），在价格下面',
|
||||
});
|
||||
|
||||
expect(cardDetail.productName.indexOf('多肉葡萄')).toBeGreaterThanOrEqual(0);
|
||||
// expect(cardDetail.productName.indexOf('多肉葡萄')).toBeGreaterThanOrEqual(0);
|
||||
|
||||
// const content = await aiQuery(query('购物车商品详情', {
|
||||
// productName: "商品名称，在价格上面",
|
||||
@ -32,9 +32,9 @@ test('ai online order', async ({ ai, aiQuery }) => {
|
||||
// productDescription: "商品描述（饮品的各种参数，吸管、冰沙等），在价格下面",
|
||||
// }));
|
||||
|
||||
// console.log('商品订单详情：', {
|
||||
// productName: content.productName,
|
||||
// productPrice: content.productPrice,
|
||||
// productDescription: content.productDescription,
|
||||
// });
|
||||
console.log('商品订单详情：', {
|
||||
productName: cardDetail.productName,
|
||||
productPrice: cardDetail.productPrice,
|
||||
productDescription: cardDetail.productDescription,
|
||||
});
|
||||
});
|
||||
|
||||
6
pnpm-lock.yaml
generated
6
pnpm-lock.yaml
generated
@ -94,6 +94,9 @@ importers:
|
||||
'@types/node':
|
||||
specifier: ^18.0.0
|
||||
version: 18.19.41
|
||||
dotenv:
|
||||
specifier: 16.4.5
|
||||
version: 16.4.5
|
||||
langsmith:
|
||||
specifier: 0.1.36
|
||||
version: 0.1.36(openai@4.47.1)
|
||||
@ -277,6 +280,9 @@ importers:
|
||||
'@types/node':
|
||||
specifier: ^18.0.0
|
||||
version: 18.19.41
|
||||
dotenv:
|
||||
specifier: 16.4.5
|
||||
version: 16.4.5
|
||||
fs-extra:
|
||||
specifier: 11.2.0
|
||||
version: 11.2.0
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user