mirror of
https://github.com/web-infra-dev/midscene.git
synced 2025-12-25 05:59:31 +00:00
feat(playwright): add aiTap/aiInput and other AI actions for Playwright integration (#489)
* feat(playwright): add aiTap/aiInput and other AI actions for Playwright integration * chore: update doc --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
This commit is contained in:
Zhou Xiao
GitHub
parent
649aeceb43
commit
36d47e4aef
@ -2,30 +2,30 @@
|
||||
|
||||
import { PackageManagerTabs } from '@theme';
|
||||
|
||||
[Playwright.js](https://playwright.com/) is an open-source automation library developed by Microsoft, primarily designed for end-to-end testing and web scraping of web applications.
|
||||
[Playwright.js](https://playwright.com/) is an open-source automation library developed by Microsoft, primarily used for end-to-end testing and web scraping of web applications.
|
||||
|
||||
We assume you already have a project with Playwright.
|
||||
Here we assume you already have a repository with Playwright integration.
|
||||
|
||||
:::info Demo Project
|
||||
you can check the demo project of Playwright here: [https://github.com/web-infra-dev/midscene-example/blob/main/playwright-demo](https://github.com/web-infra-dev/midscene-example/blob/main/playwright-demo)
|
||||
:::info Example Project
|
||||
You can find an example project of Playwright integration here: [https://github.com/web-infra-dev/midscene-example/blob/main/playwright-demo](https://github.com/web-infra-dev/midscene-example/blob/main/playwright-demo)
|
||||
:::
|
||||
|
||||
## Preparation
|
||||
## Prerequisites
|
||||
|
||||
Config the OpenAI API key, or [config model and provider](./model-provider)
|
||||
Configure OpenAI API Key, or [Custom Model and Provider](./model-provider)
|
||||
|
||||
```bash
|
||||
# replace with your own
|
||||
# Update with your own Key
|
||||
export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz"
|
||||
```
|
||||
|
||||
## Step 1. install dependency, update configuration
|
||||
## Step 1: Add Dependencies and Update Configuration
|
||||
|
||||
add the dependency
|
||||
Add dependencies
|
||||
|
||||
<PackageManagerTabs command="install @midscene/web --save-dev" />
|
||||
|
||||
update playwright.config.ts
|
||||
Update playwright.config.ts
|
||||
|
||||
```diff
|
||||
export default defineConfig({
|
||||
@ -35,9 +35,9 @@ export default defineConfig({
|
||||
});
|
||||
```
|
||||
|
||||
## Step 2. extend the `test` instance
|
||||
## Step 2: Extend the `test` Instance
|
||||
|
||||
Save the following code as `./e2e/fixture.ts`;
|
||||
Save the following code as `./e2e/fixture.ts`:
|
||||
|
||||
```typescript
|
||||
import { test as base } from '@playwright/test';
|
||||
@ -47,49 +47,236 @@ import { PlaywrightAiFixture } from '@midscene/web/playwright';
|
||||
export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
|
||||
```
|
||||
|
||||
## Step 3. write the test case
|
||||
## Step 3: Write Test Cases
|
||||
|
||||
Save the following code as `./e2e/ebay-search.spec.ts`
|
||||
### Basic AI Operation APIs
|
||||
|
||||
#### `ai` - General AI Command
|
||||
```typescript
|
||||
ai<T = any>(
|
||||
prompt: string,
|
||||
opts?: {
|
||||
type?: 'action' | 'query'; // Specify operation type
|
||||
trackNewTab?: boolean; // Whether to track new tabs
|
||||
}
|
||||
): Promise<T>
|
||||
```
|
||||
Used to execute general AI commands, handling various interaction scenarios.
|
||||
|
||||
#### `aiAction` - Execute AI Action
|
||||
```typescript
|
||||
aiAction(taskPrompt: string): Promise<void>
|
||||
```
|
||||
Execute specific AI actions, such as clicking, inputting, etc.
|
||||
|
||||
#### `aiTap` - Click Operation
|
||||
```typescript
|
||||
aiTap(
|
||||
target: string | {
|
||||
prompt: string; // Target element description
|
||||
searchArea?: string; // Search area
|
||||
deepThink?: boolean; // Whether to think deeply
|
||||
},
|
||||
options?: { // Optional configuration
|
||||
timeout?: number; // Timeout duration
|
||||
retry?: number; // Retry attempts
|
||||
force?: boolean; // Whether to force click
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
Execute click operation, AI will intelligently identify target elements.
|
||||
|
||||
#### `aiHover` - Hover Operation
|
||||
```typescript
|
||||
aiHover(
|
||||
target: string | {
|
||||
prompt: string; // Target element description
|
||||
searchArea?: string; // Search area
|
||||
deepThink?: boolean; // Whether to think deeply
|
||||
},
|
||||
options?: { // Optional configuration
|
||||
timeout?: number; // Timeout duration
|
||||
retry?: number; // Retry attempts
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
Execute mouse hover operation.
|
||||
|
||||
#### `aiInput` - Input Operation
|
||||
```typescript
|
||||
aiInput(
|
||||
text: string, // Text to input
|
||||
target: string | {
|
||||
prompt: string; // Target element description
|
||||
searchArea?: string; // Search area
|
||||
deepThink?: boolean; // Whether to think deeply
|
||||
},
|
||||
options?: { // Optional configuration
|
||||
timeout?: number; // Timeout duration
|
||||
retry?: number; // Retry attempts
|
||||
clear?: boolean; // Whether to clear input first
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
Execute text input operation.
|
||||
|
||||
#### `aiKeyboardPress` - Keyboard Operation
|
||||
```typescript
|
||||
aiKeyboardPress(
|
||||
key: string, // Key name
|
||||
target?: string | {
|
||||
prompt: string; // Target element description
|
||||
searchArea?: string; // Search area
|
||||
deepThink?: boolean; // Whether to think deeply
|
||||
},
|
||||
options?: { // Optional configuration
|
||||
timeout?: number; // Timeout duration
|
||||
retry?: number; // Retry attempts
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
Execute keyboard key press operation.
|
||||
|
||||
#### `aiScroll` - Scroll Operation
|
||||
```typescript
|
||||
aiScroll(
|
||||
scroll: {
|
||||
direction: 'down' | 'up' | 'right' | 'left'; // Scroll direction
|
||||
scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft'; // Scroll type
|
||||
distance?: number; // Scroll distance
|
||||
},
|
||||
target?: string | {
|
||||
prompt: string; // Target element description
|
||||
searchArea?: string; // Search area
|
||||
deepThink?: boolean; // Whether to think deeply
|
||||
},
|
||||
options?: { // Optional configuration
|
||||
timeout?: number; // Timeout duration
|
||||
retry?: number; // Retry attempts
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
Execute page scroll operation.
|
||||
|
||||
### Advanced APIs
|
||||
|
||||
#### `generateMidsceneAgent` - Generate AI Agent
|
||||
```typescript
|
||||
generateMidsceneAgent(
|
||||
page?: Page, // Optional page instance
|
||||
opts?: { // Agent configuration options
|
||||
selector?: string; // Selector
|
||||
ignoreMarker?: boolean; // Whether to ignore markers
|
||||
forceSameTabNavigation?: boolean; // Whether to force navigation in the same tab
|
||||
bridgeMode?: false | 'newTabWithUrl' | 'currentTab'; // Bridge mode
|
||||
closeNewTabsAfterDisconnect?: boolean; // Whether to close new tabs after disconnection
|
||||
}
|
||||
): Promise<PageAgent>
|
||||
```
|
||||
Generate an independent AI Agent instance for more complex interaction scenarios.
|
||||
|
||||
### Query and Assertion APIs
|
||||
|
||||
#### `aiQuery` - AI Query
|
||||
```typescript
|
||||
aiQuery<T = any>(
|
||||
query: string | Record<string, string>, // Query description or structured query
|
||||
options?: { // Optional configuration
|
||||
timeout?: number; // Timeout duration
|
||||
retry?: number; // Retry attempts
|
||||
}
|
||||
): Promise<T>
|
||||
```
|
||||
Use AI to execute query operations, returning structured data.
|
||||
|
||||
#### `aiAssert` - AI Assertion
|
||||
```typescript
|
||||
aiAssert(
|
||||
assertion: string, // Assertion description
|
||||
options?: { // Optional configuration
|
||||
timeout?: number; // Timeout duration
|
||||
retry?: number; // Retry attempts
|
||||
keepRawResponse?: boolean; // Whether to keep raw response
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
Use AI to execute assertion checks.
|
||||
|
||||
#### `aiWaitFor` - AI Wait
|
||||
```typescript
|
||||
aiWaitFor(
|
||||
assertion: string, // Wait condition description
|
||||
options?: { // Wait options
|
||||
checkIntervalMs?: number; // Check interval
|
||||
timeoutMs?: number; // Timeout duration
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
Wait for specific conditions to be met.
|
||||
|
||||
### Example Code
|
||||
|
||||
```typescript title="./e2e/ebay-search.spec.ts"
|
||||
import { expect } from "@playwright/test";
|
||||
import { test } from "./fixture";
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
page.setViewportSize({ width: 1280, height: 800 });
|
||||
page.setViewportSize({ width: 400, height: 905 });
|
||||
await page.goto("https://www.ebay.com");
|
||||
await page.waitForLoadState("networkidle");
|
||||
});
|
||||
|
||||
test("search headphone on ebay", async ({ ai, aiQuery, aiAssert }) => {
|
||||
// 👀 type keywords, perform a search
|
||||
await ai('type "Headphones" in search box, hit Enter');
|
||||
|
||||
// 👀 find the items
|
||||
const items = await aiQuery(
|
||||
"{itemTitle: string, price: Number}[], find item in list and corresponding price"
|
||||
test("search headphone on ebay", async ({
|
||||
ai,
|
||||
aiQuery,
|
||||
aiAssert,
|
||||
aiInput,
|
||||
aiTap,
|
||||
aiScroll
|
||||
}) => {
|
||||
// Use aiInput to enter search keyword
|
||||
await aiInput('Headphones', 'search box', { clear: true });
|
||||
|
||||
// Use aiTap to click search button
|
||||
await aiTap('search button');
|
||||
|
||||
// Wait for search results to load
|
||||
await aiWaitFor('search results list loaded', { timeoutMs: 5000 });
|
||||
|
||||
// Use aiScroll to scroll to bottom
|
||||
await aiScroll(
|
||||
{
|
||||
direction: 'down',
|
||||
scrollType: 'untilBottom'
|
||||
},
|
||||
'search results list'
|
||||
);
|
||||
|
||||
|
||||
// Use aiQuery to get product information
|
||||
const items = await aiQuery<Array<{title: string, price: number}>>(
|
||||
'get product titles and prices from search results'
|
||||
);
|
||||
|
||||
console.log("headphones in stock", items);
|
||||
expect(items?.length).toBeGreaterThan(0);
|
||||
|
||||
// 👀 assert by AI
|
||||
await aiAssert("There is a category filter on the left");
|
||||
|
||||
// Use aiAssert to verify filter functionality
|
||||
await aiAssert("category filter exists on the left side");
|
||||
});
|
||||
```
|
||||
|
||||
For the agent's more APIs, please refer to [API](./API).
|
||||
For more Agent API details, please refer to [API Reference](./API).
|
||||
|
||||
## Step 4. run the test case
|
||||
## Step 4. Run Test Cases
|
||||
|
||||
```bash
|
||||
npx playwright test ./e2e/ebay-search.spec.ts
|
||||
```
|
||||
|
||||
## Step 5. view test report
|
||||
## Step 5. View Test Report
|
||||
|
||||
After the above command executes successfully, the console will output: `Midscene - report file updated: ./current_cwd/midscene_run/report/some_id.html`. You can open this file in a browser to view the report.
|
||||
After the command executes successfully, it will output: `Midscene - report file updated: ./current_cwd/midscene_run/report/some_id.html`. Open this file in your browser to view the report.
|
||||
|
||||
## More
|
||||
|
||||
You may also be interested in [Prompting Tips](./prompting-tips)
|
||||
You might also want to learn about [Prompting Tips](./prompting-tips)
|
||||
@ -50,7 +50,172 @@ export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
|
||||
|
||||
## 第三步：编写测试用例
|
||||
|
||||
编写下方代码，保存为 `./e2e/ebay-search.spec.ts`
|
||||
### 基础 AI 操作 API
|
||||
|
||||
#### `ai` - 通用 AI 交互
|
||||
```typescript
|
||||
ai<T = any>(
|
||||
prompt: string,
|
||||
opts?: {
|
||||
type?: 'action' | 'query'; // 指定操作类型
|
||||
trackNewTab?: boolean; // 是否追踪新标签页
|
||||
}
|
||||
): Promise<T>
|
||||
```
|
||||
用于执行通用的 AI 指令，可以处理各种交互场景。
|
||||
|
||||
#### `aiAction` - 执行 AI 动作
|
||||
```typescript
|
||||
aiAction(taskPrompt: string): Promise<void>
|
||||
```
|
||||
执行特定的 AI 动作，如点击、输入等。
|
||||
|
||||
#### `aiTap` - 点击操作
|
||||
```typescript
|
||||
aiTap(
|
||||
target: string | {
|
||||
prompt: string; // 目标元素描述
|
||||
searchArea?: string; // 搜索区域
|
||||
deepThink?: boolean; // 是否深度思考
|
||||
},
|
||||
options?: { // 可选配置
|
||||
timeout?: number; // 超时时间
|
||||
retry?: number; // 重试次数
|
||||
force?: boolean; // 是否强制点击
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
执行点击操作，AI 会智能识别目标元素。
|
||||
|
||||
#### `aiHover` - 悬停操作
|
||||
```typescript
|
||||
aiHover(
|
||||
target: string | {
|
||||
prompt: string; // 目标元素描述
|
||||
searchArea?: string; // 搜索区域
|
||||
deepThink?: boolean; // 是否深度思考
|
||||
},
|
||||
options?: { // 可选配置
|
||||
timeout?: number; // 超时时间
|
||||
retry?: number; // 重试次数
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
执行鼠标悬停操作。
|
||||
|
||||
#### `aiInput` - 输入操作
|
||||
```typescript
|
||||
aiInput(
|
||||
text: string, // 要输入的文本
|
||||
target: string | {
|
||||
prompt: string; // 目标元素描述
|
||||
searchArea?: string; // 搜索区域
|
||||
deepThink?: boolean; // 是否深度思考
|
||||
},
|
||||
options?: { // 可选配置
|
||||
timeout?: number; // 超时时间
|
||||
retry?: number; // 重试次数
|
||||
clear?: boolean; // 是否先清空输入框
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
执行文本输入操作。
|
||||
|
||||
#### `aiKeyboardPress` - 键盘操作
|
||||
```typescript
|
||||
aiKeyboardPress(
|
||||
key: string, // 按键名称
|
||||
target?: string | {
|
||||
prompt: string; // 目标元素描述
|
||||
searchArea?: string; // 搜索区域
|
||||
deepThink?: boolean; // 是否深度思考
|
||||
},
|
||||
options?: { // 可选配置
|
||||
timeout?: number; // 超时时间
|
||||
retry?: number; // 重试次数
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
执行键盘按键操作。
|
||||
|
||||
#### `aiScroll` - 滚动操作
|
||||
```typescript
|
||||
aiScroll(
|
||||
scroll: {
|
||||
direction: 'down' | 'up' | 'right' | 'left'; // 滚动方向
|
||||
scrollType: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft'; // 滚动类型
|
||||
distance?: number; // 滚动距离
|
||||
},
|
||||
target?: string | {
|
||||
prompt: string; // 目标元素描述
|
||||
searchArea?: string; // 搜索区域
|
||||
deepThink?: boolean; // 是否深度思考
|
||||
},
|
||||
options?: { // 可选配置
|
||||
timeout?: number; // 超时时间
|
||||
retry?: number; // 重试次数
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
执行页面滚动操作。
|
||||
|
||||
### 高级 API
|
||||
|
||||
#### `generateMidsceneAgent` - 生成 AI Agent
|
||||
```typescript
|
||||
generateMidsceneAgent(
|
||||
page?: Page, // 可选的页面实例
|
||||
opts?: { // Agent 配置选项
|
||||
selector?: string; // 选择器
|
||||
ignoreMarker?: boolean; // 是否忽略标记
|
||||
forceSameTabNavigation?: boolean; // 是否强制在同一标签页导航
|
||||
bridgeMode?: false | 'newTabWithUrl' | 'currentTab'; // 桥接模式
|
||||
closeNewTabsAfterDisconnect?: boolean; // 断开连接后是否关闭新标签页
|
||||
}
|
||||
): Promise<PageAgent>
|
||||
```
|
||||
生成一个独立的 AI Agent 实例，可以用于更复杂的交互场景。
|
||||
|
||||
### 查询和断言 API
|
||||
|
||||
#### `aiQuery` - AI 查询
|
||||
```typescript
|
||||
aiQuery<T = any>(
|
||||
query: string | Record<string, string>, // 查询描述或结构化查询
|
||||
options?: { // 可选配置
|
||||
timeout?: number; // 超时时间
|
||||
retry?: number; // 重试次数
|
||||
}
|
||||
): Promise<T>
|
||||
```
|
||||
使用 AI 执行查询操作，返回结构化数据。
|
||||
|
||||
#### `aiAssert` - AI 断言
|
||||
```typescript
|
||||
aiAssert(
|
||||
assertion: string, // 断言描述
|
||||
options?: { // 可选配置
|
||||
timeout?: number; // 超时时间
|
||||
retry?: number; // 重试次数
|
||||
keepRawResponse?: boolean; // 是否保留原始响应
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
使用 AI 执行断言检查。
|
||||
|
||||
#### `aiWaitFor` - AI 等待
|
||||
```typescript
|
||||
aiWaitFor(
|
||||
assertion: string, // 等待条件描述
|
||||
options?: { // 等待选项
|
||||
checkIntervalMs?: number; // 检查间隔
|
||||
timeoutMs?: number; // 超时时间
|
||||
}
|
||||
): Promise<void>
|
||||
```
|
||||
等待特定条件满足。
|
||||
|
||||
### 示例代码
|
||||
|
||||
```typescript title="./e2e/ebay-search.spec.ts"
|
||||
import { expect } from "@playwright/test";
|
||||
@ -62,20 +227,41 @@ test.beforeEach(async ({ page }) => {
|
||||
await page.waitForLoadState("networkidle");
|
||||
});
|
||||
|
||||
test("search headphone on ebay", async ({ ai, aiQuery, aiAssert }) => {
|
||||
// 👀 输入关键字，执行搜索
|
||||
// 注：尽管这是一个英文页面，你也可以用中文指令控制它
|
||||
await ai('在搜索框输入 "Headphones" ，敲回车');
|
||||
|
||||
// 👀 找到列表里耳机相关的信息
|
||||
const items = await aiQuery(
|
||||
'{itemTitle: string, price: Number}[], 找到列表里的商品标题和价格'
|
||||
test("search headphone on ebay", async ({
|
||||
ai,
|
||||
aiQuery,
|
||||
aiAssert,
|
||||
aiInput,
|
||||
aiTap,
|
||||
aiScroll
|
||||
}) => {
|
||||
// 使用 aiInput 输入搜索关键词
|
||||
await aiInput('Headphones', '搜索框', { clear: true });
|
||||
|
||||
// 使用 aiTap 点击搜索按钮
|
||||
await aiTap('搜索按钮');
|
||||
|
||||
// 等待搜索结果加载
|
||||
await aiWaitFor('搜索结果列表已加载', { timeoutMs: 5000 });
|
||||
|
||||
// 使用 aiScroll 滚动到页面底部
|
||||
await aiScroll(
|
||||
{
|
||||
direction: 'down',
|
||||
scrollType: 'untilBottom'
|
||||
},
|
||||
'搜索结果列表'
|
||||
);
|
||||
|
||||
|
||||
// 使用 aiQuery 获取商品信息
|
||||
const items = await aiQuery<Array<{title: string, price: number}>>(
|
||||
'获取搜索结果中的商品标题和价格'
|
||||
);
|
||||
|
||||
console.log("headphones in stock", items);
|
||||
expect(items?.length).toBeGreaterThan(0);
|
||||
|
||||
// 👀 用 AI 断言
|
||||
|
||||
// 使用 aiAssert 验证筛选功能
|
||||
await aiAssert("界面左侧有类目筛选功能");
|
||||
});
|
||||
```
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import type { PageAgent } from '@/common/agent';
|
||||
import type { PageAgent, PageAgentOpt } from '@/common/agent';
|
||||
import { PlaywrightAgent } from '@/playwright/index';
|
||||
import type { AgentWaitForOpt } from '@midscene/core';
|
||||
import { type TestInfo, type TestType, test } from '@playwright/test';
|
||||
@ -28,6 +28,7 @@ const groupAndCaseForTest = (testInfo: TestInfo) => {
|
||||
|
||||
const midsceneAgentKeyId = '_midsceneAgentId';
|
||||
export const midsceneDumpAnnotationId = 'MIDSCENE_DUMP_ANNOTATION';
|
||||
|
||||
export const PlaywrightAiFixture = (options?: {
|
||||
forceSameTabNavigation?: boolean;
|
||||
}) => {
|
||||
@ -36,6 +37,7 @@ export const PlaywrightAiFixture = (options?: {
|
||||
const agentForPage = (
|
||||
page: OriginPlaywrightPage,
|
||||
testInfo: TestInfo, // { testId: string; taskFile: string; taskTitle: string },
|
||||
opts?: PageAgentOpt,
|
||||
) => {
|
||||
let idForPage = (page as any)[midsceneAgentKeyId];
|
||||
if (!idForPage) {
|
||||
@ -50,11 +52,53 @@ export const PlaywrightAiFixture = (options?: {
|
||||
groupName: taskTitle,
|
||||
groupDescription: taskFile,
|
||||
generateReport: false, // we will generate it in the reporter
|
||||
...opts,
|
||||
});
|
||||
}
|
||||
return pageAgentMap[idForPage];
|
||||
};
|
||||
|
||||
async function generateAiAction(options: {
|
||||
page: OriginPlaywrightPage;
|
||||
testInfo: TestInfo;
|
||||
use: any;
|
||||
aiActionType:
|
||||
| 'ai'
|
||||
| 'aiAction'
|
||||
| 'aiHover'
|
||||
| 'aiInput'
|
||||
| 'aiKeyboardPress'
|
||||
| 'aiScroll'
|
||||
| 'aiTap'
|
||||
| 'aiQuery'
|
||||
| 'aiAssert'
|
||||
| 'aiWaitFor';
|
||||
}) {
|
||||
const { page, testInfo, use, aiActionType } = options;
|
||||
const agent = agentForPage(page, testInfo);
|
||||
await use(async (taskPrompt: string, ...args: any[]) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
test.step(`ai-${aiActionType} - ${JSON.stringify(taskPrompt)}`, async () => {
|
||||
await waitForNetworkIdle(page);
|
||||
try {
|
||||
type AgentMethod = (
|
||||
prompt: string,
|
||||
...restArgs: any[]
|
||||
) => Promise<any>;
|
||||
const result = await (agent[aiActionType] as AgentMethod)(
|
||||
taskPrompt,
|
||||
...(args || []),
|
||||
);
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
||||
}
|
||||
|
||||
const updateDumpAnnotation = (test: TestInfo, dump: string) => {
|
||||
const currentAnnotation = test.annotations.find((item) => {
|
||||
return item.type === midsceneDumpAnnotationId;
|
||||
@ -70,130 +114,172 @@ export const PlaywrightAiFixture = (options?: {
|
||||
};
|
||||
|
||||
return {
|
||||
generateMidsceneAgent: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
await use(
|
||||
async (
|
||||
propsPage?: OriginPlaywrightPage | undefined,
|
||||
opts?: PageAgentOpt,
|
||||
) => {
|
||||
const agent = agentForPage(propsPage || page, testInfo, opts);
|
||||
return agent;
|
||||
},
|
||||
);
|
||||
},
|
||||
ai: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
const agent = agentForPage(page, testInfo);
|
||||
|
||||
await use(
|
||||
async (
|
||||
taskPrompt: string,
|
||||
opts?: { type?: 'action' | 'query'; trackNewTab?: boolean },
|
||||
) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const { type = 'action' } = opts || {};
|
||||
|
||||
test.step(`ai - ${taskPrompt}`, async () => {
|
||||
await waitForNetworkIdle(page);
|
||||
try {
|
||||
const result = await agent.ai(taskPrompt, type);
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
},
|
||||
);
|
||||
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'ai',
|
||||
});
|
||||
},
|
||||
aiAction: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
const agent = agentForPage(page, testInfo);
|
||||
await use(async (taskPrompt: string) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
test.step(`aiAction - ${taskPrompt}`, async () => {
|
||||
await waitForNetworkIdle(page);
|
||||
try {
|
||||
const result = await agent.aiAction(taskPrompt);
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiAction',
|
||||
});
|
||||
},
|
||||
aiTap: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiTap',
|
||||
});
|
||||
},
|
||||
aiHover: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiHover',
|
||||
});
|
||||
},
|
||||
aiInput: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiInput',
|
||||
});
|
||||
},
|
||||
aiKeyboardPress: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiKeyboardPress',
|
||||
});
|
||||
},
|
||||
aiScroll: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiScroll',
|
||||
});
|
||||
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
||||
},
|
||||
aiQuery: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
const agent = agentForPage(page, testInfo);
|
||||
await use(async (demand: any) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
|
||||
await waitForNetworkIdle(page);
|
||||
try {
|
||||
const result = await agent.aiQuery(demand);
|
||||
resolve(result);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiQuery',
|
||||
});
|
||||
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
||||
},
|
||||
aiAssert: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
const agent = agentForPage(page, testInfo);
|
||||
await use(async (assertion: string, errorMsg?: string) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
test.step(`aiAssert - ${assertion}`, async () => {
|
||||
await waitForNetworkIdle(page);
|
||||
try {
|
||||
await agent.aiAssert(assertion, errorMsg);
|
||||
resolve(null);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiAssert',
|
||||
});
|
||||
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
||||
},
|
||||
aiWaitFor: async (
|
||||
{ page }: { page: OriginPlaywrightPage },
|
||||
use: any,
|
||||
testInfo: TestInfo,
|
||||
) => {
|
||||
const agent = agentForPage(page, testInfo);
|
||||
await use(async (assertion: string, opt?: AgentWaitForOpt) => {
|
||||
return new Promise((resolve, reject) => {
|
||||
test.step(`aiWaitFor - ${assertion}`, async () => {
|
||||
await waitForNetworkIdle(page);
|
||||
try {
|
||||
await agent.aiWaitFor(assertion, opt);
|
||||
resolve(null);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
await generateAiAction({
|
||||
page,
|
||||
testInfo,
|
||||
use,
|
||||
aiActionType: 'aiWaitFor',
|
||||
});
|
||||
updateDumpAnnotation(testInfo, agent.dumpDataString());
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export type PlayWrightAiFixtureType = {
|
||||
generateMidsceneAgent: (page?: any, opts?: any) => Promise<PageAgent>;
|
||||
ai: <T = any>(
|
||||
prompt: string,
|
||||
opts?: { type?: 'action' | 'query'; trackNewTab?: boolean },
|
||||
) => Promise<T>;
|
||||
aiAction: (taskPrompt: string) => ReturnType<PageTaskExecutor['action']>;
|
||||
aiQuery: <T = any>(demand: any) => Promise<T>;
|
||||
aiAssert: (assertion: string, errorMsg?: string) => Promise<void>;
|
||||
aiAction: (taskPrompt: string) => ReturnType<PageAgent['aiAction']>;
|
||||
aiTap: (
|
||||
...args: Parameters<PageAgent['aiTap']>
|
||||
) => ReturnType<PageAgent['aiTap']>;
|
||||
aiHover: (
|
||||
...args: Parameters<PageAgent['aiHover']>
|
||||
) => ReturnType<PageAgent['aiHover']>;
|
||||
aiInput: (
|
||||
...args: Parameters<PageAgent['aiInput']>
|
||||
) => ReturnType<PageAgent['aiInput']>;
|
||||
aiKeyboardPress: (
|
||||
...args: Parameters<PageAgent['aiKeyboardPress']>
|
||||
) => ReturnType<PageAgent['aiKeyboardPress']>;
|
||||
aiScroll: (
|
||||
...args: Parameters<PageAgent['aiScroll']>
|
||||
) => ReturnType<PageAgent['aiScroll']>;
|
||||
aiQuery: <T = any>(
|
||||
...args: Parameters<PageAgent['aiQuery']>
|
||||
) => ReturnType<PageAgent['aiQuery']>;
|
||||
aiAssert: (
|
||||
...args: Parameters<PageAgent['aiAssert']>
|
||||
) => ReturnType<PageAgent['aiAssert']>;
|
||||
aiWaitFor: (assertion: string, opt?: AgentWaitForOpt) => Promise<void>;
|
||||
};
|
||||
|
||||
|
||||
@ -0,0 +1,40 @@
|
||||
import { AiAssert } from '@midscene/core/.';
|
||||
import { expect } from 'playwright/test';
|
||||
import { test } from './fixture';
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
await page.goto('https://www.saucedemo.com/');
|
||||
await page.setViewportSize({ width: 1920, height: 1080 });
|
||||
});
|
||||
|
||||
const CACHE_TIME_OUT = process.env.MIDSCENE_CACHE;
|
||||
|
||||
test('ai shop', async ({
|
||||
ai,
|
||||
aiInput,
|
||||
aiAssert,
|
||||
aiQuery,
|
||||
aiTap,
|
||||
generateMidsceneAgent,
|
||||
page,
|
||||
}) => {
|
||||
if (CACHE_TIME_OUT) {
|
||||
test.setTimeout(1000 * 1000);
|
||||
}
|
||||
// login
|
||||
const agent = await generateMidsceneAgent(page);
|
||||
await aiInput('standard_user', 'in user name input');
|
||||
await aiInput('secret_sauce', 'in password input');
|
||||
await agent.aiTap('Login Button');
|
||||
|
||||
// check the login success
|
||||
await aiAssert('the page title is "Swag Labs"');
|
||||
|
||||
// add to cart
|
||||
await aiTap('"add to cart" for black t-shirt products');
|
||||
|
||||
await aiTap({
|
||||
prompt: 'click right top cart icon',
|
||||
deepThink: true,
|
||||
});
|
||||
});
|
||||
@ -1,5 +1,5 @@
|
||||
import type { PlayWrightAiFixtureType } from '@/index';
|
||||
import { PlaywrightAiFixture } from '@/index';
|
||||
import type { PlayWrightAiFixtureType } from '@/playwright/ai-fixture';
|
||||
import { PlaywrightAiFixture } from '@/playwright/ai-fixture';
|
||||
import { test as base } from '@playwright/test';
|
||||
|
||||
export const test = base.extend<PlayWrightAiFixtureType>(PlaywrightAiFixture());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user