yujunjun/midscene
1
0
Fork 0
mirror of https://github.com/web-infra-dev/midscene.git synced 2025-12-27 15:10:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: optimize ai e2e test (#99)

Browse Source
This commit is contained in:
Zhou xiao 2024-09-29 17:51:59 +08:00 committed by GitHub
parent 10757a8ba1
commit 8ec5622342
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 526 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

511
packages/midscene/tests/ai/evaluate/__snapshots__/planning-input.test.ts.snap Normal file
View File

@ -0,0 +1,511 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`automation - planning input - openAI > input value 1`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value 2`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
{
"param": {
"value": "Enter",
},
"thought": undefined,
"type": "KeyboardPress",
},
]
`;
exports[`automation - planning input - openAI > input value 3`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value 4`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
{
"param": {
"value": "Enter",
},
"thought": undefined,
"type": "KeyboardPress",
},
]
`;
exports[`automation - planning input - openAI > input value 5`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value 6`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
{
"param": {
"value": "Enter",
},
"thought": undefined,
"type": "KeyboardPress",
},
]
`;
exports[`automation - planning input - openAI > input value 7`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value 8`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
{
"param": {
"value": "Enter",
},
"thought": undefined,
"type": "KeyboardPress",
},
]
`;
exports[`automation - planning input - openAI > input value 9`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value 10`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "learning english",
},
"thought": undefined,
"type": "Input",
},
{
"param": {
"value": "Enter",
},
"thought": undefined,
"type": "KeyboardPress",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 1`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn English tomorrow",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 2`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn Skiing",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 3`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 4`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn English tomorrow",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 5`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn Skiing",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 6`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 7`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn English tomorrow",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 8`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn Skiing",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 9`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 10`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn English tomorrow",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 11`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn Skiing",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 12`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 13`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn English tomorrow",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 14`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn Skiing",
},
"thought": undefined,
"type": "Input",
},
]
`;
exports[`automation - planning input - openAI > input value Add, delete, correct and check 15`] = `
[
{
"param": {
"prompt": "",
},
"thought": undefined,
"type": "Locate",
},
{
"param": {
"value": "Learn",
},
"thought": undefined,
"type": "Input",
},
]
`;

13
packages/midscene/tests/ai/evaluate/__snapshots__/planning.test.ts.snap Normal file
View File

@ -0,0 +1,13 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
exports[`automation - planning openAI > basic run 1`] = `
{
"timeMs": 3500,
}
`;
exports[`automation - planning openAI > basic run 2`] = `
{
"value": "Enter",
}
`;

2
packages/midscene/tests/ai/evaluate/test-suite/util.ts
View File

@ -163,6 +163,6 @@ export async function getPageTestData(targetDir: string) {
}
export async function getPageDataOfTestName(testName: string) {
const targetDir = path.join(__dirname, `test-data/${testName}`);
const targetDir = path.join(__dirname, `../test-data/${testName}`);
return await getPageTestData(targetDir);
}

2
packages/midscene/tests/ai/util.ts
View File

@ -16,4 +16,4 @@ export const modelList: Array<'openAI' | 'coze'> = useCozeModel('coze')
? ['openAI', 'coze']
: ['openAI'];
export const repeatTime = process.env.GITHUB_ACTIONS ? 2 : 4;
export const repeatTime = process.env.GITHUB_ACTIONS ? 2 : 1;

2
packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts
View File

@ -21,8 +21,6 @@ test('ai report', async ({ page, ai, aiAssert }, testInfo) => {
type: 'query',
},
);
const parseList = JSON.stringify(actionsList, null, 4);
expect(parseList).toMatchSnapshot();
await aiAssert(
'On the left taskbar, check whether the specific execution content of the right task is normal',
);

66
packages/web-integration/tests/ai/web/playwright-report-test/todo-report.spec.ts-snapshots/ai-report-1-report.txt
View File

@ -1,66 +0,0 @@
[
{
"title": "Enter \"Learn\" in the task box, don't press enter",
"actions": [
"Planning",
"Insight / Locate",
"Action / Input"
]
},
{
"title": "Add \"JS today\" to base on the existing content(important) of the task box, then press enter",
"actions": [
"Planning",
"Insight / Locate",
"Action / Input",
"Action / KeyboardPress"
]
},
{
"title": "Enter \"Learn Rust tomorrow\" in the task box, then press Enter to create",
"actions": [
"Planning",
"Insight / Locate",
"Action / Input",
"Action / KeyboardPress"
]
},
{
"title": "Enter \"Learning AI the day after tomorrow\" in the task box, then press Enter to create",
"actions": [
"Planning",
"Insight / Locate",
"Action / Input",
"Action / KeyboardPress"
]
},
{
"title": "string[], tasks in the list",
"actions": [
"Insight / Query"
]
},
{
"title": "Move your mouse over the second item in the task list",
"actions": [
"Planning",
"Insight / Locate",
"Action / Hover"
]
},
{
"title": "Click the delete button to the right of the second task",
"actions": [
"Planning",
"Insight / Locate",
"Action / Tap"
]
},
{
"title": "Click the check button to the left of the second task",
"actions": [
"Planning",
"Insight / Locate"
]
}
]