midscene/packages/core/tests/ai/llm-planning/basic.test.ts

import { plan } from '@/ai-model';
import { vlLocateMode } from '@midscene/shared/env';
import { getContextFromFixture } from 'tests/evaluation';
/* eslint-disable max-lines-per-function */
import { describe, expect, it, vi } from 'vitest';

vi.setConfig({
  testTimeout: 180 * 1000,
  hookTimeout: 30 * 1000,
});

const vlMode = vlLocateMode();

describe.skipIf(vlMode)('automation - llm planning', () => {
  it('basic run', async () => {
    const { context } = await getContextFromFixture('todo');

    const { actions } = await plan(
      'type "Why is the earth a sphere?", wait 3.5s, hit Enter',
      {
        context,
      },
    );
    expect(actions).toBeTruthy();
    expect(actions!.length).toBe(3);
    expect(actions![0].type).toBe('Input');
    expect(actions![1].type).toBe('Sleep');
    expect(actions![1].param).toMatchSnapshot();
    expect(actions![2].type).toBe('KeyboardPress');
    expect(actions![2].param).toMatchSnapshot();
  });

  it('scroll page', async () => {
    const { context } = await getContextFromFixture('todo');
    const { actions } = await plan(
      'Scroll down the page by 200px, scroll up the page by 100px, scroll right the second item of the task list by 300px',
      { context },
    );
    expect(actions).toBeTruthy();
    expect(actions!.length).toBe(3);
    expect(actions![0].type).toBe('Scroll');
    expect(actions![0].locate).toBeNull();
    expect(actions![0].param).toBeDefined();

    expect(actions![2].locate).toBeTruthy();
    expect(actions![2].param).toBeDefined();
  });
});

describe('planning', () => {
  const todoInstructions = [
    {
      name: 'input first todo item',
      instruction: '在任务框 input 输入 今天学习 JS，按回车键',
    },
    {
      name: 'input second todo item',
      instruction: '在任务框 input 输入 明天学习 Rust，按回车键',
    },
    {
      name: 'input third todo item',
      instruction: '在任务框 input 输入后天学习 AI，按回车键',
    },
    {
      name: 'delete second todo item',
      instruction:
        '将鼠标移动到任务列表中的第二项，点击第二项任务右边的删除按钮',
    },
    {
      name: 'check second todo item',
      instruction: '点击第二条任务左边的勾选按钮',
    },
    {
      name: 'filter completed items',
      instruction: '点击任务列表下面的 completed 状态按钮',
    },
  ];

  todoInstructions.forEach(({ name, instruction }) => {
    it(`todo mvc - ${name}`, async () => {
      const { context } = await getContextFromFixture('todo');
      const { actions } = await plan(instruction, { context });
      expect(actions).toBeTruthy();
      expect(actions![0].locate).toBeTruthy();
      expect(actions![0].locate?.prompt).toBeTruthy();
      expect(actions![0].locate?.id || actions![0].locate?.bbox).toBeTruthy();
    });
  });

  it('scroll some element', async () => {
    const { context } = await getContextFromFixture('todo');
    const { actions } = await plan(
      'Scroll left the status filters (with a button named "completed")',
      {
        context,
      },
    );
    expect(actions).toBeTruthy();
    expect(actions![0].type).toBe('Scroll');
    expect(actions![0].locate).toBeTruthy();
  });

  it.skip('should not throw in an "if" statement', async () => {
    const { context } = await getContextFromFixture('todo');
    const { actions, error } = await plan(
      'If there is a cookie prompt, close it',
      { context },
    );

    expect(actions?.length === 1).toBeTruthy();
    expect(actions?.[0]!.type).toBe('ExpectedFalsyCondition');
  });

  it('should make mark unfinished when something is not found', async () => {
    const { context } = await getContextFromFixture('todo');
    const res = await plan(
      'click the input box, wait 300ms, click the close button of the cookie prompt',
      { context },
    );

    expect(res.more_actions_needed_by_instruction).toBeTruthy();
    expect(res.log).toBeDefined();
  });
});