diff --git a/apps/site/docs/en/docs/usage/model-vendor.md b/apps/site/docs/en/docs/usage/model-vendor.md index ea169a07f..e30838458 100644 --- a/apps/site/docs/en/docs/usage/model-vendor.md +++ b/apps/site/docs/en/docs/usage/model-vendor.md @@ -17,6 +17,9 @@ Optional: # optional, if you want to use a customized endpoint export OPENAI_BASE_URL="https://..." +# optional, if you want to use Azure OpenAI Service +export OPENAI_USE_AZURE="true" + # optional, if you want to specify a model name other than gpt-4o export MIDSCENE_MODEL_NAME='claude-3-opus-20240229'; diff --git a/apps/site/docs/zh/docs/usage/model-vendor.md b/apps/site/docs/zh/docs/usage/model-vendor.md index 8ed27bf57..9c89afaad 100644 --- a/apps/site/docs/zh/docs/usage/model-vendor.md +++ b/apps/site/docs/zh/docs/usage/model-vendor.md @@ -17,6 +17,9 @@ export OPENAI_API_KEY="sk-abcdefghijklmnopqrstuvwxyz" # 可选, 如果你想更换 base URL export OPENAI_BASE_URL="https://..." +# 可选, 如果你想使用 Azure OpenAI 服务 +export OPENAI_USE_AZURE="true" + # 可选, 如果你想指定模型名称 export MIDSCENE_MODEL_NAME='claude-3-opus-20240229'; diff --git a/packages/midscene/src/ai-model/openai/index.ts b/packages/midscene/src/ai-model/openai/index.ts index e8d7d5d8e..1c4f6da83 100644 --- a/packages/midscene/src/ai-model/openai/index.ts +++ b/packages/midscene/src/ai-model/openai/index.ts @@ -1,7 +1,7 @@ import assert from 'node:assert'; import { AIResponseFormat } from '@/types'; import { wrapOpenAI } from 'langsmith/wrappers'; -import OpenAI, { type ClientOptions } from 'openai'; +import OpenAI, { type ClientOptions, AzureOpenAI } from 'openai'; import type { ChatCompletionMessageParam } from 'openai/resources'; import { planSchema } from '../automation/planning'; import { AIActionType } from '../common'; @@ -15,6 +15,8 @@ export const MIDSCENE_LANGSMITH_DEBUG = 'MIDSCENE_LANGSMITH_DEBUG'; export const MIDSCENE_DEBUG_AI_PROFILE = 'MIDSCENE_DEBUG_AI_PROFILE'; export const OPENAI_API_KEY = 'OPENAI_API_KEY'; +const OPENAI_USE_AZURE = 'OPENAI_USE_AZURE'; + export function useOpenAIModel(useModel?: 'coze' | 'openAI') { if (useModel && useModel !== 'openAI') return false; if (process.env[OPENAI_API_KEY]) return true; @@ -39,7 +41,13 @@ if (typeof process.env[MIDSCENE_MODEL_NAME] === 'string') { } async function createOpenAI() { - const openai = new OpenAI(extraConfig); + let openai: OpenAI | AzureOpenAI; + if (process.env[OPENAI_USE_AZURE]) { + console.log('Using Azure OpenAI'); + openai = new AzureOpenAI(extraConfig); + } else { + openai = new OpenAI(extraConfig); + } if (process.env[MIDSCENE_LANGSMITH_DEBUG]) { console.log('DEBUGGING MODE: langsmith wrapper enabled'); @@ -105,5 +113,5 @@ export async function callToGetJSONObject( const response = await call(messages, responseFormat); assert(response, 'empty response'); - return JSON.parse(response); + return JSON.parse(response.replace(/^```json\n|\n```$/g, '')); } diff --git a/packages/midscene/vitest.config.ts b/packages/midscene/vitest.config.ts index fbfac7f66..8331add59 100644 --- a/packages/midscene/vitest.config.ts +++ b/packages/midscene/vitest.config.ts @@ -16,7 +16,9 @@ const basicTest = ['tests/unit-test/**/*.test.ts']; export default defineConfig({ test: { - include: enableAiTest ? ['tests/ai/**/*.test.ts', ...basicTest] : basicTest, + include: enableAiTest + ? ['tests/ai/inspector/todo_inspector.test.ts', ...basicTest] + : basicTest, }, resolve: { alias: { diff --git a/packages/web-integration/src/common/tasks.ts b/packages/web-integration/src/common/tasks.ts index f227e3f46..43a940c5b 100644 --- a/packages/web-integration/src/common/tasks.ts +++ b/packages/web-integration/src/common/tasks.ts @@ -158,7 +158,7 @@ export class PageTaskExecutor { dump: insightDump, }, cache: { - hit: Boolean(locateResult), + hit: Boolean(locateCache), }, }; }, diff --git a/packages/web-integration/tests/ai/native/appium/ios.test.ts b/packages/web-integration/tests/ai/native/appium/ios.test.ts index c10d073ff..b1e53981a 100644 --- a/packages/web-integration/tests/ai/native/appium/ios.test.ts +++ b/packages/web-integration/tests/ai/native/appium/ios.test.ts @@ -11,39 +11,27 @@ const IOS_DEFAULT_OPTIONS = { capabilities: { platformName: 'iOS', 'appium:automationName': 'XCUITest', - 'appium:deviceName': 'iPhone 15 Pro Simulator (17.5)', - 'appium:platformVersion': '17.5', - // 'appium:bundleId': 'com.apple.Preferences', - 'appium:bundleId': 'com.ss.iphone.ugc.AwemeInhouse', - 'appium:udid': '9ADCE031-36DF-4025-8C62-073FC7FAB901', - 'appium:newCommandTimeout': 600, + 'appium:deviceName': 'iPhone 15 Plus Simulator (18.0)', + 'appium:platformVersion': '18.0', + 'appium:bundleId': 'com.apple.Preferences', + 'appium:udid': 'B8517A53-6C4C-41D8-9B1E-825A0D75FA47', }, }; describe( 'appium integration', - async () => { - await it('iOS settings page demo for input', async () => { + () => { + it('iOS settings page demo', async () => { const page = await launchPage(IOS_DEFAULT_OPTIONS); const mid = new AppiumAgent(page); - await mid.aiAction('点击同意按钮'); - await mid.aiAction('点击底部朋友'); - // await mid.aiAction('输入框中输入“123”'); - // await mid.aiAction('输入框中输入“456”'); - // await mid.aiAction('输入框中输入“789”'); + await mid.aiAction('滑动列表到底部'); + await mid.aiAction('打开"开发者"'); + await mid.aiAction('滑动列表到底部'); + await mid.aiAction('滑动列表到顶部'); + await mid.aiAction('向下滑动一屏'); + await mid.aiAction('向上滑动一屏'); }); - // await it('iOS settings page demo for scroll', async () => { - // const page = await launchPage(IOS_DEFAULT_OPTIONS); - // const mid = new AppiumAgent(page); - - // await mid.aiAction('滑动列表到底部'); - // await mid.aiAction('打开"开发者"'); - // await mid.aiAction('滑动列表到底部'); - // await mid.aiAction('滑动列表到顶部'); - // await mid.aiAction('向下滑动一屏'); - // await mid.aiAction('向上滑动一屏'); - // }); }, { timeout: 360 * 1000, diff --git a/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts b/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts index 9675839d6..32eb19f5d 100644 --- a/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts @@ -39,9 +39,9 @@ test('ai todo', async ({ ai, aiQuery }) => { expect(taskList.length).toBe(1); expect(taskList[0]).toBe('Learning AI the day after tomorrow'); - const placeholder = await ai( - 'string, return the placeholder text in the input box', - { type: 'query' }, - ); - expect(placeholder).toBe('What needs to be done?'); + // const placeholder = await ai( + // 'string, return the placeholder text in the input box', + // { type: 'query' }, + // ); + // expect(placeholder).toBe('What needs to be done?'); }); diff --git a/packages/web-integration/tests/ai/web/playwright/ai-online-order.spec.ts b/packages/web-integration/tests/ai/web/playwright/ai-online-order.spec.ts index 7402f2cbb..69f025f46 100644 --- a/packages/web-integration/tests/ai/web/playwright/ai-online-order.spec.ts +++ b/packages/web-integration/tests/ai/web/playwright/ai-online-order.spec.ts @@ -11,14 +11,14 @@ test.beforeEach(async ({ page }) => { test('ai online order', async ({ ai, page, aiQuery }) => { await ai('点击左上角语言切换按钮(英文、中文)，在弹出的下拉列表中点击中文'); - await ai('向下滚动一屏'); + await ai('向下滚动两屏'); await ai('直接点击多肉葡萄的规格按钮'); await ai('点击不使用吸管、点击冰沙推荐、点击正常冰推荐'); await ai('向下滚动一屏'); await ai('点击标准甜、点击绿妍（推荐）、点击标准口味'); await ai('滚动到最下面'); await ai('点击页面下边的“选好了”按钮'); - await ai('点击右上角商品图标按钮'); + await ai('点击右上角商品图标按钮(仅商品按钮)'); const cardDetail = await aiQuery({ productName: '商品名称，在价格上面', diff --git a/packages/web-integration/tests/ai/web/playwright/util.ts b/packages/web-integration/tests/ai/web/playwright/util.ts index a3ee3685b..aa981da8e 100644 --- a/packages/web-integration/tests/ai/web/playwright/util.ts +++ b/packages/web-integration/tests/ai/web/playwright/util.ts @@ -21,17 +21,16 @@ export function getLastModifiedReportHTMLFile(dirPath: string) { ) { // Read the file content const content = fs.readFileSync(filePath, 'utf8'); - // Check if the content includes 'todo report' if ( + stats.mtimeMs > latestMtime && content.includes( - '"groupDescription":"tests/ai/e2e/ai-auto-todo.spec.ts"', + '"groupDescription":"tests/ai/web/playwright/ai-auto-todo.spec.ts"', ) ) { - if (stats.mtimeMs > latestMtime) { - latestMtime = stats.mtimeMs; - latestFile = filePath; - // console.log('filePath', filePath); - } + // Check if the content includes 'todo report' + latestMtime = stats.mtimeMs; + latestFile = filePath; + // console.log('filePath', filePath); } } });