diff --git a/README.md b/README.md index 82e92b531..10751831b 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ Midscene.js

-

Midscene.js

diff --git a/apps/site/docs/en/bridge-mode-by-chrome-extension.mdx b/apps/site/docs/en/bridge-mode-by-chrome-extension.mdx new file mode 100644 index 000000000..fa4f49d02 --- /dev/null +++ b/apps/site/docs/en/bridge-mode-by-chrome-extension.mdx @@ -0,0 +1,94 @@ +# Bridge Mode by Chrome Extension + +import { PackageManagerTabs } from '@theme'; + +The bridge mode in the Midscene Chrome extension is a tool that allows you to use local scripts to control the desktop version of Chrome. Your scripts can connect to either a new tab or the currently active tab. + +Using the desktop version of Chrome allows you to reuse all cookies, plugins, page status, and everything else you want. You can work with automation scripts to complete your tasks. This mode is commonly referred to as 'man-in-the-loop' in the context of automation. + +![bridge mode](/midscene-bridge-mode.jpg) + +:::info Demo Project +you can check the demo project of bridge mode here: [https://github.com/web-infra-dev/midscene-example/blob/main/bridge-mode-demo](https://github.com/web-infra-dev/midscene-example/blob/main/bridge-mode-demo) +::: + +## Preparation + +Install [Midscene extension from Chrome web store](https://chromewebstore.google.com/detail/midscene/gbldofcpkknbggpkmbdaefngejllnief). We will use it later. + +## Step 1. install dependencies + + + +## Step 2. write scripts + +Write and save the following code as `./demo-new-tab.ts`. + +```typescript +import { AgentOverChromeBridge } from "@midscene/web/bridge-mode"; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +Promise.resolve( + (async () => { + const agent = new AgentOverChromeBridge(); + + // This will connect to a new tab on your desktop Chrome + // remember to start your chrome extension, click 'allow connection' button. Otherwise you will get an timeout error + await agent.connectNewTabWithUrl("https://www.bing.com"); + + // these are the same as normal Midscene agent + await agent.ai('type "AI 101" and hit Enter'); + await sleep(3000); + + await agent.aiAssert("there are some search results"); + await agent.destroy(); + })() +); +``` + +## Step 3. run + +Launch your desktop Chrome. Start Midscene extension and switch to 'Bridge Mode' tab. Click "Allow connection". + +Run your scripts + +```bash +tsx demo-new-tab.ts +``` + +After executing the script, you should see the status of the Chrome extension switched to 'connected', and a new tab has been opened. Now this tab is controlled by your scripts. + +:::info +⁠Whether the scripts are run before or after clicking 'Allow connection' in the browser is not significant. +::: + +## API + +Except [the normal agent interface](./api), `AgentOverChromeBridge` provides some other interfaces to control the desktop Chrome. + +:::info +You should always call `connectCurrentTab` or `connectNewTabWithUrl` before doing further actions. + +Each of the agent instance can only connect to one tab instance, and it cannot be reconnected after destroy. +::: + +### `connectCurrentTab` + +Connect to the current active tab on Chrome. + +### `connectNewTabWithUrl(ur: string)` + +Create a new tab with url and connect to immediately. + +### `destroy` + +Destroy the connection. + +## Use bridge mode in yaml-script + +We are still building this, and it will be ready soon. + + + + + diff --git a/apps/site/docs/en/index.mdx b/apps/site/docs/en/index.mdx index 8e43ce6e8..3bf01d4f4 100644 --- a/apps/site/docs/en/index.mdx +++ b/apps/site/docs/en/index.mdx @@ -46,11 +46,12 @@ await aiAssert("There is a category filter on the left"); ## Multiple ways to integrate -To start experiencing the core feature of Midscene, we recommend you use [The Chrome Extension](./quick-experience). You can call Action / Query / Assert by natural language on any webpage, without needing to set up a code project. +To start experiencing the core feature of Midscene, we recommend you use [the Chrome Extension](./quick-experience). You can call Action / Query / Assert by natural language on any webpage, without needing to set up a code project. Also, there are several ways to integrate Midscene into your code project: -* [Automate with Scripts in YAML](./automate-with-scripts-in-yaml) +* [Automate with Scripts in YAML](./automate-with-scripts-in-yaml), use this if you prefer to write YAML file instead of code +* [Bridge Mode by Chrome Extension](./bridge-mode-by-chrome-extension), use this to control the desktop Chrome by scripts * [Integrate with Puppeteer](./integrate-with-puppeteer) * [Integrate with Playwright](./integrate-with-playwright) diff --git a/apps/site/docs/en/model-provider.md b/apps/site/docs/en/model-provider.md index 764568c74..52457391f 100644 --- a/apps/site/docs/en/model-provider.md +++ b/apps/site/docs/en/model-provider.md @@ -38,7 +38,9 @@ export OPENAI_MAX_TOKENS=2048 Use ADT token provider ```bash +# this is always true when using Azure OpenAI Service export MIDSCENE_USE_AZURE_OPENAI=1 + export MIDSCENE_AZURE_OPENAI_SCOPE="https://cognitiveservices.azure.com/.default" export AZURE_OPENAI_ENDPOINT="..." export AZURE_OPENAI_API_VERSION="2024-05-01-preview" @@ -110,6 +112,15 @@ export OPENAI_API_KEY="..." export MIDSCENE_MODEL_NAME="ep-202....." ``` +## Example: config request headers (like for openrouter) + +```bash +export OPENAI_BASE_URL="https://openrouter.ai/api/v1" +export OPENAI_API_KEY="..." +export MIDSCENE_MODEL_NAME="..." +export MIDSCENE_OPENAI_INIT_CONFIG_JSON='{"defaultHeaders":{"HTTP-Referer":"...","X-Title":"..."}}' +``` + ## Troubleshooting LLM Service Connectivity Issues If you want to troubleshoot connectivity issues, you can use the 'connectivity-test' folder in our example project: [https://github.com/web-infra-dev/midscene-example/tree/main/connectivity-test](https://github.com/web-infra-dev/midscene-example/tree/main/connectivity-test) diff --git a/apps/site/docs/public/midscene-bridge-mode.jpg b/apps/site/docs/public/midscene-bridge-mode.jpg new file mode 100644 index 000000000..d90d91206 Binary files /dev/null and b/apps/site/docs/public/midscene-bridge-mode.jpg differ diff --git a/apps/site/docs/zh/bridge-mode-by-chrome-extension.mdx b/apps/site/docs/zh/bridge-mode-by-chrome-extension.mdx new file mode 100644 index 000000000..923cfd524 --- /dev/null +++ b/apps/site/docs/zh/bridge-mode-by-chrome-extension.mdx @@ -0,0 +1,94 @@ +# 使用 Chrome 插件的桥接模式(Bridge Mode) + +import { PackageManagerTabs } from '@theme'; + +使用 Midscene 的 Chrome 插件的桥接模式,你可以用本地脚本控制桌面版本的 Chrome。你的脚本可以连接到新标签页或当前已激活的标签页。 + +使用桌面版本的 Chrome 可以让你复用已有的 cookie、插件、页面状态等。你可以使用自动化脚本与操作者互动,来完成你的任务。 + +![bridge mode](/midscene-bridge-mode.jpg) + +:::info Demo Project +you can check the demo project of bridge mode here: [https://github.com/web-infra-dev/midscene-example/blob/main/bridge-mode-demo](https://github.com/web-infra-dev/midscene-example/blob/main/bridge-mode-demo) +::: + +## 准备工作 + +安装 [Midscene 插件](https://chromewebstore.google.com/detail/midscene/gbldofcpkknbggpkmbdaefngejllnief)。 + +## 第一步:安装依赖 + + + +## 第二步:编写脚本 + +编写并保存以下代码为 `./demo-new-tab.ts`。 + +```typescript +import { AgentOverChromeBridge } from "@midscene/web/bridge-mode"; + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +Promise.resolve( + (async () => { + const agent = new AgentOverChromeBridge(); + + // 这个方法将连接到你的桌面 Chrome 的新标签页 + // 记得启动你的 Chrome 插件,并点击 'allow connection' 按钮。否则你会得到一个 timeout 错误 + await agent.connectNewTabWithUrl("https://www.bing.com"); + + // 这些方法与普通 Midscene agent 相同 + await agent.ai('type "AI 101" and hit Enter'); + await sleep(3000); + + await agent.aiAssert("there are some search results"); + await agent.destroy(); + })() +); +``` + +## 第三步:运行脚本 + +启动你的桌面 Chrome。启动 Midscene 插件,并切换到 'Bridge Mode' 标签页。点击 "Allow connection"。 + +运行你的脚本 + +```bash +tsx demo-new-tab.ts +``` + +执行脚本后,你应该看到 Chrome 插件的状态展示切换为 'connected',并且新标签页已打开。现在这个标签页由你的脚本控制。 + +:::info +执行脚本和点击插件中的 'Allow connection' 按钮没有顺序要求。 +::: + +## API + +除了 [普通的 agent 接口](./api),`AgentOverChromeBridge` 还提供了一些额外的接口来控制桌面 Chrome。 + +:::info +你应该在执行其他操作前,先调用 `connectCurrentTab` 或 `connectNewTabWithUrl`。 + +每个 agent 实例只能连接到一个标签页实例,并且一旦被销毁,就无法重新连接。 +::: + +### `connectCurrentTab` + +连接到当前已激活的标签页。 + +### `connectNewTabWithUrl(ur: string)` + +创建一个新标签页,并立即连接到它。 + +### `destroy` + +销毁连接。 + +## 在 YAML 脚本中使用桥接模式 + +这个功能正在开发中,很快就会与你见面。 + + + + + diff --git a/apps/site/docs/zh/index.mdx b/apps/site/docs/zh/index.mdx index 05421f184..b2bf44223 100644 --- a/apps/site/docs/zh/index.mdx +++ b/apps/site/docs/zh/index.mdx @@ -37,7 +37,8 @@ console.log("headphones in stock", items); 此外,还有几种形式将 Midscene 集成到代码: -* [使用 YAML 格式的自动化脚本](./automate-with-scripts-in-yaml) +* [使用 YAML 格式的自动化脚本](./automate-with-scripts-in-yaml),如果你更喜欢写 YAML 文件而不是代码 +* [使用 Chrome 插件的桥接模式](./bridge-mode-by-chrome-extension),用它来通过脚本控制桌面 Chrome * [集成到 Puppeteer](./integrate-with-puppeteer) * [集成到 Playwright](./integrate-with-playwright) diff --git a/apps/site/docs/zh/model-provider.md b/apps/site/docs/zh/model-provider.md index a3fe840db..604a748c7 100644 --- a/apps/site/docs/zh/model-provider.md +++ b/apps/site/docs/zh/model-provider.md @@ -35,7 +35,9 @@ export OPENAI_MAX_TOKENS=2048 使用 ADT token provider ```bash +# 使用 Azure OpenAI 服务时,配置为 1 export MIDSCENE_USE_AZURE_OPENAI=1 + export MIDSCENE_AZURE_OPENAI_SCOPE="https://cognitiveservices.azure.com/.default" export AZURE_OPENAI_ENDPOINT="..." export AZURE_OPENAI_API_VERSION="2024-05-01-preview" diff --git a/apps/site/rspress.config.ts b/apps/site/rspress.config.ts index d9ba15336..95639e15c 100644 --- a/apps/site/rspress.config.ts +++ b/apps/site/rspress.config.ts @@ -26,16 +26,6 @@ export default defineConfig({ 'https://applink.larkoffice.com/client/chat/chatter/add_by_link?link_token=291q2b25-e913-411a-8c51-191e59aab14d', }, ], - // footer: { - // message: ` - //
- // - //
- // `, - // }, locales: [ { lang: 'en', @@ -70,6 +60,10 @@ export default defineConfig({ text: 'Automate with Scripts in YAML', link: '/automate-with-scripts-in-yaml', }, + { + text: 'Bridge Mode by Chrome Extension', + link: '/bridge-mode-by-chrome-extension', + }, { text: 'Integrate with Playwright', link: '/integrate-with-playwright', @@ -127,6 +121,10 @@ export default defineConfig({ text: '使用 YAML 格式的自动化脚本', link: '/zh/automate-with-scripts-in-yaml', }, + { + text: '使用 Chrome 插件的桥接模式(Bridge Mode)', + link: '/zh/bridge-mode-by-chrome-extension', + }, { text: '集成到 Playwright', link: '/zh/integrate-with-playwright', diff --git a/nx.json b/nx.json index 9c805f64a..e94705fe6 100644 --- a/nx.json +++ b/nx.json @@ -5,14 +5,12 @@ "dependsOn": ["^build"] }, "build": { - "dependsOn": ["^build"], - "cache": true + "dependsOn": ["^build"] }, "build:watch": { "dependsOn": ["^build"] }, "test": { - "dependsOn": ["^build"], "cache": false }, "e2e": { diff --git a/packages/midscene/package.json b/packages/midscene/package.json index f630e3fcd..370e0f904 100644 --- a/packages/midscene/package.json +++ b/packages/midscene/package.json @@ -38,20 +38,18 @@ "prepublishOnly": "npm run build" }, "dependencies": { - "@anthropic-ai/sdk": "0.33.1", "@azure/identity": "4.5.0", - "@langchain/core": "0.3.26", + "@anthropic-ai/sdk": "0.33.1", "@midscene/shared": "workspace:*", - "dirty-json": "0.9.2", - "langchain": "0.3.8", - "openai": "4.57.1", - "optional": "0.1.4", - "socks-proxy-agent": "8.0.4" + "@langchain/core": "0.3.26", + "socks-proxy-agent": "8.0.4", + "openai": "4.57.1" }, "devDependencies": { "@modern-js/module-tools": "2.60.6", "@types/node": "^18.0.0", "@types/node-fetch": "2.6.11", + "dirty-json": "0.9.2", "dotenv": "16.4.5", "langsmith": "0.1.36", "typescript": "~5.0.4", diff --git a/packages/midscene/src/action/executor.ts b/packages/midscene/src/action/executor.ts index 1b74aed55..9362dc7a9 100644 --- a/packages/midscene/src/action/executor.ts +++ b/packages/midscene/src/action/executor.ts @@ -143,7 +143,8 @@ export class Executor { taskIndex++; } catch (e: any) { successfullyCompleted = false; - task.error = e?.message || 'error-without-message'; + task.error = + e?.message || (typeof e === 'string' ? e : 'error-without-message'); task.errorStack = e.stack; task.status = 'failed'; diff --git a/packages/midscene/src/ai-model/inspect.ts b/packages/midscene/src/ai-model/inspect.ts index ce7fd68be..b5ac2c59c 100644 --- a/packages/midscene/src/ai-model/inspect.ts +++ b/packages/midscene/src/ai-model/inspect.ts @@ -161,6 +161,7 @@ export async function AiInspectElement< type: 'image_url', image_url: { url: screenshotBase64WithElementMarker || screenshotBase64, + detail: 'high', }, }, { @@ -228,6 +229,7 @@ export async function AiExtractElementInfo< type: 'image_url', image_url: { url: screenshotBase64, + detail: 'high', }, }, { @@ -251,10 +253,7 @@ export async function AiExtractElementInfo< export async function AiAssert< ElementType extends BaseElement = BaseElement, ->(options: { - assertion: string; - context: UIContext; -}) { +>(options: { assertion: string; context: UIContext }) { const { assertion, context } = options; assert(assertion, 'assertion should be a string'); @@ -272,6 +271,7 @@ export async function AiAssert< type: 'image_url', image_url: { url: screenshotBase64, + detail: 'high', }, }, { diff --git a/packages/midscene/src/ai-model/openai/index.ts b/packages/midscene/src/ai-model/openai/index.ts index 69ffd7c80..af5a8ed98 100644 --- a/packages/midscene/src/ai-model/openai/index.ts +++ b/packages/midscene/src/ai-model/openai/index.ts @@ -118,6 +118,7 @@ async function createChatClient({ endpoint: getAIConfig(AZURE_OPENAI_ENDPOINT), apiVersion: getAIConfig(AZURE_OPENAI_API_VERSION), deployment: getAIConfig(AZURE_OPENAI_DEPLOYMENT), + dangerouslyAllowBrowser: true, ...extraConfig, ...extraAzureConfig, }); diff --git a/packages/midscene/src/insight/index.ts b/packages/midscene/src/insight/index.ts index 495199455..9c3d9c175 100644 --- a/packages/midscene/src/insight/index.ts +++ b/packages/midscene/src/insight/index.ts @@ -208,7 +208,7 @@ export default class Insight< let errorLog: string | undefined; if (parseResult.errors?.length) { - errorLog = `segment - AI response error: \n${parseResult.errors.join('\n')}`; + errorLog = `AI response error: \n${parseResult.errors.join('\n')}`; } const dumpData: PartialInsightDumpFromSDK = { @@ -225,12 +225,12 @@ export default class Insight< }; const logId = writeInsightDump(dumpData, undefined, dumpSubscriber); - if (errorLog) { + const { data } = parseResult; + if (errorLog && !data) { console.error(errorLog); throw new Error(errorLog); } - const { data } = parseResult; let mergedData = data; // expand elements in object style data diff --git a/packages/visualizer/modern.config.ts b/packages/visualizer/modern.config.ts index ac3188f8d..a96cccb79 100644 --- a/packages/visualizer/modern.config.ts +++ b/packages/visualizer/modern.config.ts @@ -2,7 +2,7 @@ import path from 'node:path'; import { defineConfig, moduleTools } from '@modern-js/module-tools'; import { modulePluginNodePolyfill } from '@modern-js/plugin-module-node-polyfill'; import { version } from './package.json'; -const externals = ['playwright']; +const externals = ['playwright', 'bufferutil', 'utf-8-validate']; const commonConfig = { asset: { @@ -17,8 +17,7 @@ const commonConfig = { } : undefined, define: { - __VERSION__: JSON.stringify(version), - global: 'globalThis', + __VERSION__: version, }, }; diff --git a/packages/visualizer/package.json b/packages/visualizer/package.json index e5cc32bb6..8cc0a3994 100644 --- a/packages/visualizer/package.json +++ b/packages/visualizer/package.json @@ -56,5 +56,8 @@ "sideEffects": ["**/*.css", "**/*.less", "**/*.sass", "**/*.scss"], "publishConfig": { "access": "public" + }, + "dependencies": { + "buffer": "6.0.3" } } diff --git a/packages/visualizer/src/component/common.less b/packages/visualizer/src/component/common.less index e1b64474f..7d2d02e4c 100644 --- a/packages/visualizer/src/component/common.less +++ b/packages/visualizer/src/component/common.less @@ -11,8 +11,9 @@ @selected-bg: #bfc4da80; @hover-bg: #dcdcdc80; -@weak-text: #777; @weak-bg: #F3F3F3; +@weak-text: #777; +@footer-text: #CCC; @toolbar-btn-bg: #E9E9E9; @@ -23,4 +24,4 @@ @layout-extension-space-horizontal: 20px; -@layout-extension-space-vertical: 30px; +@layout-extension-space-vertical: 20px; diff --git a/packages/visualizer/src/component/env-config.tsx b/packages/visualizer/src/component/env-config.tsx index 22a59249f..c43d9c33a 100644 --- a/packages/visualizer/src/component/env-config.tsx +++ b/packages/visualizer/src/component/env-config.tsx @@ -8,6 +8,7 @@ export function EnvConfig() { const [isModalOpen, setIsModalOpen] = useState(false); const [tempConfigString, setTempConfigString] = useState(configString); + const popupTab = useEnvConfig((state) => state.popupTab); const showModal = (e: React.MouseEvent) => { setIsModalOpen(true); e.preventDefault(); @@ -36,9 +37,9 @@ export function EnvConfig() { {iconForStatus('failed')} No config

+ ); + } else if (bridgeStatus === 'open-for-connection') { + statusElement = ( + + } size="small" /> + {' '} + + Listening for connection... + + + ); + statusBtn = ; + } else if (bridgeStatus === 'connected') { + statusElement = ( + + {iconForStatus('connected')} + {' '} + Connected + + - {bridgeAgentStatus} + + + ); + statusBtn = ; + } else { + statusElement = Unknown Status - {bridgeStatus}; + statusBtn = ; + } + + const logs = [...bridgeLog].reverse().map((log, index) => { + return ( +

+
+ {log.time} - {log.content} +
+
+ ); + }); + + return ( +
+

+ In Bridge Mode, you can control this browser by the Midscene SDK running + in the local terminal. This is useful for interacting both through + scripts and manually, or to reuse cookies.{' '} + + More about bridge mode + +

+ +
+
+

Bridge Status

+
+
{statusElement}
+
{statusBtn}
+
+
+
+

+ Bridge Log{' '} + +

+
{logs}
+
+
+
+ ); +} diff --git a/packages/visualizer/src/extension/playground-entry.tsx b/packages/visualizer/src/extension/playground-entry.tsx index 260823c10..6ee9f71b6 100644 --- a/packages/visualizer/src/extension/playground-entry.tsx +++ b/packages/visualizer/src/extension/playground-entry.tsx @@ -6,12 +6,16 @@ import { useEffect, useMemo, useState } from 'react'; import ReactDOM from 'react-dom/client'; import { globalThemeConfig } from '../component/color'; import { StaticPlayground } from '../component/playground-component'; +import { setSideEffect } from '../init'; import type { WorkerResponseGetContext } from './utils'; import { sendToWorker } from './utils'; import type { WorkerRequestGetContext } from './utils'; import { workerMessageTypes } from './utils'; + import './playground-entry.less'; +setSideEffect(); + const PlaygroundEntry = () => { // extension proxy agent const query = useMemo( diff --git a/packages/visualizer/src/extension/popup.less b/packages/visualizer/src/extension/popup.less index e9bd32f04..fa4c8b825 100644 --- a/packages/visualizer/src/extension/popup.less +++ b/packages/visualizer/src/extension/popup.less @@ -7,6 +7,7 @@ body { font-size: 14px; } +@footer-height: 40px; .popup-wrapper { width: 100%; height: 100%; @@ -16,6 +17,15 @@ body { display: flex; flex-direction: column; + .tabs-container { + flex-grow: 1; + } + + .ant-tabs-nav { + padding: 0 @layout-extension-space-horizontal; + box-sizing: border-box; + } + .popup-header { padding: 0 @layout-extension-space-horizontal; } @@ -28,12 +38,18 @@ body { box-sizing: border-box; } - .popup-playground-container { + .popup-playground-container, + .popup-bridge-container { flex-grow: 1; } + .popup-bridge-container { + padding: 0 @layout-extension-space-horizontal; + box-sizing: border-box; + } + .popup-footer { - color: @weak-text; + color: @footer-text; text-align: center; width: 100%; } diff --git a/packages/visualizer/src/extension/popup.tsx b/packages/visualizer/src/extension/popup.tsx index 8450be43a..e0c5cf745 100644 --- a/packages/visualizer/src/extension/popup.tsx +++ b/packages/visualizer/src/extension/popup.tsx @@ -1,6 +1,7 @@ -/// -import { Button, ConfigProvider, message } from 'antd'; +import { Button, ConfigProvider, Tabs, message } from 'antd'; import ReactDOM from 'react-dom/client'; +import { setSideEffect } from '../init'; +/// import './popup.less'; import { @@ -19,9 +20,15 @@ import { extensionAgentForTabId, } from '@/component/playground-component'; import { useChromeTabInfo } from '@/component/store'; -import { SendOutlined } from '@ant-design/icons'; +import { useEnvConfig } from '@/component/store'; +import { ApiOutlined, SendOutlined } from '@ant-design/icons'; import type { ChromeExtensionProxyPageAgent } from '@midscene/web/chrome-extension'; import { useEffect, useState } from 'react'; +import Bridge from './bridge'; + +setSideEffect(); + +declare const __VERSION__: string; const shotAndOpenPlayground = async ( agent?: ChromeExtensionProxyPageAgent | null, @@ -46,10 +53,26 @@ const shotAndOpenPlayground = async ( }); }; +// { +// /*

+// To keep the current page context, you can also{' '} +// +//

*/ +// } + function PlaygroundPopup() { const [loading, setLoading] = useState(false); const extensionVersion = getExtensionVersion(); const { tabId, windowId } = useChromeTabInfo(); + const { popupTab, setPopupTab } = useEnvConfig(); const handleSendToPlayground = async () => { if (!tabId || !windowId) { @@ -58,7 +81,7 @@ function PlaygroundPopup() { } setLoading(true); try { - const agent = extensionAgentForTabId(tabId, windowId); + const agent = extensionAgentForTabId(tabId); await shotAndOpenPlayground(agent); await agent!.page.destroy(); } catch (e: any) { @@ -67,45 +90,63 @@ function PlaygroundPopup() { setLoading(false); }; - return ( - -
-
- -

- Midscene.js helps to automate browser actions, perform assertions, - and extract data in JSON format using natural language.{' '} - - Learn more - -

-

This is a panel for experimenting with Midscene.js.

-

- To keep the current page context, you can also{' '} - -

-
- -
+ const items = [ + { + key: 'playground', + label: 'Playground', + icon: , + children: (
{ - return extensionAgentForTabId(tabId, windowId); + return extensionAgentForTabId(tabId); }} showContextPreview={false} />
+ ), + }, + { + key: 'bridge', + label: 'Bridge Mode', + children: ( +
+ +
+ ), + icon: , + }, + ]; + + return ( + +
+
+ +

+ Automate browser actions, extract data, and perform assertions using + AI, including a Chrome extension, JavaScript SDK, and support for + scripting in YAML.{' '} + + Learn more + +

+
+
+ setPopupTab(key as 'playground' | 'bridge')} + /> +
+
-

Midscene.js Chrome Extension v{extensionVersion}

+

+ Midscene.js Chrome Extension v{extensionVersion} (SDK v{__VERSION__} + ) +

diff --git a/packages/visualizer/src/index.tsx b/packages/visualizer/src/index.tsx index 670d90f8f..224fb4b8c 100644 --- a/packages/visualizer/src/index.tsx +++ b/packages/visualizer/src/index.tsx @@ -1,4 +1,6 @@ import './index.less'; +import { setSideEffect } from './init'; + import DetailSide from '@/component/detail-side'; import Sidebar from '@/component/sidebar'; import { useExecutionDump } from '@/component/store'; @@ -27,6 +29,8 @@ import { iconForStatus, timeCostStrElement } from './component/misc'; import Player from './component/player'; import Timeline from './component/timeline'; +setSideEffect(); + const { Dragger } = Upload; let globalRenderCount = 1; diff --git a/packages/visualizer/src/init.ts b/packages/visualizer/src/init.ts new file mode 100644 index 000000000..9bed4bc98 --- /dev/null +++ b/packages/visualizer/src/init.ts @@ -0,0 +1,14 @@ +// biome-ignore lint/style/useNodejsImportProtocol: +import { Buffer } from 'buffer'; + +// To solve the '"global is not defined" in randomBytes +// https://www.perplexity.ai/search/how-to-solve-global-is-not-def-xOrpDcfOSKqz_IXtwmK4_Q +window.global ||= window; +window.Buffer = Buffer; + +let sideEffect = 0; + +export const setSideEffect = () => { + sideEffect++; + return sideEffect; +}; diff --git a/packages/web-integration/modern.config.ts b/packages/web-integration/modern.config.ts index 8f4377de0..1ff409fed 100644 --- a/packages/web-integration/modern.config.ts +++ b/packages/web-integration/modern.config.ts @@ -1,4 +1,5 @@ import { defineConfig, moduleTools } from '@modern-js/module-tools'; +import { version } from './package.json'; export default defineConfig({ plugins: [moduleTools()], @@ -7,6 +8,8 @@ export default defineConfig({ format: 'cjs', input: { index: 'src/index.ts', + 'bridge-mode': 'src/bridge-mode/index.ts', + 'bridge-mode-browser': 'src/bridge-mode/browser.ts', utils: 'src/common/utils.ts', 'ui-utils': 'src/common/ui-utils.ts', debug: 'src/debug/index.ts', @@ -20,6 +23,15 @@ export default defineConfig({ yaml: 'src/yaml/index.ts', }, target: 'es2018', - externals: ['@midscene/core', '@midscene/shared', 'puppeteer'], + externals: [ + '@midscene/core', + '@midscene/shared', + 'puppeteer', + 'bufferutil', + 'utf-8-validate', + ], + define: { + __VERSION__: version, + }, }, }); diff --git a/packages/web-integration/package.json b/packages/web-integration/package.json index 9e29a8310..df9f2c435 100644 --- a/packages/web-integration/package.json +++ b/packages/web-integration/package.json @@ -11,22 +11,79 @@ "midscene-playground": "./bin/midscene-playground" }, "exports": { - ".": "./dist/lib/index.js", - "./utils": "./dist/lib/utils.js", - "./ui-utils": "./dist/lib/ui-utils.js", - "./puppeteer": "./dist/lib/puppeteer.js", - "./playwright": "./dist/lib/playwright.js", - "./playwright-report": "./dist/lib/playwright-report.js", - "./playground": "./dist/lib/playground.js", - "./debug": "./dist/lib/debug.js", - "./constants": "./dist/lib/constants.js", - "./html-element": "./dist/lib/html-element/index.js", - "./chrome-extension": "./dist/lib/chrome-extension.js", - "./yaml": "./dist/lib/yaml.js" + ".": { + "require": "./dist/lib/index.js", + "import": "./dist/es/index.js", + "types": "./dist/types/index.d.ts" + }, + "./bridge-mode": { + "require": "./dist/lib/bridge-mode.js", + "import": "./dist/es/bridge-mode.js", + "types": "./dist/types/bridge-mode.d.ts" + }, + "./bridge-mode-browser": { + "require": "./dist/lib/bridge-mode-browser.js", + "import": "./dist/es/bridge-mode-browser.js", + "types": "./dist/types/bridge-mode-browser.d.ts" + }, + "./utils": { + "require": "./dist/lib/utils.js", + "import": "./dist/es/utils.js", + "types": "./dist/types/utils.d.ts" + }, + "./ui-utils": { + "require": "./dist/lib/ui-utils.js", + "import": "./dist/es/ui-utils.js", + "types": "./dist/types/ui-utils.d.ts" + }, + "./puppeteer": { + "require": "./dist/lib/puppeteer.js", + "import": "./dist/es/puppeteer.js", + "types": "./dist/types/puppeteer.d.ts" + }, + "./playwright": { + "require": "./dist/lib/playwright.js", + "import": "./dist/es/playwright.js", + "types": "./dist/types/playwright.d.ts" + }, + "./playwright-report": { + "require": "./dist/lib/playwright-report.js", + "types": "./dist/types/playwright-report.d.ts" + }, + "./playground": { + "require": "./dist/lib/playground.js", + "import": "./dist/es/playground.js", + "types": "./dist/types/playground.d.ts" + }, + "./debug": { + "require": "./dist/lib/debug.js", + "types": "./dist/types/debug.d.ts" + }, + "./constants": { + "require": "./dist/lib/constants.js", + "import": "./dist/es/constants.js", + "types": "./dist/types/constants.d.ts" + }, + "./html-element": { + "require": "./dist/lib/html-element/index.js", + "types": "./dist/types/html-element/index.d.ts" + }, + "./chrome-extension": { + "require": "./dist/lib/chrome-extension.js", + "import": "./dist/es/chrome-extension.js", + "types": "./dist/types/chrome-extension.d.ts" + }, + "./yaml": { + "require": "./dist/lib/yaml.js", + "import": "./dist/es/yaml.js", + "types": "./dist/types/yaml.d.ts" + } }, "typesVersions": { "*": { ".": ["./dist/types/index.d.ts"], + "bridge-mode": ["./dist/types/bridge-mode.d.ts"], + "bridge-mode-browser": ["./dist/types/bridge-mode-browser.d.ts"], "utils": ["./dist/types/utils.d.ts"], "ui-utils": ["./dist/types/ui-utils.d.ts"], "puppeteer": ["./dist/types/puppeteer.d.ts"], @@ -71,23 +128,25 @@ "cors": "2.8.5", "express": "4.21.1", "inquirer": "10.1.5", - "openai": "4.57.1" + "openai": "4.57.1", + "socket.io": "4.8.1", + "socket.io-client": "4.8.1" }, "devDependencies": { - "@types/js-yaml": "4.0.9", - "js-yaml": "4.1.0", "@modern-js/module-tools": "2.60.6", "@playwright/test": "1.44.1", "@types/chrome": "0.0.279", "@types/cors": "2.8.12", "@types/express": "4.17.14", "@types/fs-extra": "11.0.4", + "@types/js-yaml": "4.0.9", "@types/node": "^18.0.0", "@wdio/types": "9.0.4", "devtools-protocol": "0.0.1380148", "dotenv": "16.4.5", "fs-extra": "11.2.0", "js-sha256": "0.11.0", + "js-yaml": "4.1.0", "playwright": "1.44.1", "puppeteer": "23.0.2", "typescript": "~5.0.4", diff --git a/packages/web-integration/src/bridge-mode/agent-cli-side.ts b/packages/web-integration/src/bridge-mode/agent-cli-side.ts new file mode 100644 index 000000000..c596b5ca2 --- /dev/null +++ b/packages/web-integration/src/bridge-mode/agent-cli-side.ts @@ -0,0 +1,119 @@ +import assert from 'node:assert'; +import { PageAgent } from '@/common/agent'; +import { paramStr, typeStr } from '@/common/ui-utils'; +import type { KeyboardAction, MouseAction } from '@/page'; +import { + BridgeEvent, + BridgePageType, + DefaultBridgeServerPort, + KeyboardEvent, + MouseEvent, +} from './common'; +import { BridgeServer } from './io-server'; +import type { ChromeExtensionPageBrowserSide } from './page-browser-side'; + +interface ChromeExtensionPageCliSide extends ChromeExtensionPageBrowserSide { + showStatusMessage: (message: string) => Promise; +} + +// actually, this is a proxy to the page in browser side +export const getBridgePageInCliSide = (): ChromeExtensionPageCliSide => { + const server = new BridgeServer(DefaultBridgeServerPort); + server.listen(); + const bridgeCaller = (method: string) => { + return async (...args: any[]) => { + const response = await server.call(method, args); + return response; + }; + }; + const page = { + showStatusMessage: async (message: string) => { + await server.call(BridgeEvent.UpdateAgentStatus, [message]); + }, + }; + + return new Proxy(page, { + get(target, prop, receiver) { + assert(typeof prop === 'string', 'prop must be a string'); + + if (prop === 'toJSON') { + return () => { + return { + pageType: BridgePageType, + }; + }; + } + + if (prop === 'pageType') { + return BridgePageType; + } + + if (prop === '_forceUsePageContext') { + return undefined; + } + + if (Object.keys(page).includes(prop)) { + return page[prop as keyof typeof page]; + } + + if (prop === 'mouse') { + const mouse: MouseAction = { + click: bridgeCaller(MouseEvent.Click), + wheel: bridgeCaller(MouseEvent.Wheel), + move: bridgeCaller(MouseEvent.Move), + }; + return mouse; + } + + if (prop === 'keyboard') { + const keyboard: KeyboardAction = { + type: bridgeCaller(KeyboardEvent.Type), + press: bridgeCaller(KeyboardEvent.Press), + }; + return keyboard; + } + + if (prop === 'destroy') { + return async () => { + try { + await bridgeCaller('destroy'); + } catch (e) { + console.error('error calling destroy', e); + } + return server.close(); + }; + } + + return bridgeCaller(prop); + }, + }) as ChromeExtensionPageCliSide; +}; + +export class AgentOverChromeBridge extends PageAgent { + constructor() { + const page = getBridgePageInCliSide(); + super(page, {}); + } + + async connectNewTabWithUrl(url: string) { + await this.page.connectNewTabWithUrl(url); + } + + async connectCurrentTab() { + await this.page.connectCurrentTab(); + } + + async aiAction(prompt: string, options?: any) { + if (options) { + console.warn( + 'the `options` parameter of aiAction is not supported in cli side', + ); + } + return await super.aiAction(prompt, { + onTaskStart: (task) => { + const tip = `${typeStr(task)} - ${paramStr(task)}`; + this.page.showStatusMessage(tip); + }, + }); + } +} diff --git a/packages/web-integration/src/bridge-mode/browser.ts b/packages/web-integration/src/bridge-mode/browser.ts new file mode 100644 index 000000000..1cd92cacb --- /dev/null +++ b/packages/web-integration/src/bridge-mode/browser.ts @@ -0,0 +1,3 @@ +import { ChromeExtensionPageBrowserSide } from '../bridge-mode/page-browser-side'; + +export { ChromeExtensionPageBrowserSide }; diff --git a/packages/web-integration/src/bridge-mode/common.ts b/packages/web-integration/src/bridge-mode/common.ts new file mode 100644 index 000000000..d6e383d09 --- /dev/null +++ b/packages/web-integration/src/bridge-mode/common.ts @@ -0,0 +1,57 @@ +export const DefaultBridgeServerPort = 3766; +export const DefaultLocalEndpoint = `http://127.0.0.1:${DefaultBridgeServerPort}`; +export const BridgeCallTimeout = 30000; + +export enum BridgeEvent { + Call = 'bridge-call', + CallResponse = 'bridge-call-response', + UpdateAgentStatus = 'bridge-update-agent-status', + Message = 'bridge-message', + Connected = 'bridge-connected', + Refused = 'bridge-refused', + ConnectNewTabWithUrl = 'connectNewTabWithUrl', + ConnectCurrentTab = 'connectCurrentTab', +} + +export enum MouseEvent { + PREFIX = 'mouse.', + Click = 'mouse.click', + Wheel = 'mouse.wheel', + Move = 'mouse.move', +} + +export enum KeyboardEvent { + PREFIX = 'keyboard.', + Type = 'keyboard.type', + Press = 'keyboard.press', +} + +export const BridgePageType = 'page-over-chrome-extension-bridge'; + +export const BridgeErrorCodeNoClientConnected = 'no-client-connected'; + +export interface BridgeCall { + method: string; + args: any[]; + response: any; + callTime: number; + responseTime: number; + callback: (error: Error | undefined, response: any) => void; + error?: Error; +} + +export interface BridgeCallRequest { + id: string; + method: string; + args: any[]; +} + +export interface BridgeCallResponse { + id: string; + response: any; + error?: any; +} + +export interface BridgeConnectedEventPayload { + version: string; +} diff --git a/packages/web-integration/src/bridge-mode/index.ts b/packages/web-integration/src/bridge-mode/index.ts new file mode 100644 index 000000000..07f2cf83d --- /dev/null +++ b/packages/web-integration/src/bridge-mode/index.ts @@ -0,0 +1,3 @@ +import { AgentOverChromeBridge } from './agent-cli-side'; + +export { AgentOverChromeBridge }; diff --git a/packages/web-integration/src/bridge-mode/io-client.ts b/packages/web-integration/src/bridge-mode/io-client.ts new file mode 100644 index 000000000..44a42a640 --- /dev/null +++ b/packages/web-integration/src/bridge-mode/io-client.ts @@ -0,0 +1,83 @@ +import assert from 'node:assert'; +import { io as ClientIO, type Socket as ClientSocket } from 'socket.io-client'; +import { + type BridgeCallRequest, + type BridgeCallResponse, + type BridgeConnectedEventPayload, + BridgeEvent, +} from './common'; + +declare const __VERSION__: string; + +// ws client, this is where the request is processed +export class BridgeClient { + private socket: ClientSocket | null = null; + public serverVersion: string | null = null; + constructor( + public endpoint: string, + public onBridgeCall: (method: string, args: any[]) => Promise, + public onDisconnect?: () => void, + ) {} + + async connect() { + return new Promise((resolve, reject) => { + this.socket = ClientIO(this.endpoint, { + reconnection: false, + query: { + version: __VERSION__, + }, + }); + + const timeout = setTimeout(() => { + reject(new Error('failed to connect to bridge server after timeout')); + }, 1 * 1000); + + // on disconnect + this.socket.on('disconnect', (reason: string) => { + // console.log('bridge-disconnected, reason:', reason); + this.socket = null; + this.onDisconnect?.(); + }); + + this.socket.on( + BridgeEvent.Connected, + (payload: BridgeConnectedEventPayload) => { + clearTimeout(timeout); + // console.log('bridge-connected'); + this.serverVersion = payload?.version || 'unknown'; + resolve(this.socket); + }, + ); + this.socket.on(BridgeEvent.Refused, (e: any) => { + console.error('bridge-refused', e); + reject(new Error(e || 'bridge refused')); + }); + this.socket.on(BridgeEvent.Call, (call: BridgeCallRequest) => { + const id = call.id; + assert(typeof id !== 'undefined', 'call id is required'); + Promise.resolve().then(async () => { + let response: any; + try { + response = await this.onBridgeCall(call.method, call.args); + } catch (e: any) { + const errorContent = `Error from bridge client when calling, method: ${call.method}, args: ${call.args}, error: ${e?.message || e}\n${e?.stack || ''}`; + console.error(errorContent); + return this.socket?.emit(BridgeEvent.CallResponse, { + id, + error: errorContent, + } as BridgeCallResponse); + } + this.socket?.emit(BridgeEvent.CallResponse, { + id, + response, + } as BridgeCallResponse); + }); + }); + }); + } + + disconnect() { + this.socket?.disconnect(); + this.socket = null; + } +} diff --git a/packages/web-integration/src/bridge-mode/io-server.ts b/packages/web-integration/src/bridge-mode/io-server.ts new file mode 100644 index 000000000..9e8c3aa58 --- /dev/null +++ b/packages/web-integration/src/bridge-mode/io-server.ts @@ -0,0 +1,220 @@ +import { Server, type Socket as ServerSocket } from 'socket.io'; +import { + type BridgeCall, + type BridgeCallResponse, + BridgeCallTimeout, + type BridgeConnectedEventPayload, + BridgeErrorCodeNoClientConnected, + BridgeEvent, +} from './common'; + +declare const __VERSION__: string; + +// ws server, this is where the request is sent +export class BridgeServer { + private callId = 0; + private io: Server | null = null; + private socket: ServerSocket | null = null; + private listeningTimeoutId: NodeJS.Timeout | null = null; + private connectionTipTimer: NodeJS.Timeout | null = null; + public calls: Record = {}; + + private connectionLost = false; + private connectionLostReason = ''; + + constructor( + public port: number, + public onConnect?: () => void, + public onDisconnect?: (reason: string) => void, + ) {} + + async listen(timeout = 30000): Promise { + return new Promise((resolve, reject) => { + if (this.listeningTimeoutId) { + return reject(new Error('already listening')); + } + + this.listeningTimeoutId = setTimeout(() => { + reject( + new Error( + `no extension connected after ${timeout}ms (${BridgeErrorCodeNoClientConnected})`, + ), + ); + }, timeout); + + this.connectionTipTimer = + timeout > 3000 + ? setTimeout(() => { + console.log('waiting for bridge to connect...'); + }, 2000) + : null; + + this.io = new Server(this.port, { + maxHttpBufferSize: 100 * 1024 * 1024, // 100MB + }); + this.io.on('connection', (socket) => { + this.connectionLost = false; + this.connectionLostReason = ''; + this.listeningTimeoutId && clearTimeout(this.listeningTimeoutId); + this.listeningTimeoutId = null; + this.connectionTipTimer && clearTimeout(this.connectionTipTimer); + this.connectionTipTimer = null; + if (this.socket) { + console.log('server already connected, refusing new connection'); + socket.emit(BridgeEvent.Refused); + reject(new Error('server already connected by another client')); + } + + try { + // console.log('one client connected'); + this.socket = socket; + + const clientVersion = socket.handshake.query.version; + console.log( + 'Bridge connected, cli-side version:', + __VERSION__, + ', browser-side version:', + clientVersion, + ); + + socket.on(BridgeEvent.CallResponse, (params: BridgeCallResponse) => { + const id = params.id; + const response = params.response; + const error = params.error; + + this.triggerCallResponseCallback(id, error, response); + }); + + socket.on('disconnect', (reason: string) => { + this.connectionLost = true; + this.connectionLostReason = reason; + this.onDisconnect?.(reason); + + // flush all pending calls as error + for (const id in this.calls) { + const call = this.calls[id]; + + if (!call.responseTime) { + const errorMessage = this.connectionLostErrorMsg(); + this.triggerCallResponseCallback( + id, + new Error(errorMessage), + null, + ); + } + } + }); + + setTimeout(() => { + this.onConnect?.(); + + const payload = { + version: __VERSION__, + } as BridgeConnectedEventPayload; + socket.emit(BridgeEvent.Connected, payload); + Promise.resolve().then(() => { + for (const id in this.calls) { + if (this.calls[id].callTime === 0) { + this.emitCall(id); + } + } + }); + }, 0); + + resolve(); + } catch (e) { + console.error('failed to handle connection event', e); + reject(e); + } + }); + }); + } + + private connectionLostErrorMsg = () => { + return `Connection lost, reason: ${this.connectionLostReason}`; + }; + + private async triggerCallResponseCallback( + id: string | number, + error: Error | null, + response: any, + ) { + const call = this.calls[id]; + if (!call) { + throw new Error(`call ${id} not found`); + } + call.error = error || undefined; + call.response = response; + call.responseTime = Date.now(); + + call.callback(call.error, response); + } + + private async emitCall(id: string) { + const call = this.calls[id]; + if (!call) { + throw new Error(`call ${id} not found`); + } + + if (this.connectionLost) { + const message = `Connection lost, reason: ${this.connectionLostReason}`; + call.callback(new Error(message), null); + return; + } + + if (this.socket) { + this.socket.emit(BridgeEvent.Call, { + id, + method: call.method, + args: call.args, + }); + call.callTime = Date.now(); + } + } + + async call( + method: string, + args: any[], + timeout = BridgeCallTimeout, + ): Promise { + const id = `${this.callId++}`; + + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + console.log( + `bridge call timeout, id=${id}, method=${method}, args=`, + args, + ); + this.calls[id].error = new Error( + `Bridge call timeout after ${timeout}ms: ${method}`, + ); + reject(this.calls[id].error); + }, timeout); + + this.calls[id] = { + method, + args, + response: null, + callTime: 0, + responseTime: 0, + callback: (error: Error | undefined, response: any) => { + clearTimeout(timeoutId); + if (error) { + reject(error); + } else { + resolve(response); + } + }, + }; + + this.emitCall(id); + }); + } + + close() { + this.listeningTimeoutId && clearTimeout(this.listeningTimeoutId); + this.connectionTipTimer && clearTimeout(this.connectionTipTimer); + this.io?.close(); + this.io = null; + } +} diff --git a/packages/web-integration/src/bridge-mode/page-browser-side.ts b/packages/web-integration/src/bridge-mode/page-browser-side.ts new file mode 100644 index 000000000..9bac4c859 --- /dev/null +++ b/packages/web-integration/src/bridge-mode/page-browser-side.ts @@ -0,0 +1,134 @@ +import assert from 'node:assert'; +import type { KeyboardAction, MouseAction } from '@/page'; +import ChromeExtensionProxyPage from '../chrome-extension/page'; +import { + BridgeEvent, + DefaultBridgeServerPort, + KeyboardEvent, + MouseEvent, +} from './common'; +import { BridgeClient } from './io-client'; + +declare const __VERSION__: string; + +export class ChromeExtensionPageBrowserSide extends ChromeExtensionProxyPage { + public bridgeClient: BridgeClient | null = null; + + constructor( + public onDisconnect: () => void = () => {}, + public onLogMessage: ( + message: string, + type: 'log' | 'status', + ) => void = () => {}, + ) { + super(0); + } + + private async setupBridgeClient() { + this.bridgeClient = new BridgeClient( + `ws://localhost:${DefaultBridgeServerPort}`, + async (method, args: any[]) => { + console.log('bridge call from cli side', method, args); + if (method === BridgeEvent.ConnectNewTabWithUrl) { + return this.connectNewTabWithUrl.apply( + this, + args as unknown as [string], + ); + } + + if (method === BridgeEvent.ConnectCurrentTab) { + return this.connectCurrentTab.apply(this, args as any); + } + + if (method === BridgeEvent.UpdateAgentStatus) { + return this.onLogMessage(args[0] as string, 'status'); + } + + if (!this.tabId || this.tabId === 0) { + throw new Error('no tab is connected'); + } + + // this.onLogMessage(`calling method: ${method}`); + + if (method.startsWith(MouseEvent.PREFIX)) { + const actionName = method.split('.')[1] as keyof MouseAction; + return this.mouse[actionName].apply(this.mouse, args as any); + } + + if (method.startsWith(KeyboardEvent.PREFIX)) { + const actionName = method.split('.')[1] as keyof KeyboardAction; + return this.keyboard[actionName].apply(this.keyboard, args as any); + } + + try { + // @ts-expect-error + const result = await this[method as keyof ChromeExtensionProxyPage]( + ...args, + ); + return result; + } catch (e) { + const errorMessage = e instanceof Error ? e.message : 'Unknown error'; + console.error('error calling method', method, args, e); + this.onLogMessage( + `Error calling method: ${method}, ${errorMessage}`, + 'log', + ); + throw new Error(errorMessage, { cause: e }); + } + }, + // on disconnect + () => { + return this.destroy(); + }, + ); + await this.bridgeClient.connect(); + this.onLogMessage( + `Bridge connected, cli-side version ${this.bridgeClient.serverVersion}, browser-side version: ${__VERSION__}`, + 'log', + ); + } + + public async connect() { + return await this.setupBridgeClient(); + } + + public async connectNewTabWithUrl(url: string) { + assert(url, 'url is required to create a new tab'); + if (this.tabId) { + throw new Error('tab is already connected'); + } + + const tab = await chrome.tabs.create({ url }); + const tabId = tab.id; + assert(tabId, 'failed to get tabId after creating a new tab'); + this.tabId = tabId; + + // new tab + this.onLogMessage(`Creating new tab: ${url}`, 'log'); + } + + public async connectCurrentTab() { + if (this.tabId) { + throw new Error( + `already connected with tab id ${this.tabId}, cannot reconnect`, + ); + } + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + console.log('current tab', tabs); + const tabId = tabs[0]?.id; + assert(tabId, 'failed to get tabId'); + this.tabId = tabId; + + this.onLogMessage(`Connected to current tab: ${tabs[0]?.url}`, 'log'); + } + + async destroy() { + if (this.bridgeClient) { + this.bridgeClient.disconnect(); + this.bridgeClient = null; + this.onDisconnect(); + } + super.destroy(); + this.tabId = 0; + } +} diff --git a/packages/web-integration/src/chrome-extension/page.ts b/packages/web-integration/src/chrome-extension/page.ts index ba73ddf06..85812c2ce 100644 --- a/packages/web-integration/src/chrome-extension/page.ts +++ b/packages/web-integration/src/chrome-extension/page.ts @@ -9,7 +9,7 @@ import fs from 'node:fs'; import type { WebKeyInput } from '@/common/page'; import type { ElementInfo } from '@/extractor'; import type { AbstractPage } from '@/page'; -import type { Point, Rect, Size } from '@midscene/core'; +import type { Point, Size } from '@midscene/core'; import { ifInBrowser } from '@midscene/shared/utils'; import type { Protocol as CDPTypes } from 'devtools-protocol'; import { CdpKeyboard } from './cdpInput'; @@ -27,39 +27,10 @@ const scriptFileContent = async () => { return fs.readFileSync(scriptFileToRetrieve, 'utf8'); }; -const lastTwoCallTime = [0, 0]; -const callInterval = 1050; -async function getScreenshotBase64FromWindowId(windowId: number) { - // check if this window is active - const activeWindow = await chrome.windows.getAll({ populate: true }); - if (activeWindow.find((w) => w.id === windowId) === undefined) { - throw new Error(`Window with id ${windowId} is not active`); - } - - // avoid MAX_CAPTURE_VISIBLE_TAB_CALLS_PER_SECOND - const now = Date.now(); - if (now - lastTwoCallTime[0] < callInterval) { - const sleepTime = callInterval - (now - lastTwoCallTime[0]); - console.warn( - `Sleep for ${sleepTime}ms to avoid too frequent screenshot calls`, - ); - await new Promise((resolve) => setTimeout(resolve, sleepTime)); - } - const base64 = await chrome.tabs.captureVisibleTab(windowId, { - format: 'jpeg', - quality: 70, - }); - lastTwoCallTime.shift(); - lastTwoCallTime.push(Date.now()); - return base64; -} - export default class ChromeExtensionProxyPage implements AbstractPage { pageType = 'chrome-extension-proxy'; - private tabId: number; - - private windowId: number; + public tabId: number; private viewportSize?: Size; @@ -67,9 +38,8 @@ export default class ChromeExtensionProxyPage implements AbstractPage { private attachingDebugger: Promise | null = null; - constructor(tabId: number, windowId: number) { + constructor(tabId: number) { this.tabId = tabId; - this.windowId = windowId; } private async attachDebugger() { @@ -152,30 +122,39 @@ export default class ChromeExtensionProxyPage implements AbstractPage { }); if (!returnValue.result.value) { - throw new Error('Failed to get page content from page'); + const errorDescription = + returnValue.exceptionDetails?.exception?.description || ''; + if (!errorDescription) { + console.error('returnValue from cdp', returnValue); + } + throw new Error( + `Failed to get page content from page, error: ${errorDescription}`, + ); } // console.log('returnValue', returnValue.result.value); return returnValue.result.value; } - // private async rectOfNodeId(nodeId: number): Promise { - // try { - // const { model } = - // await this.sendCommandToDebugger( - // 'DOM.getBoxModel', - // { nodeId }, - // ); - // return { - // left: model.border[0], - // top: model.border[1], - // width: model.border[2] - model.border[0], - // height: model.border[5] - model.border[1], - // }; - // } catch (error) { - // console.error('Error getting box model for nodeId', nodeId, error); - // return null; - // } - // } + // current implementation is wait until domReadyState is complete + public async waitUntilNetworkIdle() { + const timeout = 10000; + const startTime = Date.now(); + let lastReadyState = ''; + while (Date.now() - startTime < timeout) { + const result = await this.sendCommandToDebugger('Runtime.evaluate', { + expression: 'document.readyState', + }); + lastReadyState = result.result.value; + if (lastReadyState === 'complete') { + await new Promise((resolve) => setTimeout(resolve, 300)); + return; + } + await new Promise((resolve) => setTimeout(resolve, 300)); + } + throw new Error( + `Failed to wait until network idle, last readyState: ${lastReadyState}`, + ); + } async getElementInfos() { const content = await this.getPageContentByCDP(); diff --git a/packages/web-integration/src/common/agent.ts b/packages/web-integration/src/common/agent.ts index 2a9b5460d..d7a55c89b 100644 --- a/packages/web-integration/src/common/agent.ts +++ b/packages/web-integration/src/common/agent.ts @@ -34,8 +34,8 @@ export interface PageAgentOpt { autoPrintReportMsg?: boolean; } -export class PageAgent { - page: WebPage; +export class PageAgent { + page: PageType; insight: Insight; @@ -54,7 +54,7 @@ export class PageAgent { */ dryMode = false; - constructor(page: WebPage, opts?: PageAgentOpt) { + constructor(page: PageType, opts?: PageAgentOpt) { this.page = page; this.opts = Object.assign( { diff --git a/packages/web-integration/src/extractor/web-extractor.ts b/packages/web-integration/src/extractor/web-extractor.ts index 70f7fffa2..2a0a94fc3 100644 --- a/packages/web-integration/src/extractor/web-extractor.ts +++ b/packages/web-integration/src/extractor/web-extractor.ts @@ -265,6 +265,11 @@ export function extractTextWithPosition( return null; } + if (node.nodeType && node.nodeType === 10) { + // Doctype node + return null; + } + const elementInfo = collectElementInfo(node, nodePath, baseZoom); // stop collecting if the node is a Button or Image if ( @@ -291,7 +296,8 @@ export function extractTextWithPosition( return elementInfo; } - dfs(initNode || getDocument(), '0'); + const rootNode = initNode || getDocument(); + dfs(rootNode, '0'); if (currentFrame.left !== 0 || currentFrame.top !== 0) { for (let i = 0; i < elementInfoArray.length; i++) { diff --git a/packages/web-integration/src/page.ts b/packages/web-integration/src/page.ts index b4c3c2c34..e42a8f14a 100644 --- a/packages/web-integration/src/page.ts +++ b/packages/web-integration/src/page.ts @@ -5,6 +5,21 @@ import type { ElementInfo } from './extractor'; export type MouseButton = 'left' | 'right' | 'middle'; +export interface MouseAction { + click: ( + x: number, + y: number, + options: { button: MouseButton }, + ) => Promise; + wheel: (deltaX: number, deltaY: number) => Promise; + move: (x: number, y: number) => Promise; +} + +export interface KeyboardAction { + type: (text: string) => Promise; + press: (key: WebKeyInput) => Promise; +} + export abstract class AbstractPage { abstract pageType: string; abstract getElementInfos(): Promise; @@ -12,7 +27,7 @@ export abstract class AbstractPage { abstract screenshotBase64?(): Promise; abstract size(): Promise; - get mouse() { + get mouse(): MouseAction { return { click: async ( x: number, @@ -24,7 +39,7 @@ export abstract class AbstractPage { }; } - get keyboard() { + get keyboard(): KeyboardAction { return { type: async (text: string) => {}, press: async (key: WebKeyInput) => {}, diff --git a/packages/web-integration/tests/ai/bridge/agent.test.ts b/packages/web-integration/tests/ai/bridge/agent.test.ts new file mode 100644 index 000000000..81694a21b --- /dev/null +++ b/packages/web-integration/tests/ai/bridge/agent.test.ts @@ -0,0 +1,69 @@ +import { + AgentOverChromeBridge, + getBridgePageInCliSide, +} from '@/bridge-mode/agent-cli-side'; +import { describe, expect, it } from 'vitest'; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); +describe.skipIf(process.env.CI)( + 'fully functional agent in server(cli) side', + () => { + it('basic', async () => { + const page = getBridgePageInCliSide(); + expect(page).toBeDefined(); + + // server should be destroyed as well + await page.destroy(); + }); + + it( + 'page in cli side', + async () => { + const page = getBridgePageInCliSide(); + + // make sure the extension bridge is launched before timeout + await page.connectNewTabWithUrl('https://www.baidu.com'); + + // sleep 3s + await sleep(3000); + + await page.destroy(); + }, + 40 * 1000, // longer than the timeout of the bridge io + ); + + it( + 'agent in cli side, new tab', + async () => { + const agent = new AgentOverChromeBridge(); + + await agent.connectNewTabWithUrl('https://www.bing.com'); + await sleep(3000); + + await agent.ai('type "AI 101" and hit Enter'); + await sleep(3000); + + await agent.aiAssert('there are some search results'); + await agent.destroy(); + }, + 60 * 1000, + ); + + it( + 'agent in cli side, current tab', + async () => { + const agent = new AgentOverChromeBridge(); + await agent.connectCurrentTab(); + await sleep(3000); + const answer = await agent.aiQuery( + 'name of the current page? return {name: string}', + ); + + console.log(answer); + expect(answer.name).toBeTruthy(); + await agent.destroy(); + }, + 60 * 1000, + ); + }, +); diff --git a/packages/web-integration/tests/ai/web/playwright/todo-app-midscene.spec.ts b/packages/web-integration/tests/ai/web/playwright/todo-app-midscene.spec.ts deleted file mode 100644 index 4a966bacc..000000000 --- a/packages/web-integration/tests/ai/web/playwright/todo-app-midscene.spec.ts +++ /dev/null @@ -1,98 +0,0 @@ -// import { test, expect, type Page } from '@playwright/test'; -// import Insight, { TextElement, query } from 'midscene'; -// import { retrieveElements, retrieveOneElement } from 'midscene/query'; - -// test.beforeEach(async ({ page }) => { -// await page.goto('https://todomvc.com/examples/react/dist/'); -// }); - -// const TODO_ITEMS = ['buy some cheese', 'feed the cat', 'book a doctors appointment']; - -// interface InputBoxSection { -// element: TextElement; -// toggleAllBtn: TextElement; -// placeholder: string; -// inputValue: string; -// } - -// interface TodoItem { -// name: string; -// finished: boolean; -// } - -// interface ControlLayerSection { -// numbersLeft: number; -// tipElement: TextElement; -// controlElements: TextElement[]; -// } - -// // A comprehensive parser for page content -// const parsePage = async (page: Page) => { -// const insight = await Insight.fromPlaywrightPage(page); -// const todoListPage = await insight.segment({ -// 'input-box': query('an input box to type item and a "toggle-all" button', { -// element: retrieveOneElement('input box'), -// toggleAllBtn: retrieveOneElement('toggle all button, if exists'), -// placeholder: 'placeholder string in the input box, string, if exists', -// inputValue: 'the value in the input box, string, if exists', -// }), -// 'todo-list': query<{ todoItems: TodoItem[] }>('a list with todo-data (if exists)', { -// todoItems: '{name: string, finished: boolean}[]', -// }), -// 'control-layer': query('status and control layer of todo (if exists)', { -// numbersLeft: 'number', -// tipElement: retrieveOneElement( -// 'the element indicates the number of remaining items, like ` items left`', -// ), -// controlElements: retrieveElements('control elements, used to filter items'), -// }), -// }); - -// return todoListPage; -// }; - -// test.describe('New Todo', () => { -// test('should allow me to add todo items', async ({ page }) => { -// // add a todo item -// const todoPage = await parsePage(page); -// const inputBox = todoPage['input-box']; -// expect(inputBox).toBeTruthy(); - -// await page.mouse.click(...inputBox!.element.center); -// await page.keyboard.type(TODO_ITEMS[0], { delay: 100 }); -// await page.keyboard.press('Enter'); - -// // update page parsing result, and check the interface -// const todoPage2 = await parsePage(page); -// expect(todoPage2['input-box'].inputValue).toBeFalsy(); -// expect(todoPage2['input-box'].placeholder).toBeTruthy(); -// expect(todoPage2['todo-list'].todoItems.length).toBe(1); -// expect(todoPage2['todo-list'].todoItems[0].name).toBe(TODO_ITEMS[0]); - -// // add another item -// await page.mouse.click(...todoPage2['input-box'].element.center); -// await page.keyboard.type(TODO_ITEMS[1], { delay: 100 }); -// await page.keyboard.press('Enter'); - -// // update page parsing result -// const todoPage3 = await parsePage(page); -// const items = todoPage3['todo-list'].todoItems; -// expect(items.length).toBe(2); -// expect(items[1].name).toEqual(TODO_ITEMS[1]); -// expect(items.some((item) => item.finished)).toBeFalsy(); -// expect(todoPage3['control-layer'].numbersLeft).toBe(2); - -// // will mark all as completed -// const toggleBtn = todoPage3['input-box'].toggleAllBtn; -// expect(toggleBtn).toBeTruthy(); -// expect(todoPage3['todo-list'].todoItems.filter((item) => item.finished).length).toBe(0); - -// await page.mouse.click(...toggleBtn!.center, { delay: 500 }); -// await page.waitForTimeout(3000); - -// const todoPage4 = await parsePage(page); -// const allItems = todoPage4['todo-list'].todoItems; -// expect(allItems.length).toBe(2); -// expect(allItems.filter((item) => item.finished).length).toBe(allItems.length); -// }); -// }); diff --git a/packages/web-integration/tests/unit-test/bridge/io.test.ts b/packages/web-integration/tests/unit-test/bridge/io.test.ts new file mode 100644 index 000000000..285a78c71 --- /dev/null +++ b/packages/web-integration/tests/unit-test/bridge/io.test.ts @@ -0,0 +1,183 @@ +import { BridgeClient } from '@/bridge-mode/io-client'; +import { BridgeServer } from '@/bridge-mode/io-server'; +import { describe, expect, it, vi } from 'vitest'; + +let testPort = 1234; +describe('bridge-io', () => { + it('server launch and close', () => { + const server = new BridgeServer(testPort++); + server.listen(); + server.close(); + }); + + it('server already listening', async () => { + const port = testPort++; + const server = new BridgeServer(port); + server.listen(); + await expect(server.listen()).rejects.toThrow(); + server.close(); + }); + + it('refuse 2nd client connection', async () => { + const port = testPort++; + const server = new BridgeServer(port); + server.listen(); + const client = new BridgeClient( + `ws://localhost:${port}`, + (method, args) => { + return Promise.resolve('ok'); + }, + ); + await client.connect(); + + const client2 = new BridgeClient( + `ws://localhost:${port}`, + (method, args) => { + return Promise.resolve('ok'); + }, + ); + await expect(client2.connect()).rejects.toThrow(); + + server.close(); + client.disconnect(); + }); + + it('server listen timeout', async () => { + const server = new BridgeServer(testPort++); + await expect(server.listen(100)).rejects.toThrow(); + }); + + it('server and client communicate', async () => { + const port = testPort++; + const method = 'test'; + const args = ['a', 'b', { foo: 'bar' }]; + const responseValue = { hello: 'world' }; + + const server = new BridgeServer(port); + server.listen(); + const client = new BridgeClient( + `ws://localhost:${port}`, + (method, args) => { + expect(method).toBe(method); + expect(args).toEqual(args); + return Promise.resolve(responseValue); + }, + ); + await client.connect(); + + const response = await server.call(method, args); + expect(response).toEqual(responseValue); + + server.close(); + client.disconnect(); + }); + + it('client call error', async () => { + const port = testPort++; + const server = new BridgeServer(port); + const errMsg = 'internal error'; + server.listen(); + + const client = new BridgeClient( + `ws://localhost:${port}`, + (method, args) => { + return Promise.reject(new Error(errMsg)); + }, + ); + + await client.connect(); + // await server.call('test', ['a', 'b']); + expect(server.call('test', ['a', 'b'])).rejects.toThrow(errMsg); + }); + + it('client disconnect event', async () => { + const port = testPort++; + const server = new BridgeServer(port); + server.listen(); + + const fn = vi.fn(); + + const client = new BridgeClient( + `ws://localhost:${port}`, + (method, args) => { + return Promise.resolve('ok'); + }, + fn, + ); + + await client.connect(); + + await server.close(); + + // sleep 1s + await new Promise((resolve) => setTimeout(resolve, 1000)); + expect(fn).toHaveBeenCalled(); + }); + + it('client close before server', async () => { + const port = testPort++; + const onConnect = vi.fn(); + const onDisconnect = vi.fn(); + const server = new BridgeServer(port, onConnect, onDisconnect); + server.listen(); + + const client = new BridgeClient(`ws://localhost:${port}`, () => { + return Promise.resolve('ok'); + }); + await client.connect(); + + await new Promise((resolve) => setTimeout(resolve, 500)); + expect(onConnect).toHaveBeenCalled(); + + expect(onDisconnect).not.toHaveBeenCalled(); + await client.disconnect(); + await new Promise((resolve) => setTimeout(resolve, 500)); + expect(onDisconnect).toHaveBeenCalled(); + }); + + it('flush all calls before connecting', async () => { + const port = testPort++; + const server = new BridgeServer(port); + server.listen(); + + const client = new BridgeClient( + `ws://localhost:${port}`, + (method, args) => { + return Promise.resolve('ok'); + }, + ); + + const call = server.call('test', ['a', 'b']); + const call2 = server.call('test2', ['a', 'b']); + + await new Promise((resolve) => setTimeout(resolve, 100)); + await client.connect(); + const response = await call; + expect(response).toEqual('ok'); + + const response2 = await call2; + expect(response2).toEqual('ok'); + + server.close(); + client.disconnect(); + }); + + it('server timeout', async () => { + const port = testPort++; + const server = new BridgeServer(port); + server.listen(); + + const client = new BridgeClient( + `ws://localhost:${port}`, + (method, args) => { + throw new Error('internal error'); + }, + ); + await client.connect(); + + expect(server.call('test', ['a', 'b'], 1000)).rejects.toThrow(); + + server.close(); + client.disconnect(); + }); +}); diff --git a/packages/web-integration/vitest.config.ts b/packages/web-integration/vitest.config.ts index 78c765dd3..3f87439f6 100644 --- a/packages/web-integration/vitest.config.ts +++ b/packages/web-integration/vitest.config.ts @@ -2,6 +2,7 @@ import path from 'node:path'; //@ts-ignore import dotenv from 'dotenv'; import { defineConfig } from 'vitest/config'; +import { version } from './package.json'; /** * Read environment variables from file. @@ -13,7 +14,10 @@ dotenv.config({ const aiTestType = process.env.AI_TEST_TYPE; const unitTests = ['tests/unit-test/**/*.test.ts']; -const aiWebTests = ['tests/ai/web/**/*.test.ts']; +const aiWebTests = [ + 'tests/ai/web/**/*.test.ts', + 'tests/ai/bridge/**/*.test.ts', +]; const aiNativeTests = ['tests/ai/native/**/*.test.ts']; // const aiNativeTests = ['tests/ai/native/appium/dongchedi.test.ts']; const testFiles = (() => { @@ -36,4 +40,7 @@ export default defineConfig({ test: { include: testFiles, }, + define: { + __VERSION__: `'${version}'`, + }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01d978d7f..554302d2d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -154,18 +154,9 @@ importers: '@midscene/shared': specifier: workspace:* version: link:../shared - dirty-json: - specifier: 0.9.2 - version: 0.9.2 - langchain: - specifier: 0.3.8 - version: 0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(axios@1.7.7)(cheerio@1.0.0)(openai@4.57.1(zod@3.23.8)) openai: specifier: 4.57.1 version: 4.57.1(zod@3.23.8) - optional: - specifier: 0.1.4 - version: 0.1.4 socks-proxy-agent: specifier: 8.0.4 version: 8.0.4 @@ -179,12 +170,15 @@ importers: '@types/node-fetch': specifier: 2.6.11 version: 2.6.11 + dirty-json: + specifier: 0.9.2 + version: 0.9.2 dotenv: specifier: 16.4.5 version: 16.4.5 langsmith: specifier: 0.1.36 - version: 0.1.36(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(langchain@0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(axios@1.7.7)(cheerio@1.0.0)(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)) + version: 0.1.36(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(langchain@0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)) typescript: specifier: ~5.0.4 version: 5.0.4 @@ -215,6 +209,10 @@ importers: version: 1.6.0(@types/node@18.19.62)(jsdom@24.1.1)(sass-embedded@1.80.5)(terser@5.36.0) packages/visualizer: + dependencies: + buffer: + specifier: 6.0.3 + version: 6.0.3 devDependencies: '@ant-design/icons': specifier: 5.3.7 @@ -324,6 +322,12 @@ importers: openai: specifier: 4.57.1 version: 4.57.1(zod@3.23.8) + socket.io: + specifier: 4.8.1 + version: 4.8.1 + socket.io-client: + specifier: 4.8.1 + version: 4.8.1 devDependencies: '@modern-js/module-tools': specifier: 2.60.6 @@ -3340,6 +3344,9 @@ packages: resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} engines: {node: '>=18'} + '@socket.io/component-emitter@3.1.2': + resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} + '@svgr/babel-plugin-add-jsx-attribute@8.0.0': resolution: {integrity: sha512-b9MIk7yhdS1pMCZM8VeNfUlSKVRhsHZNMl5O9SfaX0l0t5wjdgu4IDzGB8bpnGBBOjGST3rRFVsaaEtI4W6f7g==} engines: {node: '>=14'} @@ -3492,6 +3499,9 @@ packages: '@types/conventional-commits-parser@5.0.0': resolution: {integrity: sha512-loB369iXNmAZglwWATL+WRe+CRMmmBPtpolYzIebFaX4YA3x+BEfLqhUAV9WanycKI3TG1IMr5bMJDajDKLlUQ==} + '@types/cookie@0.4.1': + resolution: {integrity: sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q==} + '@types/cors@2.8.12': resolution: {integrity: sha512-vt+kDhq/M2ayberEtJcIN/hxXy1Pk+59g2FV/ZQceeaTyCtCucjL2Q7FXlFjtWn4n15KCr1NE2lNNFhp0lEThw==} @@ -4148,6 +4158,10 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + base64id@2.0.0: + resolution: {integrity: sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog==} + engines: {node: ^4.5.0 || >= 5.9} + basic-auth@2.0.1: resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==} engines: {node: '>= 0.8'} @@ -5110,6 +5124,17 @@ packages: end-of-stream@1.4.4: resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==} + engine.io-client@6.6.2: + resolution: {integrity: sha512-TAr+NKeoVTjEVW8P3iHguO1LO6RlUz9O5Y8o7EY0fU+gY1NYqas7NN3slpFtbXEsLMHk0h90fJMfKjRkQ0qUIw==} + + engine.io-parser@5.2.3: + resolution: {integrity: sha512-HqD3yTBfnBxIrbnM1DoD6Pcq8NECnh8d4As1Qgh0z5Gg3jRRIqijury0CL3ghu/edArpUYiYqQiDUQBIs4np3Q==} + engines: {node: '>=10.0.0'} + + engine.io@6.6.2: + resolution: {integrity: sha512-gmNvsYi9C8iErnZdVcJnvCpSKbWTt1E8+JZo8b+daLninywUWi5NQ5STSHZ9rFjFO7imNcvb8Pc5pe/wMR5xEw==} + engines: {node: '>=10.2.0'} + enhanced-resolve@5.12.0: resolution: {integrity: sha512-QHTXI/sZQmko1cbDoNAa3mJ5qhWUUNAq3vR0/YiD379fWQrcfuoX1+HW2S0MTt7XmoPLapdaDKUtelUSPic7hQ==} engines: {node: '>=10.13.0'} @@ -7352,9 +7377,6 @@ packages: resolution: {integrity: sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==} hasBin: true - optional@0.1.4: - resolution: {integrity: sha512-gtvrrCfkE08wKcgXaVwQVgwEQ8vel2dc5DDBn9RLQZ3YtmtkBss6A2HY6BnJH4N/4Ku97Ri/SF8sNWE2225WJw==} - ora@5.3.0: resolution: {integrity: sha512-zAKMgGXUim0Jyd6CXK9lraBnD3H5yPGBPPOkC23a2BG6hsm4Zu6OQSjQuEtV0BHDf4aKHcUFvJiGRrFuW3MG8g==} engines: {node: '>=10'} @@ -9007,6 +9029,21 @@ packages: snake-case@3.0.4: resolution: {integrity: sha512-LAOh4z89bGQvl9pFfNF8V146i7o7/CqFPbqzYgP+yYzDIDeS9HaNFtXABamRW+AQzEVODcvE79ljJ+8a9YSdMg==} + socket.io-adapter@2.5.5: + resolution: {integrity: sha512-eLDQas5dzPgOWCk9GuuJC2lBqItuhKI4uxGgo9aIV7MYbk2h9Q6uULEh8WBzThoI7l+qU9Ast9fVUmkqPP9wYg==} + + socket.io-client@4.8.1: + resolution: {integrity: sha512-hJVXfu3E28NmzGk8o1sHhN3om52tRvwYeidbj7xKy2eIIse5IoKX3USlS6Tqt3BHAtflLIkCQBkzVrEEfWUyYQ==} + engines: {node: '>=10.0.0'} + + socket.io-parser@4.2.4: + resolution: {integrity: sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew==} + engines: {node: '>=10.0.0'} + + socket.io@4.8.1: + resolution: {integrity: sha512-oZ7iUCxph8WYRHHcjBEc9unw3adt5CmSNlppj/5Q4k2RIrhl8Z5yY2Xr4j9zj0+wzVZ0bxmYoGSzKJnRl6A4yg==} + engines: {node: '>=10.2.0'} + socks-proxy-agent@8.0.4: resolution: {integrity: sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==} engines: {node: '>= 14'} @@ -9974,6 +10011,18 @@ packages: wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.17.1: + resolution: {integrity: sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==} + engines: {node: '>=10.0.0'} + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: '>=5.0.2' + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + ws@8.18.0: resolution: {integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==} engines: {node: '>=10.0.0'} @@ -10011,6 +10060,10 @@ packages: xmlchars@2.2.0: resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==} + xmlhttprequest-ssl@2.1.2: + resolution: {integrity: sha512-TEU+nJVUUnA4CYJFLvK5X9AOeH4KvDvhIfm0vV1GaQRtchnG0hgK5p8hw/xjv8cunWYCsiPCSDzObPyhEwq3KQ==} + engines: {node: '>=0.4.0'} + xtend@4.0.2: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} @@ -12328,11 +12381,13 @@ snapshots: zod-to-json-schema: 3.24.1(zod@3.23.8) transitivePeerDependencies: - encoding + optional: true '@langchain/textsplitters@0.1.0(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))': dependencies: '@langchain/core': 0.3.26(openai@4.57.1(zod@3.23.8)) js-tiktoken: 1.0.16 + optional: true '@loadable/babel-plugin@5.15.3(@babel/core@7.26.0)': dependencies: @@ -14255,6 +14310,8 @@ snapshots: '@sindresorhus/merge-streams@4.0.0': {} + '@socket.io/component-emitter@3.1.2': {} + '@svgr/babel-plugin-add-jsx-attribute@8.0.0(@babel/core@7.26.0)': dependencies: '@babel/core': 7.26.0 @@ -14432,6 +14489,8 @@ snapshots: '@types/node': 18.19.62 optional: true + '@types/cookie@0.4.1': {} + '@types/cors@2.8.12': {} '@types/css-font-loading-module@0.0.12': {} @@ -15273,6 +15332,8 @@ snapshots: base64-js@1.5.1: {} + base64id@2.0.0: {} + basic-auth@2.0.1: dependencies: safe-buffer: 5.1.2 @@ -16402,6 +16463,37 @@ snapshots: dependencies: once: 1.4.0 + engine.io-client@6.6.2: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.7(supports-color@5.5.0) + engine.io-parser: 5.2.3 + ws: 8.17.1 + xmlhttprequest-ssl: 2.1.2 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + engine.io-parser@5.2.3: {} + + engine.io@6.6.2: + dependencies: + '@types/cookie': 0.4.1 + '@types/cors': 2.8.12 + '@types/node': 18.19.62 + accepts: 1.3.8 + base64id: 2.0.0 + cookie: 0.7.2 + cors: 2.8.5 + debug: 4.3.7(supports-color@5.5.0) + engine.io-parser: 5.2.3 + ws: 8.17.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + enhanced-resolve@5.12.0: dependencies: graceful-fs: 4.2.11 @@ -18139,7 +18231,8 @@ snapshots: jsonparse@1.3.1: {} - jsonpointer@5.0.1: {} + jsonpointer@5.0.1: + optional: true jsonwebtoken@9.0.2: dependencies: @@ -18202,7 +18295,7 @@ snapshots: transitivePeerDependencies: - supports-color - langchain@0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(axios@1.7.7)(cheerio@1.0.0)(openai@4.57.1(zod@3.23.8)): + langchain@0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)): dependencies: '@langchain/core': 0.3.26(openai@4.57.1(zod@3.23.8)) '@langchain/openai': 0.3.16(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8))) @@ -18217,14 +18310,12 @@ snapshots: yaml: 2.6.1 zod: 3.23.8 zod-to-json-schema: 3.24.1(zod@3.23.8) - optionalDependencies: - axios: 1.7.7 - cheerio: 1.0.0 transitivePeerDependencies: - encoding - openai + optional: true - langsmith@0.1.36(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(langchain@0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(axios@1.7.7)(cheerio@1.0.0)(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)): + langsmith@0.1.36(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(langchain@0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)): dependencies: '@types/uuid': 9.0.8 commander: 10.0.1 @@ -18233,7 +18324,7 @@ snapshots: uuid: 9.0.1 optionalDependencies: '@langchain/core': 0.3.26(openai@4.57.1(zod@3.23.8)) - langchain: 0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(axios@1.7.7)(cheerio@1.0.0)(openai@4.57.1(zod@3.23.8)) + langchain: 0.3.8(@langchain/core@0.3.26(openai@4.57.1(zod@3.23.8)))(openai@4.57.1(zod@3.23.8)) openai: 4.57.1(zod@3.23.8) langsmith@0.2.14(openai@4.57.1(zod@3.23.8)): @@ -19312,13 +19403,13 @@ snapshots: zod: 3.23.8 transitivePeerDependencies: - encoding + optional: true - openapi-types@12.1.3: {} + openapi-types@12.1.3: + optional: true opener@1.5.2: {} - optional@0.1.4: {} - ora@5.3.0: dependencies: bl: 4.1.0 @@ -21203,6 +21294,47 @@ snapshots: dot-case: 3.0.4 tslib: 2.8.1 + socket.io-adapter@2.5.5: + dependencies: + debug: 4.3.7(supports-color@5.5.0) + ws: 8.17.1 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-client@4.8.1: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.7(supports-color@5.5.0) + engine.io-client: 6.6.2 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + socket.io-parser@4.2.4: + dependencies: + '@socket.io/component-emitter': 3.1.2 + debug: 4.3.7(supports-color@5.5.0) + transitivePeerDependencies: + - supports-color + + socket.io@4.8.1: + dependencies: + accepts: 1.3.8 + base64id: 2.0.0 + cors: 2.8.5 + debug: 4.3.7(supports-color@5.5.0) + engine.io: 6.6.2 + socket.io-adapter: 2.5.5 + socket.io-parser: 4.2.4 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + socks-proxy-agent@8.0.4: dependencies: agent-base: 7.1.1 @@ -22303,6 +22435,8 @@ snapshots: wrappy@1.0.2: {} + ws@8.17.1: {} + ws@8.18.0: {} xhr@2.6.0: @@ -22328,6 +22462,8 @@ snapshots: xmlchars@2.2.0: {} + xmlhttprequest-ssl@2.1.2: {} + xtend@4.0.2: {} y18n@4.0.3: {} @@ -22347,7 +22483,8 @@ snapshots: yaml@1.10.2: {} - yaml@2.6.1: {} + yaml@2.6.1: + optional: true yargs-parser@18.1.3: dependencies: