yujunjun/midscene
1
0
Fork 0
mirror of https://github.com/web-infra-dev/midscene.git synced 2025-12-28 23:49:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: some minor bug fixes (#146)

Browse Source 
* feat: always use latest process.env as fallback

---------

Co-authored-by: zhouxiao.shaw <zhouxiao.shaw@bytedance.com>
This commit is contained in:
yuyutaotao 2024-11-07 11:29:50 +08:00 committed by GitHub
parent b30b223473
commit f8507efd50
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 87 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/ci.yml vendored
View File

@ -45,7 +45,12 @@ jobs:
${{ runner.os }}-puppeteer-
- name: Install dependencies
run: pnpm install --frozen-lockfile
run: pnpm install --frozen-lockfile --ignore-scripts
- name: Install puppeteer dependencies
run: |
cd packages/web-integration
npx puppeteer browsers install chrome
- name: Build project
run: pnpm run build

28
packages/midscene/modern.config.ts
View File

@ -4,20 +4,18 @@ import { version } from './package.json';
export default defineConfig({
plugins: [moduleTools()],
buildPreset: 'npm-library',
buildConfig: [
{
format: 'umd',
input: {
index: 'src/index.ts',
utils: 'src/utils.ts',
'ai-model': 'src/ai-model/index.ts',
},
outDir: 'dist/lib',
externals: ['langsmith'],
target: 'es6',
define: {
__VERSION__: version,
},
buildConfig: {
format: 'cjs',
input: {
index: 'src/index.ts',
utils: 'src/utils.ts',
'ai-model': 'src/ai-model/index.ts',
},
],
outDir: 'dist/lib',
externals: ['langsmith'],
target: 'es2018',
define: {
__VERSION__: version,
},
},
});

8
packages/midscene/package.json
View File

@ -7,7 +7,7 @@
"jsnext:source": "./src/index.ts",
"type": "commonjs",
"main": "./dist/lib/index.js",
"types": "./dist/types/index.d.ts",
"types": "./dist/lib/types/index.d.ts",
"files": ["dist", "report", "README.md"],
"exports": {
".": "./dist/lib/index.js",
@ -16,9 +16,9 @@
},
"typesVersions": {
"*": {
".": ["./dist/types/index.d.ts"],
"utils": ["./dist/lib/utils.d.ts"],
"ai-model": ["./dist/lib/ai-model.d.ts"]
".": ["./dist/lib/types/index.d.ts"],
"utils": ["./dist/lib/types/utils.d.ts"],
"ai-model": ["./dist/lib/types/ai-model.d.ts"]
}
},
"scripts": {

66
packages/midscene/src/ai-model/openai/index.ts
View File

@ -14,6 +14,8 @@ export const MIDSCENE_OPENAI_INIT_CONFIG_JSON =
export const MIDSCENE_MODEL_NAME = 'MIDSCENE_MODEL_NAME';
export const MIDSCENE_LANGSMITH_DEBUG = 'MIDSCENE_LANGSMITH_DEBUG';
export const MIDSCENE_DEBUG_AI_PROFILE = 'MIDSCENE_DEBUG_AI_PROFILE';
export const MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG =
'MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG';
export const OPENAI_API_KEY = 'OPENAI_API_KEY';
export const OPENAI_BASE_URL = 'OPENAI_BASE_URL';
export const MIDSCENE_MODEL_TEXT_ONLY = 'MIDSCENE_MODEL_TEXT_ONLY';
@ -21,37 +23,46 @@ export const OPENAI_USE_AZURE = 'OPENAI_USE_AZURE';
export const MIDSCENE_CACHE = 'MIDSCENE_CACHE';
export const MATCH_BY_POSITION = 'MATCH_BY_POSITION';
let config: Record<string, string | undefined> = {
[MIDSCENE_OPENAI_INIT_CONFIG_JSON]:
process.env[MIDSCENE_OPENAI_INIT_CONFIG_JSON] || undefined,
[MIDSCENE_MODEL_NAME]: process.env[MIDSCENE_MODEL_NAME] || undefined,
[MIDSCENE_LANGSMITH_DEBUG]:
process.env[MIDSCENE_LANGSMITH_DEBUG] || undefined,
[MIDSCENE_DEBUG_AI_PROFILE]:
process.env[MIDSCENE_DEBUG_AI_PROFILE] || undefined,
[OPENAI_API_KEY]: process.env[OPENAI_API_KEY] || undefined,
[OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || undefined,
[MIDSCENE_MODEL_TEXT_ONLY]:
process.env[MIDSCENE_MODEL_TEXT_ONLY] || undefined,
[OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || undefined,
[MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || undefined,
[MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || undefined,
const allConfigFromEnv = () => {
return {
[MIDSCENE_OPENAI_INIT_CONFIG_JSON]:
process.env[MIDSCENE_OPENAI_INIT_CONFIG_JSON] || undefined,
[MIDSCENE_MODEL_NAME]: process.env[MIDSCENE_MODEL_NAME] || undefined,
[MIDSCENE_LANGSMITH_DEBUG]:
process.env[MIDSCENE_LANGSMITH_DEBUG] || undefined,
[MIDSCENE_DEBUG_AI_PROFILE]:
process.env[MIDSCENE_DEBUG_AI_PROFILE] || undefined,
[MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG]:
process.env[MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG] || undefined,
[OPENAI_API_KEY]: process.env[OPENAI_API_KEY] || undefined,
[OPENAI_BASE_URL]: process.env[OPENAI_BASE_URL] || undefined,
[MIDSCENE_MODEL_TEXT_ONLY]:
process.env[MIDSCENE_MODEL_TEXT_ONLY] || undefined,
[OPENAI_USE_AZURE]: process.env[OPENAI_USE_AZURE] || undefined,
[MIDSCENE_CACHE]: process.env[MIDSCENE_CACHE] || undefined,
[MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || undefined,
};
};
let userConfig: ReturnType<typeof allConfigFromEnv> = {} as any;
export const getAIConfig = (
configKey: keyof typeof config,
configKey: keyof typeof userConfig,
): string | undefined => {
return config[configKey];
if (typeof userConfig[configKey] !== 'undefined') {
return userConfig[configKey];
}
return allConfigFromEnv()[configKey];
};
export const allAIConfig = () => {
return config;
return { ...allConfigFromEnv(), ...userConfig };
};
export const overrideAIConfig = (
newConfig: Record<string, string | undefined>,
) => {
config = { ...config, ...newConfig };
userConfig = { ...userConfig, ...newConfig };
};
export function preferOpenAIModel(preferVendor?: 'coze' | 'openAI') {
@ -63,7 +74,6 @@ export function preferOpenAIModel(preferVendor?: 'coze' | 'openAI') {
// default model
const defaultModel = 'gpt-4o';
export function getModelName() {
let modelName = defaultModel;
const nameInConfig = getAIConfig(MIDSCENE_MODEL_NAME);
@ -73,19 +83,10 @@ export function getModelName() {
return modelName;
}
const defaultExtraConfig: ClientOptions = {};
function getExtraConfig() {
let extraConfig = defaultExtraConfig;
const configInEnv = getAIConfig(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
if (configInEnv) {
extraConfig = JSON.parse(configInEnv);
}
return extraConfig;
}
async function createOpenAI() {
let openai: OpenAI | AzureOpenAI;
const extraConfig = getExtraConfig();
const extraConfigString = getAIConfig(MIDSCENE_OPENAI_INIT_CONFIG_JSON);
const extraConfig = extraConfigString ? JSON.parse(extraConfigString) : {};
if (getAIConfig(OPENAI_USE_AZURE)) {
openai = new AzureOpenAI({
baseURL: getAIConfig(OPENAI_BASE_URL),
@ -123,6 +124,9 @@ export async function call(
const openai = await createOpenAI();
const shouldPrintTiming =
typeof getAIConfig(MIDSCENE_DEBUG_AI_PROFILE) === 'string';
if (getAIConfig(MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG)) {
console.log(allAIConfig());
}
const startTime = Date.now();
const model = getModelName();
const completion = await openai.chat.completions.create({

7
packages/visualizer/modern.config.ts
View File

@ -10,7 +10,7 @@ const commonConfig = {
},
autoExternal: false,
externals: [...externals],
target: 'es6',
target: 'es2018',
minify: process.env.CI
? {
compress: true,
@ -42,8 +42,7 @@ export default defineConfig({
},
platform: 'browser',
outDir: 'dist',
target: 'es6',
target: 'es2018',
},
{
...commonConfig,
@ -59,7 +58,7 @@ export default defineConfig({
},
platform: 'browser',
outDir: 'unpacked-extension/lib',
target: 'es6',
target: 'es2018',
},
],
plugins: [

6
packages/visualizer/src/component/blackboard.tsx
View File

@ -4,11 +4,7 @@ import { Checkbox } from 'antd';
import type { CheckboxProps } from 'antd';
import * as PIXI from 'pixi.js';
import { type ReactElement, useEffect, useMemo, useRef, useState } from 'react';
import type {
BaseElement,
Rect,
UIContext,
} from '../../../midscene/dist/types';
import type { BaseElement, Rect, UIContext } from '../../../midscene';
import { colorForName, highlightColorForType } from './color';
import './blackboard.less';
import { DropShadowFilter } from 'pixi-filters';

23
packages/visualizer/src/component/playground-component.tsx
View File

@ -305,22 +305,6 @@ export function Playground({
(serviceMode === 'Server' && serverValid) ||
(serviceMode === 'In-Browser-Extension' && agent && configAlreadySet);
// use cmd + enter to run
useEffect(() => {
const handleKeyDown = (e: KeyboardEvent) => {
if (e.key === 'Enter') {
handleRun();
e.preventDefault();
e.stopPropagation();
}
};
window.addEventListener('keydown', handleKeyDown);
return () => {
window.removeEventListener('keydown', handleKeyDown);
};
}, [handleRun]);
let resultDataToShow: any = (
<div className="result-empty-tip">
<span>The result will be shown here</span>
@ -492,6 +476,13 @@ export function Playground({
rows={2}
placeholder={placeholder}
autoFocus
onKeyDown={(e) => {
if (e.key === 'Enter' && e.metaKey) {
handleRun();
e.preventDefault();
e.stopPropagation();
}
}}
/>
</Form.Item>
{actionBtn}

2
packages/visualizer/src/component/store.tsx
View File

@ -6,7 +6,7 @@ import type {
ExecutionTaskInsightLocate,
GroupedActionDump,
InsightDump,
} from '../../../midscene/dist/types';
} from '../../../midscene';
import type { AnimationScript } from './replay-scripts';
import { allScriptsFromDump, generateAnimationScripts } from './replay-scripts';

4
packages/web-integration/modern.config.ts
View File

@ -4,7 +4,7 @@ export default defineConfig({
plugins: [moduleTools()],
buildPreset: 'npm-library',
buildConfig: {
format: 'umd',
format: 'cjs',
input: {
index: 'src/index.ts',
utils: 'src/common/utils.ts',
@ -17,7 +17,7 @@ export default defineConfig({
'playwright-report': './src/playwright/reporter/index.ts',
'chrome-extension': 'src/chrome-extension/index.ts',
},
target: 'es6',
target: 'es2018',
externals: ['@midscene/core', '@midscene/shared'],
},
});

6
packages/web-integration/src/chrome-extension/page.ts
View File

@ -246,12 +246,8 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
keyboard = {
type: async (text: string) => {
for (const char of text) {
// Send char event
await this.sendCommandToDebugger('Input.dispatchKeyEvent', {
type: 'char',
await this.sendCommandToDebugger('Input.insertText', {
text: char,
key: char,
unmodifiedText: char,
});
// sleep 50ms

3
packages/web-integration/src/puppeteer/base-page.ts
View File

@ -104,7 +104,8 @@ export class Page<
get keyboard() {
return {
type: async (text: string) => this.underlyingPage.keyboard.type(text),
type: async (text: string) =>
this.underlyingPage.keyboard.type(text, { delay: 80 }),
press: async (key: WebKeyInput) =>
this.underlyingPage.keyboard.press(key),
down: async (key: WebKeyInput) => this.underlyingPage.keyboard.down(key),

2
packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts
View File

@ -75,7 +75,7 @@ describe(
const { originPage, reset } = await launchPage('https://www.baidu.com/');
const mid = new PuppeteerAgent(originPage);
await mid.aiAction(
'type "Weather in Shanghai" in search box, hit Enter, wait 2s`',
'type "Shanghai 天气" in search box, hit Enter, wait 2s`',
);
await mid.aiWaitFor('there is weather info in Shanghai');

16
pnpm-lock.yaml generated
View File

@ -3498,8 +3498,8 @@ packages:
'@types/node@22.8.5':
resolution: {integrity: sha512-5iYk6AMPtsMbkZqCO1UGF9W5L38twq11S2pYWkybGHH2ogPUvXWNlQqJBzuEZWKj/WRH+QTeiv6ySWqJtvIEgA==}
'@types/node@22.8.7':
resolution: {integrity: sha512-LidcG+2UeYIWcMuMUpBKOnryBWG/rnmOHQR5apjn8myTQcx3rinFRn7DcIFhMnS0PPFSC6OafdIKEad0lj6U0Q==}
'@types/node@22.9.0':
resolution: {integrity: sha512-vuyHg81vvWA1Z1ELfvLko2c8f34gyA0zaic0+Rllc5lbCnbSyuvb2Oxpm6TAUAC/2xZN3QGqxBNggD1nNR2AfQ==}
'@types/normalize-package-data@2.4.4':
resolution: {integrity: sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==}
@ -3519,6 +3519,9 @@ packages:
'@types/qs@6.9.16':
resolution: {integrity: sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==}
'@types/qs@6.9.17':
resolution: {integrity: sha512-rX4/bPcfmvxHDv0XjfJELTTr+iB+tn032nPILqHm5wbthUUUuVtNGGqzhya9XUxjTP8Fpr0qYgSZZKxGY++svQ==}
'@types/range-parser@1.2.7':
resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==}
@ -13989,7 +13992,7 @@ snapshots:
'@types/conventional-commits-parser@5.0.0':
dependencies:
'@types/node': 22.8.7
'@types/node': 22.9.0
optional: true
'@types/cors@2.8.12': {}
@ -14033,7 +14036,7 @@ snapshots:
dependencies:
'@types/body-parser': 1.19.5
'@types/express-serve-static-core': 4.19.6
'@types/qs': 6.9.16
'@types/qs': 6.9.17
'@types/serve-static': 1.15.7
optional: true
@ -14139,7 +14142,7 @@ snapshots:
dependencies:
undici-types: 6.19.8
'@types/node@22.8.7':
'@types/node@22.9.0':
dependencies:
undici-types: 6.19.8
optional: true
@ -14161,6 +14164,9 @@ snapshots:
'@types/qs@6.9.16': {}
'@types/qs@6.9.17':
optional: true
'@types/range-parser@1.2.7': {}
'@types/react-dom@18.3.0':