176 lines
5.8 KiB
TypeScript
Raw Permalink Normal View History

feat: android playground (#542) * refactor: android api * refactor: enhance Android agent to accept options for device connection * fix: type error * fix: click after clearInput * fix: click before clearInput * feat: android playground * feat: support npx package name * feat: android playground joint * fix: git ignore conflicts * feat: ensure adb server is running before initializing adb client * fix: deps consistency * ci: add android playground * feat: integrate shared constants and improve server configuration in android playground * feat: android playground style * feat: style opt * feat: add @rsbuild/plugin-svgr dependency and improve URI handling in adb * feat: remove unused water flow scripts and update comments to English * feat: download report file * feat: standalone android playground * feat: use dynamic import * feat: migrate CSS to LESS and remove unused styles in chrome extension and report * feat: enhance Android playground with ScrcpyPlayer ref integration and device management improvements * feat: optimize styles and layout in Android playground and visualizer components * chore: add bin back * chore: update build script to exclude documentation generation * feat: add not ready message to PlaygroundResult for improved user guidance * feat: add error handling for screenshot capture in Android page * docs: update readme * feat: add PNG validation for screenshot buffer in Android page * feat: enhance UI components with improved styling and tooltips in ScrcpyPlayer and PromptInput * docs: update uri parameter description in integrate-with-android documentation and improve uri handling in launch function * style: update primary color to #2B83FF across multiple components and adjust margin in App.less * refactor: replace userConfig with globalConfig for environment configuration management and update related functions * feat: integrate server validation logic in App, AdbDevice, and ScrcpyPlayer components for improved connection handling * style: enhance player component layout with overflow handling and margin adjustments * style: refine player component layout with flex adjustments and improved spacing * feat: add midscene model name display and improve layout in EnvConfig component * feat: integrate ShinyText component for enhanced loading progress display in PlaygroundResult * test: add test for isValidPNGImageBuffer * style: remove background color from App.less and adjust AI config override behavior in env.ts --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-17 17:44:11 +08:00
import {
MATCH_BY_POSITION,
MIDSCENE_MODEL_NAME,
MIDSCENE_USE_DOUBAO_VISION,
MIDSCENE_USE_QWEN_VL,
MIDSCENE_USE_VLM_UI_TARS,
MIDSCENE_USE_VL_MODEL,
getAIConfig,
getAIConfigInBoolean,
getAIConfigInJson,
overrideAIConfig,
vlLocateMode,
} from '@midscene/shared/env';
feat: android playground (#542) * refactor: android api * refactor: enhance Android agent to accept options for device connection * fix: type error * fix: click after clearInput * fix: click before clearInput * feat: android playground * feat: support npx package name * feat: android playground joint * fix: git ignore conflicts * feat: ensure adb server is running before initializing adb client * fix: deps consistency * ci: add android playground * feat: integrate shared constants and improve server configuration in android playground * feat: android playground style * feat: style opt * feat: add @rsbuild/plugin-svgr dependency and improve URI handling in adb * feat: remove unused water flow scripts and update comments to English * feat: download report file * feat: standalone android playground * feat: use dynamic import * feat: migrate CSS to LESS and remove unused styles in chrome extension and report * feat: enhance Android playground with ScrcpyPlayer ref integration and device management improvements * feat: optimize styles and layout in Android playground and visualizer components * chore: add bin back * chore: update build script to exclude documentation generation * feat: add not ready message to PlaygroundResult for improved user guidance * feat: add error handling for screenshot capture in Android page * docs: update readme * feat: add PNG validation for screenshot buffer in Android page * feat: enhance UI components with improved styling and tooltips in ScrcpyPlayer and PromptInput * docs: update uri parameter description in integrate-with-android documentation and improve uri handling in launch function * style: update primary color to #2B83FF across multiple components and adjust margin in App.less * refactor: replace userConfig with globalConfig for environment configuration management and update related functions * feat: integrate server validation logic in App, AdbDevice, and ScrcpyPlayer components for improved connection handling * style: enhance player component layout with overflow handling and margin adjustments * style: refine player component layout with flex adjustments and improved spacing * feat: add midscene model name display and improve layout in EnvConfig component * feat: integrate ShinyText component for enhanced loading progress display in PlaygroundResult * test: add test for isValidPNGImageBuffer * style: remove background color from App.less and adjust AI config override behavior in env.ts --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-17 17:44:11 +08:00
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
describe('env', () => {
// backup original environment variables
const originalEnv = { ...process.env };
// clean up before each test
beforeEach(() => {
// reset to empty object, avoid interference between tests
overrideAIConfig({}, false);
});
// restore environment variables after each test
afterEach(() => {
process.env = { ...originalEnv };
});
describe('getAIConfig', () => {
it('should get config value from global config', () => {
overrideAIConfig({ [MIDSCENE_MODEL_NAME]: 'test-model' });
expect(getAIConfig(MIDSCENE_MODEL_NAME)).toBe('test-model');
});
it('should trim config value', () => {
overrideAIConfig({ [MIDSCENE_MODEL_NAME]: ' test-model ' });
expect(getAIConfig(MIDSCENE_MODEL_NAME)).toBe('test-model');
});
it('should return undefined for non-existent config key', () => {
expect(getAIConfig(MIDSCENE_MODEL_NAME)).toBeUndefined();
});
it('should throw error when trying to get MATCH_BY_POSITION', () => {
expect(() => getAIConfig(MATCH_BY_POSITION)).toThrow(
'MATCH_BY_POSITION is deprecated',
);
});
});
describe('getAIConfigInBoolean', () => {
it('should convert "true" to boolean true', () => {
overrideAIConfig({ [MIDSCENE_USE_QWEN_VL]: 'true' });
expect(getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)).toBe(true);
});
it('should convert "1" to boolean true', () => {
overrideAIConfig({ [MIDSCENE_USE_QWEN_VL]: '1' });
expect(getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)).toBe(true);
});
it('should convert other values to boolean false', () => {
overrideAIConfig({ [MIDSCENE_USE_QWEN_VL]: 'false' });
expect(getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)).toBe(false);
overrideAIConfig({ [MIDSCENE_USE_QWEN_VL]: '0' });
expect(getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)).toBe(false);
overrideAIConfig({ [MIDSCENE_USE_QWEN_VL]: 'anything' });
feat(web-integration): add support for new agent method(aiBoolean, aiString, aiNumber, aiLocate) (#658) * feat(web-integration): add support for new agent method(aiBoolean, aiString, aiNumber, aiLocate) * fix(core, web-integration): update data extraction logic and refine return types for agent methods * feat(site): add detailed descriptions and examples * fix(web-integration): remove unused properties from aiLocate return type and update UI titles * fix(core): enhance agent methods for better extraction and querying * fix(core): update test scripts and refine data extraction prompts * fix(core): refine return type in ExecutionTaskApply and remove unused import in inspect * fix(core): update data extraction logic and enhance type handling in Insight class * fix(web-integration): update import path for puppeteerAgentForTarget in player test file * fix(core): tests fix * fix(core): fix tests * fix(shared): enhance imageInfo tests by validating individual properties and updating snapshots * fix(site): remove waitForNavigationTimeout from Playwright integration examples * fix(site): update timeout configuration examples in FAQ for clarity * test(web-integration): remove unused client-extractor test and related snapshots * chore(core): fix query implementation * fix(report): improve data handling in DetailSide component and enhance output display * fix(core): refine data extraction prompts and improve error handling in PageTaskExecutor * fix(core): update data extraction prompt for clarity and type validation --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-29 13:36:49 +08:00
expect(getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)).toBe(true);
feat: android playground (#542) * refactor: android api * refactor: enhance Android agent to accept options for device connection * fix: type error * fix: click after clearInput * fix: click before clearInput * feat: android playground * feat: support npx package name * feat: android playground joint * fix: git ignore conflicts * feat: ensure adb server is running before initializing adb client * fix: deps consistency * ci: add android playground * feat: integrate shared constants and improve server configuration in android playground * feat: android playground style * feat: style opt * feat: add @rsbuild/plugin-svgr dependency and improve URI handling in adb * feat: remove unused water flow scripts and update comments to English * feat: download report file * feat: standalone android playground * feat: use dynamic import * feat: migrate CSS to LESS and remove unused styles in chrome extension and report * feat: enhance Android playground with ScrcpyPlayer ref integration and device management improvements * feat: optimize styles and layout in Android playground and visualizer components * chore: add bin back * chore: update build script to exclude documentation generation * feat: add not ready message to PlaygroundResult for improved user guidance * feat: add error handling for screenshot capture in Android page * docs: update readme * feat: add PNG validation for screenshot buffer in Android page * feat: enhance UI components with improved styling and tooltips in ScrcpyPlayer and PromptInput * docs: update uri parameter description in integrate-with-android documentation and improve uri handling in launch function * style: update primary color to #2B83FF across multiple components and adjust margin in App.less * refactor: replace userConfig with globalConfig for environment configuration management and update related functions * feat: integrate server validation logic in App, AdbDevice, and ScrcpyPlayer components for improved connection handling * style: enhance player component layout with overflow handling and margin adjustments * style: refine player component layout with flex adjustments and improved spacing * feat: add midscene model name display and improve layout in EnvConfig component * feat: integrate ShinyText component for enhanced loading progress display in PlaygroundResult * test: add test for isValidPNGImageBuffer * style: remove background color from App.less and adjust AI config override behavior in env.ts --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-17 17:44:11 +08:00
});
it('should return false for non-existent config', () => {
expect(getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)).toBe(false);
});
});
describe('getAIConfigInJson', () => {
it('should parse valid JSON string', () => {
const jsonConfig = '{"key": "value", "number": 123}';
overrideAIConfig({ [MIDSCENE_MODEL_NAME]: jsonConfig });
expect(getAIConfigInJson(MIDSCENE_MODEL_NAME)).toEqual({
key: 'value',
number: 123,
});
});
it('should return undefined for non-existent config key', () => {
expect(getAIConfigInJson(MIDSCENE_MODEL_NAME)).toBeUndefined();
});
it('should throw error for invalid JSON', () => {
overrideAIConfig({ [MIDSCENE_MODEL_NAME]: '{invalid json}' });
expect(() => getAIConfigInJson(MIDSCENE_MODEL_NAME)).toThrow(
'Failed to parse json config',
);
});
});
describe('overrideAIConfig', () => {
it('should extend global config when extendMode is true', () => {
feat: android playground (#542) * refactor: android api * refactor: enhance Android agent to accept options for device connection * fix: type error * fix: click after clearInput * fix: click before clearInput * feat: android playground * feat: support npx package name * feat: android playground joint * fix: git ignore conflicts * feat: ensure adb server is running before initializing adb client * fix: deps consistency * ci: add android playground * feat: integrate shared constants and improve server configuration in android playground * feat: android playground style * feat: style opt * feat: add @rsbuild/plugin-svgr dependency and improve URI handling in adb * feat: remove unused water flow scripts and update comments to English * feat: download report file * feat: standalone android playground * feat: use dynamic import * feat: migrate CSS to LESS and remove unused styles in chrome extension and report * feat: enhance Android playground with ScrcpyPlayer ref integration and device management improvements * feat: optimize styles and layout in Android playground and visualizer components * chore: add bin back * chore: update build script to exclude documentation generation * feat: add not ready message to PlaygroundResult for improved user guidance * feat: add error handling for screenshot capture in Android page * docs: update readme * feat: add PNG validation for screenshot buffer in Android page * feat: enhance UI components with improved styling and tooltips in ScrcpyPlayer and PromptInput * docs: update uri parameter description in integrate-with-android documentation and improve uri handling in launch function * style: update primary color to #2B83FF across multiple components and adjust margin in App.less * refactor: replace userConfig with globalConfig for environment configuration management and update related functions * feat: integrate server validation logic in App, AdbDevice, and ScrcpyPlayer components for improved connection handling * style: enhance player component layout with overflow handling and margin adjustments * style: refine player component layout with flex adjustments and improved spacing * feat: add midscene model name display and improve layout in EnvConfig component * feat: integrate ShinyText component for enhanced loading progress display in PlaygroundResult * test: add test for isValidPNGImageBuffer * style: remove background color from App.less and adjust AI config override behavior in env.ts --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-17 17:44:11 +08:00
overrideAIConfig({ [MIDSCENE_MODEL_NAME]: 'model-1' });
overrideAIConfig({ [MIDSCENE_USE_QWEN_VL]: 'true' }, true);
feat: android playground (#542) * refactor: android api * refactor: enhance Android agent to accept options for device connection * fix: type error * fix: click after clearInput * fix: click before clearInput * feat: android playground * feat: support npx package name * feat: android playground joint * fix: git ignore conflicts * feat: ensure adb server is running before initializing adb client * fix: deps consistency * ci: add android playground * feat: integrate shared constants and improve server configuration in android playground * feat: android playground style * feat: style opt * feat: add @rsbuild/plugin-svgr dependency and improve URI handling in adb * feat: remove unused water flow scripts and update comments to English * feat: download report file * feat: standalone android playground * feat: use dynamic import * feat: migrate CSS to LESS and remove unused styles in chrome extension and report * feat: enhance Android playground with ScrcpyPlayer ref integration and device management improvements * feat: optimize styles and layout in Android playground and visualizer components * chore: add bin back * chore: update build script to exclude documentation generation * feat: add not ready message to PlaygroundResult for improved user guidance * feat: add error handling for screenshot capture in Android page * docs: update readme * feat: add PNG validation for screenshot buffer in Android page * feat: enhance UI components with improved styling and tooltips in ScrcpyPlayer and PromptInput * docs: update uri parameter description in integrate-with-android documentation and improve uri handling in launch function * style: update primary color to #2B83FF across multiple components and adjust margin in App.less * refactor: replace userConfig with globalConfig for environment configuration management and update related functions * feat: integrate server validation logic in App, AdbDevice, and ScrcpyPlayer components for improved connection handling * style: enhance player component layout with overflow handling and margin adjustments * style: refine player component layout with flex adjustments and improved spacing * feat: add midscene model name display and improve layout in EnvConfig component * feat: integrate ShinyText component for enhanced loading progress display in PlaygroundResult * test: add test for isValidPNGImageBuffer * style: remove background color from App.less and adjust AI config override behavior in env.ts --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-17 17:44:11 +08:00
expect(getAIConfig(MIDSCENE_MODEL_NAME)).toBe('model-1');
expect(getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL)).toBe(true);
});
it('should replace global config when extendMode is false', () => {
overrideAIConfig({
[MIDSCENE_MODEL_NAME]: 'model-1',
[MIDSCENE_USE_QWEN_VL]: 'true',
});
overrideAIConfig({ [MIDSCENE_MODEL_NAME]: 'model-2' }, false);
expect(getAIConfig(MIDSCENE_MODEL_NAME)).toBe('model-2');
expect(getAIConfig(MIDSCENE_USE_QWEN_VL)).toBeUndefined();
});
it('should convert numeric keys to strings without error', () => {
// numeric keys will be converted to strings automatically in JavaScript, without throwing an error
// @ts-ignore - test the behavior by using a numeric key
expect(() => overrideAIConfig({ [123]: 'value' })).not.toThrow();
});
it('should throw error for object value', () => {
expect(() =>
overrideAIConfig({
// @ts-expect-error - test the behavior by using an object value
[MIDSCENE_MODEL_NAME]: { key: 'value' },
}),
).toThrow('invalid value');
});
});
describe('vlLocateMode', () => {
it('should return false when no VL mode is enabled', () => {
expect(vlLocateMode()).toBe(false);
});
it('should return "qwen-vl" when MIDSCENE_USE_QWEN_VL is true', () => {
overrideAIConfig({ [MIDSCENE_USE_QWEN_VL]: 'true' });
expect(vlLocateMode()).toBe('qwen-vl');
});
it('should return "doubao-vision" when MIDSCENE_USE_DOUBAO_VISION is true', () => {
overrideAIConfig({ [MIDSCENE_USE_DOUBAO_VISION]: 'true' });
expect(vlLocateMode()).toBe('doubao-vision');
});
it('should return "vl-model" when MIDSCENE_USE_VL_MODEL is true', () => {
overrideAIConfig({ [MIDSCENE_USE_VL_MODEL]: 'true' });
expect(vlLocateMode()).toBe('vl-model');
});
it('should return "vlm-ui-tars" when MIDSCENE_USE_VLM_UI_TARS is true', () => {
overrideAIConfig({ [MIDSCENE_USE_VLM_UI_TARS]: 'true' });
expect(vlLocateMode()).toBe('vlm-ui-tars');
});
it('should throw error when multiple VL modes are enabled', () => {
overrideAIConfig({
[MIDSCENE_USE_QWEN_VL]: 'true',
[MIDSCENE_USE_DOUBAO_VISION]: 'true',
});
expect(() => vlLocateMode()).toThrow(
'Only one vision mode can be enabled at a time',
);
});
});
});