216 lines
6.2 KiB
TypeScript
Raw Permalink Normal View History

import { readFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
base64Encoded,
imageInfo,
imageInfoOfBase64,
feat: android playground (#542) * refactor: android api * refactor: enhance Android agent to accept options for device connection * fix: type error * fix: click after clearInput * fix: click before clearInput * feat: android playground * feat: support npx package name * feat: android playground joint * fix: git ignore conflicts * feat: ensure adb server is running before initializing adb client * fix: deps consistency * ci: add android playground * feat: integrate shared constants and improve server configuration in android playground * feat: android playground style * feat: style opt * feat: add @rsbuild/plugin-svgr dependency and improve URI handling in adb * feat: remove unused water flow scripts and update comments to English * feat: download report file * feat: standalone android playground * feat: use dynamic import * feat: migrate CSS to LESS and remove unused styles in chrome extension and report * feat: enhance Android playground with ScrcpyPlayer ref integration and device management improvements * feat: optimize styles and layout in Android playground and visualizer components * chore: add bin back * chore: update build script to exclude documentation generation * feat: add not ready message to PlaygroundResult for improved user guidance * feat: add error handling for screenshot capture in Android page * docs: update readme * feat: add PNG validation for screenshot buffer in Android page * feat: enhance UI components with improved styling and tooltips in ScrcpyPlayer and PromptInput * docs: update uri parameter description in integrate-with-android documentation and improve uri handling in launch function * style: update primary color to #2B83FF across multiple components and adjust margin in App.less * refactor: replace userConfig with globalConfig for environment configuration management and update related functions * feat: integrate server validation logic in App, AdbDevice, and ScrcpyPlayer components for improved connection handling * style: enhance player component layout with overflow handling and margin adjustments * style: refine player component layout with flex adjustments and improved spacing * feat: add midscene model name display and improve layout in EnvConfig component * feat: integrate ShinyText component for enhanced loading progress display in PlaygroundResult * test: add test for isValidPNGImageBuffer * style: remove background color from App.less and adjust AI config override behavior in env.ts --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-17 17:44:11 +08:00
isValidPNGImageBuffer,
resizeImg,
resizeImgBase64,
} from 'src/img';
import getJimp from 'src/img/get-jimp';
import {
cropByRect,
jimpFromBase64,
jimpToBase64,
paddingToMatchBlock,
saveBase64Image,
} from 'src/img/transform';
import { getFixture } from 'tests/utils';
import { describe, expect, it } from 'vitest';
describe('image utils', () => {
const image = getFixture('icon.png');
it('imageInfo', async () => {
const info = await imageInfo(image);
feat(web-integration): add support for new agent method(aiBoolean, aiString, aiNumber, aiLocate) (#658) * feat(web-integration): add support for new agent method(aiBoolean, aiString, aiNumber, aiLocate) * fix(core, web-integration): update data extraction logic and refine return types for agent methods * feat(site): add detailed descriptions and examples * fix(web-integration): remove unused properties from aiLocate return type and update UI titles * fix(core): enhance agent methods for better extraction and querying * fix(core): update test scripts and refine data extraction prompts * fix(core): refine return type in ExecutionTaskApply and remove unused import in inspect * fix(core): update data extraction logic and enhance type handling in Insight class * fix(web-integration): update import path for puppeteerAgentForTarget in player test file * fix(core): tests fix * fix(core): fix tests * fix(shared): enhance imageInfo tests by validating individual properties and updating snapshots * fix(site): remove waitForNavigationTimeout from Playwright integration examples * fix(site): update timeout configuration examples in FAQ for clarity * test(web-integration): remove unused client-extractor test and related snapshots * chore(core): fix query implementation * fix(report): improve data handling in DetailSide component and enhance output display * fix(core): refine data extraction prompts and improve error handling in PageTaskExecutor * fix(core): update data extraction prompt for clarity and type validation --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-29 13:36:49 +08:00
// test basic properties of ImageInfo
expect(info.width).toMatchSnapshot();
expect(info.height).toMatchSnapshot();
// test basic properties of jimpImage
expect(typeof info.jimpImage.getBuffer).toBe('function');
expect(typeof info.jimpImage.getBufferAsync).toBe('function');
expect(typeof info.jimpImage.getPixelColour).toBe('function');
expect(typeof info.jimpImage.setPixelColour).toBe('function');
expect(typeof info.jimpImage.writeAsync).toBe('function');
// shapeMode is inconsistent across environments
expect(info.jimpImage.bitmap).toMatchSnapshot();
});
it('base64Encoded', () => {
const base64 = base64Encoded(image);
expect(base64).toMatchSnapshot();
const headlessBase64 = base64Encoded(image, false);
expect(headlessBase64).toMatchSnapshot();
});
it('base64 + imageInfo', async () => {
const image = getFixture('icon.png');
const base64 = base64Encoded(image);
const info = await imageInfoOfBase64(base64);
expect(info.width).toMatchSnapshot();
expect(info.height).toMatchSnapshot();
});
it('jpeg + base64 + imageInfo', async () => {
const image = getFixture('heytea.jpeg');
const base64 = base64Encoded(image);
const info = await imageInfoOfBase64(base64);
expect(info.width).toMatchSnapshot();
expect(info.height).toMatchSnapshot();
});
it('jimp + imageInfo', async () => {
const image = getFixture('heytea.jpeg');
const jimp = await getJimp();
const jimpImage = await jimp.read(image);
const info = await imageInfo(jimpImage);
expect(info.width).toMatchSnapshot();
expect(info.height).toMatchSnapshot();
});
it('resizeImgBase64', async () => {
const image = getFixture('heytea.jpeg');
const base64 = base64Encoded(image);
const resizedBase64 = await resizeImgBase64(base64, {
width: 100,
height: 100,
});
expect(resizedBase64).toContain(';base64,');
});
it('resize image', async () => {
const image = getFixture('heytea.jpeg');
const buffer = await resizeImg(readFileSync(image), {
width: 100,
height: 100,
});
expect(buffer).toBeDefined();
});
it('paddingToMatchBlock', async () => {
const image = getFixture('heytea.jpeg');
const base64 = base64Encoded(image);
const jimpImage = await jimpFromBase64(base64);
const result = await paddingToMatchBlock(jimpImage);
const width = result.bitmap.width;
expect(width).toMatchSnapshot();
const height = result.bitmap.height;
expect(height).toMatchSnapshot();
const tmpFile = join(tmpdir(), 'heytea-padded.jpeg');
await saveBase64Image({
base64Data: await jimpToBase64(result),
outputPath: tmpFile,
});
// console.log('tmpFile', tmpFile);
});
it('cropByRect, with padding', async () => {
const image = getFixture('heytea.jpeg');
const base64 = base64Encoded(image);
const croppedBase64 = await cropByRect(
base64,
{
left: 200,
top: 80,
width: 100,
height: 400,
},
true,
);
expect(croppedBase64).toBeTruthy();
const info = await imageInfoOfBase64(croppedBase64);
// biome-ignore lint/style/noUnusedTemplateLiteral: by intention
expect(info.width).toMatchInlineSnapshot(`112`);
// biome-ignore lint/style/noUnusedTemplateLiteral: by intention
expect(info.height).toMatchInlineSnapshot(`420`);
const tmpFile = join(tmpdir(), 'heytea-cropped.jpeg');
await saveBase64Image({
base64Data: croppedBase64,
outputPath: tmpFile,
});
console.log('cropped image saved to', tmpFile);
});
it('cropByRect, without padding', async () => {
const image = getFixture('heytea.jpeg');
const base64 = base64Encoded(image);
const croppedBase64 = await cropByRect(
base64,
{
left: 200,
top: 80,
width: 100,
height: 400,
},
false,
);
expect(croppedBase64).toBeTruthy();
const info = await imageInfoOfBase64(croppedBase64);
// biome-ignore lint/style/noUnusedTemplateLiteral: by intention
expect(info.width).toMatchInlineSnapshot(`100`);
// biome-ignore lint/style/noUnusedTemplateLiteral: by intention
expect(info.height).toMatchInlineSnapshot(`400`);
const tmpFile = join(tmpdir(), 'heytea-cropped-2.jpeg');
await saveBase64Image({
base64Data: croppedBase64,
outputPath: tmpFile,
});
console.log('cropped image saved to', tmpFile);
});
feat: android playground (#542) * refactor: android api * refactor: enhance Android agent to accept options for device connection * fix: type error * fix: click after clearInput * fix: click before clearInput * feat: android playground * feat: support npx package name * feat: android playground joint * fix: git ignore conflicts * feat: ensure adb server is running before initializing adb client * fix: deps consistency * ci: add android playground * feat: integrate shared constants and improve server configuration in android playground * feat: android playground style * feat: style opt * feat: add @rsbuild/plugin-svgr dependency and improve URI handling in adb * feat: remove unused water flow scripts and update comments to English * feat: download report file * feat: standalone android playground * feat: use dynamic import * feat: migrate CSS to LESS and remove unused styles in chrome extension and report * feat: enhance Android playground with ScrcpyPlayer ref integration and device management improvements * feat: optimize styles and layout in Android playground and visualizer components * chore: add bin back * chore: update build script to exclude documentation generation * feat: add not ready message to PlaygroundResult for improved user guidance * feat: add error handling for screenshot capture in Android page * docs: update readme * feat: add PNG validation for screenshot buffer in Android page * feat: enhance UI components with improved styling and tooltips in ScrcpyPlayer and PromptInput * docs: update uri parameter description in integrate-with-android documentation and improve uri handling in launch function * style: update primary color to #2B83FF across multiple components and adjust margin in App.less * refactor: replace userConfig with globalConfig for environment configuration management and update related functions * feat: integrate server validation logic in App, AdbDevice, and ScrcpyPlayer components for improved connection handling * style: enhance player component layout with overflow handling and margin adjustments * style: refine player component layout with flex adjustments and improved spacing * feat: add midscene model name display and improve layout in EnvConfig component * feat: integrate ShinyText component for enhanced loading progress display in PlaygroundResult * test: add test for isValidPNGImageBuffer * style: remove background color from App.less and adjust AI config override behavior in env.ts --------- Co-authored-by: yutao <yutao.tao@bytedance.com>
2025-04-17 17:44:11 +08:00
it('isValidPNGImageBuffer', () => {
const buffer = readFileSync(getFixture('icon.png'));
const isValid = isValidPNGImageBuffer(buffer);
expect(isValid).toBe(true);
});
it('isValidPNGImageBuffer, invalid', () => {
const buffer = readFileSync(getFixture('heytea.jpeg'));
const isValid = isValidPNGImageBuffer(buffer);
expect(isValid).toBe(false);
});
it('isValidPNGImageBuffer, invalid buffer', () => {
const isValid = isValidPNGImageBuffer(
Buffer.from(
'<Buffer 49 6e 76 61 6c 69 64 20 64 69 73 70 6c 61 79 20 49 44 3a 20 4f 75 74 20 6f 66 20 72 61 6e 67 65 20 5b 30 2c 20 32 5e 36 34 29 2e 0a>',
),
);
expect(isValid).toBe(false);
});
// it(
// 'profile',
// async () => {
// let count = 100;
// console.time('alignCoordByTrim');
// while (count--) {
// const file = getFixture('long-text.png');
// await alignCoordByTrim(file, {
// left: 440,
// top: 50,
// width: 200,
// height: 150,
// });
// }
// console.timeEnd('alignCoordByTrim');
// },
// 10 * 1000,
// );
});