feat(web): playwright integration (#5)

* feat: use a resizable layout

* feat: update visualizer

* fix: resize bug

* feat: add .query for playwright integration

* fix: ts error

packages/midscene/src/action/executor.ts

@@ -7,9 +7,7 @@ import {
   ExecutionTaskReturn,
   ExecutorContext,
 } from '@/types';
-import { actionDumpFileExt, getPkgInfo, writeDumpFile } from '@/utils';
-
-const logFileExt = actionDumpFileExt;
+import { getPkgInfo } from '@/utils';
 
 export class Executor {
   name: string;
@@ -95,7 +93,10 @@ export class Executor {
           element: previousFindOutput?.element,
         };
         if (task.type === 'Insight') {
-          assert(task.subType === 'find', `unsupported insight subType: ${task.subType}`);
+          assert(
+            task.subType === 'find' || task.subType === 'query',
+            `unsupported insight subType: ${task.subType}`,
+          );
           returnValue = await task.executor(param, executorContext);
           previousFindOutput = (returnValue as ExecutionTaskReturn<ExecutionTaskInsightFindOutput>)?.output;
         } else if (task.type === 'Action' || task.type === 'Planning') {
@@ -136,7 +137,7 @@ export class Executor {
     }
   }
 
-  dump() {
+  dump(): ExecutionDump {
     const dumpData: ExecutionDump = {
       sdkVersion: getPkgInfo().version,
       logTime: Date.now(),
@@ -144,9 +145,6 @@ export class Executor {
       description: this.description,
       tasks: this.tasks,
     };
-    const fileContent = JSON.stringify(dumpData, null, 2);
-    this.dumpFileName = this.dumpFileName || `pid-${process.pid}-${Date.now()}`;
-    const dumpPath = writeDumpFile(this.dumpFileName, logFileExt, fileContent);
-    return dumpPath;
+    return dumpData;
   }
 }

packages/midscene/src/automation/planning.ts

@@ -33,7 +33,9 @@ export function systemPromptToTaskPlanning(query: string) {
   * Input: 'Weather in Shanghai'
   * KeyboardPress: 'Enter'
   
-  Remember: The actions you composed MUST be based on the page context information you get. Instead of making up actions that are not related to the page context.
+  Remember: 
+  1. The actions you composed MUST be based on the page context information you get. Instead of making up actions that are not related to the page context.
+  2. In most cases, you should Find one element first, then do other actions on it. For example, alway Find one element, then hover on it. But if you think it's necessary to do other actions first (like global scroll, global key press), you can do that.
 
   If any error occurs during the task planning (like the page content and task are irrelevant, or the element mentioned does not exist at all), please return the error message with explanation in the errors field. The thoughts、prompts、error messages should all in the same language as the user query.
   

packages/midscene/src/types.ts

@@ -70,22 +70,12 @@ export interface AISectionParseResponse<DataShape> {
  * context
  */
 
-// export type ContextDescriberFn = () => Promise<{
-//   description: string;
-//   elementById: (id: string) => BaseElement;
-// }>;
-
 export abstract class UIContext<ElementType extends BaseElement = BaseElement> {
   abstract screenshotBase64: string;
 
   abstract content: ElementType[];
 
   abstract size: Size;
-
-  // abstract describer: () => Promise<{
-  //   description: string;
-  //   elementById: (id: string) => ElementType;
-  // }>;
 }
 
 /**
@@ -215,7 +205,7 @@ export interface ExecutionRecorderItem {
   timing?: string;
 }
 
-export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action';
+export type ExecutionTaskType = 'Planning' | 'Insight' | 'Action' | 'Assertion';
 
 export interface ExecutorContext {
   task: ExecutionTask;
@@ -291,22 +281,19 @@ export type ExecutionTaskInsightFind = ExecutionTask<ExecutionTaskInsightFindApp
 /*
 task - insight-extract
 */
-// export interface ExecutionTaskInsightExtractParam {
-//   dataDemand: InsightExtractParam;
-// }
+export interface ExecutionTaskInsightQueryParam {
+  dataDemand: InsightExtractParam;
+}
 
-// export interface ExecutionTaskInsightExtractOutput {
-//   data: any;
-// }
+export interface ExecutionTaskInsightQueryOutput {
+  data: any;
+}
 
-// export type ExecutionTaskInsightExtractApply = ExecutionTaskApply<
-//   'insight-extract', // TODO: remove task-extract ?
-//   ExecutionTaskInsightExtractParam
-// >;
+export type ExecutionTaskInsightQueryApply = ExecutionTaskApply<'Insight', ExecutionTaskInsightQueryParam>;
 
-// export type ExecutionTaskInsightExtract = ExecutionTask<ExecutionTaskInsightExtractApply>;
+export type ExecutionTaskInsightQuery = ExecutionTask<ExecutionTaskInsightQueryApply>;
 
-export type ExecutionTaskInsight = ExecutionTaskInsightFind; // | ExecutionTaskInsightExtract;
+// export type ExecutionTaskInsight = ExecutionTaskInsightFind; // | ExecutionTaskInsightExtract;
 
 /*
 task - action (i.e. interact) 

packages/midscene/src/utils.ts

@@ -38,7 +38,6 @@ export function getPkgInfo(): PkgInfo {
 let logDir = join(process.cwd(), './midscene_run/');
 let logEnvReady = false;
 export const insightDumpFileExt = 'insight-dump.json';
-export const actionDumpFileExt = 'action-dump.json';
 export const groupedActionDumpFileExt = 'all-logs.json';
 
 export function getDumpDir() {

packages/midscene/tests/executor/index.test.ts

@@ -100,9 +100,8 @@ describe('executor', () => {
     expect(tapperFn.mock.calls[0][1].element).toBe(element);
     expect(tapperFn.mock.calls[0][1].task).toBeTruthy();
 
-    executor.dump();
-    const latestFile = join(getDumpDir(), 'latest.action-dump.json');
-    expect(existsSync(latestFile)).toBeTruthy();
+    const dump = executor.dump();
+    expect(dump.logTime).toBeTruthy();
 
   }, {
     timeout: 999 * 1000,
@@ -134,11 +133,8 @@ describe('executor', () => {
     expect(tapperFn).toBeCalledTimes(0);
 
 
-    const dumpPath = initExecutor.dump();
-
-    const dumpJsonContent1 = JSON.parse(readFileSync(dumpPath, 'utf-8'));
-    expect(dumpJsonContent1.tasks.length).toBe(2);
-
+    const dumpContent1 = initExecutor.dump();
+    expect(dumpContent1.tasks.length).toBe(2);
 
     // append while running
     await Promise.all([
@@ -161,9 +157,8 @@ describe('executor', () => {
     expect(initExecutor.status).toBe('pending');
 
     // same dumpPath to append
-    initExecutor.dump();
-    const dumpJsonContent2 = JSON.parse(readFileSync(dumpPath, 'utf-8'));
-    expect(dumpJsonContent2.tasks.length).toBe(4);
+    const dumpContent2 = initExecutor.dump();
+    expect(dumpContent2.tasks.length).toBe(4);
   });
 
   // it('insight - run with error', async () => {

packages/midscene/tests/image/__snapshots__/index.test.ts.snap

@@ -27,6 +27,13 @@ exports[`image utils > imageInfo 1`] = `
 }
 `;
 
+exports[`image utils > jpeg + base64 + imageInfo 1`] = `
+{
+  "height": 905,
+  "width": 400,
+}
+`;
+
 exports[`image utils > trim image 1`] = `
 {
   "height": 70,

packages/midscene/tests/image/index.test.ts

@@ -24,6 +24,13 @@ describe('image utils', () => {
     expect(info).toMatchSnapshot();
   });
 
+  it('jpeg + base64 + imageInfo', async () => {
+    const image = getFixture('heytea.jpeg');
+    const base64 = base64Encoded(image);
+    const info = await imageInfoOfBase64(base64);
+    expect(info).toMatchSnapshot();
+  });
+
   it('trim image', async () => {
     const file = getFixture('long-text.png');
     const info = await trimImage(file);

packages/visualizer/package.json

@@ -15,25 +15,25 @@
   },
   "dependencies": {
     "@ant-design/icons": "5.3.7",
+    "@midscene/core": "workspace:*",
     "@modern-js/runtime": "^2.54.2",
     "antd": "5.17.3",
     "dayjs": "1.11.11",
     "pixi.js": "8.1.1",
     "react": "~18.2.0",
     "react-dom": "~18.2.0",
-    "@midscene/core": "workspace:*",
+    "react-resizable-panels": "2.0.22",
     "zustand": "4.5.2"
   },
-  "peerDependencies": {},
   "devDependencies": {
     "@modern-js/module-tools": "^2.54.2",
     "@modern-js/plugin-module-doc": "^2.33.1",
-    "react": "~18.2.0",
-    "react-dom": "~18.2.0",
     "@types/react": "~18.2.22",
     "@types/react-dom": "~18.2.7",
-    "typescript": "~5.0.4",
-    "rimraf": "~3.0.2"
+    "react": "~18.2.0",
+    "react-dom": "~18.2.0",
+    "rimraf": "~3.0.2",
+    "typescript": "~5.0.4"
   },
   "sideEffects": [
     "**/*.css",

packages/visualizer/src/component/common.less

@@ -18,4 +18,4 @@
 @weak-bg: #F3F3F3;
 
 @side-horizontal-padding: 10px;
-@side-vertical-spacing: 8px;
+@side-vertical-spacing: 10px;

packages/visualizer/src/component/detail-side.less

@@ -15,7 +15,7 @@
   }
 
   .meta-kv {
-    padding: @side-vertical-spacing @side-horizontal-padding;
+    padding: @side-vertical-spacing @side-horizontal-padding calc(@side-vertical-spacing + 4px);
     .meta {
       box-sizing: border-box;
       padding: 2px 0;

packages/visualizer/src/component/detail-side.tsx

@@ -7,6 +7,7 @@ import {
   BaseElement,
   ExecutionTaskAction,
   ExecutionTaskInsightFind,
+  ExecutionTaskInsightQuery,
   ExecutionTaskPlanning,
   UISection,
 } from '@midscene/core';
@@ -152,6 +153,10 @@ const DetailSide = (): JSX.Element => {
       </span>
     );
 
+    if (Array.isArray(data) || typeof data !== 'object') {
+      return <pre className="description-content">{JSON.stringify(data, undefined, 2)}</pre>;
+    }
+
     return Object.keys(data).map((key) => {
       const value = data[key];
       let content;
@@ -201,14 +206,20 @@ const DetailSide = (): JSX.Element => {
     taskParam = MetaKV({
       data: [
         { key: 'type', content: (task && typeStr(task)) || '' },
-        { key: 'userPrompt', content: (task as ExecutionTaskPlanning)?.param?.userPrompt },
+        { key: 'param', content: (task as ExecutionTaskPlanning)?.param?.userPrompt },
       ],
     });
   } else if (task?.type === 'Insight') {
     taskParam = MetaKV({
       data: [
         { key: 'type', content: (task && typeStr(task)) || '' },
-        { key: 'query', content: (task as ExecutionTaskInsightFind)?.param?.query },
+        {
+          key: 'param',
+          content: JSON.stringify(
+            (task as ExecutionTaskInsightFind)?.param?.query ||
+              (task as ExecutionTaskInsightQuery)?.param?.dataDemand,
+          ),
+        },
       ],
     });
   } else if (task?.type === 'Action') {
@@ -302,20 +313,14 @@ const DetailSide = (): JSX.Element => {
   ) : null;
 
   const dataCard = dump?.data ? (
-    <Card
-      liteMode={true}
-      title="Data Extracted"
-      onMouseEnter={noop}
-      onMouseLeave={noop}
-      content={<pre>{kv(dump.data)}</pre>}
-    ></Card>
+    <Card liteMode={true} onMouseEnter={noop} onMouseLeave={noop} content={<pre>{kv(dump.data)}</pre>}></Card>
   ) : null;
 
   const plans = (task as ExecutionTaskPlanning)?.output?.plans;
   let timelineData: TimelineItemProps[] = [];
   if (plans) {
     timelineData = timelineData.concat(
-      plans.map((item, index) => {
+      plans.map((item) => {
         return {
           color: '#06B1AB',
           children: (
@@ -338,10 +343,10 @@ const DetailSide = (): JSX.Element => {
       <PanelTitle title="Task Meta" />
       {metaKVElement}
       {/* Param  */}
-      <PanelTitle title="Task Param" />
+      <PanelTitle title="Param" />
       {taskParam}
       {/* Response */}
-      <PanelTitle title="Task Output" />
+      <PanelTitle title="Output" />
       <div className="item-list item-list-space-up">
         {errorSection}
         {dataCard}

packages/visualizer/src/component/global-hover-preview.less

@@ -0,0 +1,23 @@
+@import './common.less';
+
+@max-size: 400px;
+.global-hover-preview {
+  position: fixed;
+  display: block;
+  max-width: @max-size;
+  max-height: @max-size;
+  overflow: hidden;
+  z-index: 10;
+  text-align: center;
+  border: 1px solid @border-color;
+  box-sizing: border-box;
+  background: @side-bg;
+  box-shadow: 1px 1px 5px 0 rgba(0, 0, 0, 0.2);
+  
+  img {
+    max-width: @max-size;
+    max-height: @max-size;
+    width: auto;
+    height: auto;
+  }
+}

packages/visualizer/src/component/global-hover-preview.tsx

@@ -0,0 +1,50 @@
+import { useRef, useState } from 'react';
+import { useExecutionDump } from './store';
+import './global-hover-preview.less';
+
+const size = 400; // @max-size
+const GlobalHoverPreview = () => {
+  const wrapperRef = useRef<HTMLDivElement>(null);
+  const hoverTask = useExecutionDump((store) => store.hoverTask);
+  const hoverPreviewConfig = useExecutionDump((store) => store.hoverPreviewConfig);
+  const [imageW, setImageW] = useState(size);
+  const [imageH, setImageH] = useState(size);
+
+  const images = hoverTask?.recorder
+    ?.filter((item) => {
+      return item.screenshot;
+    })
+    .map((item) => item.screenshot);
+
+  const { x, y } = hoverPreviewConfig || {};
+  let left = 0;
+  let top = 0;
+
+  const shouldShow = images?.length && typeof x !== 'undefined' && typeof y !== 'undefined';
+  if (shouldShow) {
+    const { clientWidth, clientHeight } = document.body;
+    const widthInPractice = imageW >= imageH ? size : size * (imageW / imageH);
+    const heightInPractice = imageW >= imageH ? size * (imageH / imageW) : size;
+    left = x + widthInPractice > clientWidth ? clientWidth - widthInPractice : x;
+    top = y + heightInPractice > clientHeight ? clientHeight - heightInPractice : y;
+  }
+  // if x + size exceed the screen width, use (screenWidth - size) instead
+
+  return shouldShow ? (
+    <div className="global-hover-preview" style={{ left, top }} ref={wrapperRef}>
+      {images?.length ? (
+        <img
+          src={images[0]}
+          onLoad={(img) => {
+            const imgElement = img.target as HTMLImageElement;
+            const width = imgElement.naturalWidth;
+            const height = imgElement.naturalHeight;
+            setImageW(width);
+            setImageH(height);
+          }}
+        />
+      ) : null}
+    </div>
+  ) : null;
+};
+export default GlobalHoverPreview;

packages/visualizer/src/component/sidebar.less

@@ -61,7 +61,7 @@
     cursor: pointer;
     transition: .1s;
     // margin-bottom: 9px;
-    padding: 4px 0;
+    padding: 2px 0;
     
     &:hover {
       background: @hover-bg;
@@ -105,6 +105,10 @@
       color: rgb(58, 119, 58);
     }
 
+    .status-icon-fail {
+      color: rgb(255, 10, 10);
+    }
+
     .status-text {
       color: @weak-text;
     }

packages/visualizer/src/component/sidebar.tsx

@@ -16,7 +16,12 @@ import logo from './assets/logo-plain2.svg';
 import { useAllCurrentTasks, useExecutionDump } from '@/component/store';
 import { typeStr } from '@/utils';
 
-const SideItem = (props: { task: ExecutionTask; selected?: boolean; onClick?: () => void }): JSX.Element => {
+const SideItem = (props: {
+  task: ExecutionTask;
+  selected?: boolean;
+  onClick?: () => void;
+  onItemHover?: (task: ExecutionTask | null, x?: number, y?: number) => any;
+}): JSX.Element => {
   const { task, onClick, selected } = props;
 
   const selectedClass = selected ? 'selected' : '';
@@ -40,8 +45,22 @@ const SideItem = (props: { task: ExecutionTask; selected?: boolean; onClick?: ()
 
   const contentRow =
     task.type === 'Planning' ? <div className="side-item-content">{task.param?.userPrompt} </div> : null;
+  // add hover listener
   return (
-    <div className={`side-item ${selectedClass}`} onClick={onClick}>
+    <div
+      className={`side-item ${selectedClass}`}
+      onClick={onClick}
+      // collect x,y (refer to the body) for hover preview
+      onMouseEnter={(event) => {
+        const rect = event.currentTarget.getBoundingClientRect();
+        const x = rect.left + rect.width;
+        const y = rect.top;
+        props.onItemHover?.(task, x, y);
+      }}
+      onMouseLeave={() => {
+        props.onItemHover?.(null);
+      }}
+    >
       {' '}
       <div className={`side-item-name`}>
         <span className={`status-icon status-icon-${task.status}`}>{statusIcon}</span>
@@ -57,6 +76,8 @@ const Sidebar = (): JSX.Element => {
   const groupedDumps = useExecutionDump((store) => store.dump);
   const setActiveTask = useExecutionDump((store) => store.setActiveTask);
   const activeTask = useExecutionDump((store) => store.activeTask);
+  const setHoverTask = useExecutionDump((store) => store.setHoverTask);
+  const setHoverPreviewConfig = useExecutionDump((store) => store.setHoverPreviewConfig);
   // const selectedTaskIndex = useExecutionDump((store) => store.selectedTaskIndex);
   // const setSelectedTaskIndex = useExecutionDump((store) => store.setSelectedTaskIndex);
   const reset = useExecutionDump((store) => store.reset);
@@ -106,9 +127,17 @@ const Sidebar = (): JSX.Element => {
               task={task}
               selected={task === activeTask}
               onClick={() => {
-                console.log('click', task);
                 setActiveTask(task);
               }}
+              onItemHover={(hoverTask, x, y) => {
+                if (hoverTask && x && y) {
+                  setHoverPreviewConfig({ x, y });
+                  setHoverTask(hoverTask);
+                } else {
+                  setHoverPreviewConfig(null);
+                  setHoverTask(null);
+                }
+              }}
             />
           );
         });
@@ -117,9 +146,9 @@ const Sidebar = (): JSX.Element => {
           case 0:
             seperator = <div className="side-seperator side-seperator-space-up" />;
             break;
-          case group.executions.length - 1:
-            seperator = <div className="side-seperator side-seperator-space-down" />;
-            break;
+          // case group.executions.length - 1:
+          //   seperator = <div className="side-seperator side-seperator-space-down" />;
+          //   break;
           default:
             seperator = (
               <div className="side-seperator side-seperator-line side-seperator-space-up side-seperator-space-down" />
@@ -149,9 +178,14 @@ const Sidebar = (): JSX.Element => {
     <div className="side-bar">
       <div className="top-controls">
         <div className="brand" onClick={reset}>
-          <img src={logo} alt="Logo" style={{ width: 70, height: 70, margin: 'auto' }} onClick={() => {
-            location.reload();
-          }} />
+          <img
+            src={logo}
+            alt="Logo"
+            style={{ width: 70, height: 70, margin: 'auto' }}
+            onClick={() => {
+              location.reload();
+            }}
+          />
         </div>
         <div className="task-list">{sideList}</div>
         <div className="side-seperator side-seperator-line side-seperator-space-up" />

packages/visualizer/src/component/store.tsx

@@ -2,7 +2,6 @@ import { create } from 'zustand';
 import {
   InsightDump,
   BaseElement,
-  ExecutionDump,
   ExecutionTaskInsightFind,
   ExecutionTask,
   GroupedActionDump,
@@ -28,13 +27,18 @@ export const useExecutionDump = create<{
   dump: GroupedActionDump[] | null;
   setGroupedDump: (dump: GroupedActionDump[]) => void;
   activeTask: ExecutionTask | null;
-  // selectedTaskIndex: number;
   setActiveTask: (task: ExecutionTask) => void;
+  hoverTask: ExecutionTask | null;
+  setHoverTask: (task: ExecutionTask | null) => void;
+  hoverPreviewConfig: { x: number; y: number } | null;
+  setHoverPreviewConfig: (config: { x: number; y: number } | null) => void;
   reset: () => void;
-}>((set, get) => {
+}>((set) => {
   const initData = {
     dump: null,
     activeTask: null,
+    hoverTask: null,
+    hoverPreviewConfig: null,
   };
 
   const syncToInsightDump = (dump: InsightDump) => {
@@ -71,6 +75,21 @@ export const useExecutionDump = create<{
         resetInsightDump();
       }
     },
+    setHoverTask(task: ExecutionTask | null) {
+      set({ hoverTask: task });
+    },
+    setHoverPreviewConfig(config: { x: number; y: number } | null) {
+      if (config) {
+        set({
+          hoverPreviewConfig: {
+            x: Math.floor(config.x),
+            y: Math.floor(config.y),
+          },
+        });
+      } else {
+        set({ hoverPreviewConfig: null });
+      }
+    },
     reset: () => {
       set(initData);
       resetInsightDump();
@@ -110,8 +129,8 @@ export const useInsightDump = create<{
   return {
     ...initData,
     loadData: (data: InsightDump) => {
-      console.log('will load dump data');
-      console.log(data);
+      // console.log('will load dump data');
+      // console.log(data);
       set({
         _loadId: ++loadId,
         data,

packages/visualizer/src/component/timeline.less

@@ -18,21 +18,6 @@
     height: 100%;  
   }
 
-  .timeline-zoom-view {
-    z-index: 10;
-    text-align: center;
-    border: 1px solid @border-color;
-    box-sizing: border-box;
-    background: @side-bg;
-    position: absolute;
-    left: 0px;
-    top: calc(@base-height - 1);
-    
-    img {
-      width: 100%;
-    }
-  }
-
   canvas {
     width: 100%;
     height: 100%;

packages/visualizer/src/component/timeline.tsx

@@ -1,3 +1,4 @@
+/* eslint-disable max-lines */
 import { useEffect, useMemo, useRef, useState } from 'react';
 import * as PIXI from 'pixi.js';
 
@@ -5,7 +6,7 @@ import './timeline.less';
 import { ExecutionRecorderItem, ExecutionTask } from '@midscene/core';
 import { useAllCurrentTasks, useExecutionDump } from './store';
 
-export interface TimelineItem {
+interface TimelineItem {
   id: string;
   img: string;
   timeOffset: number;
@@ -15,12 +16,17 @@ export interface TimelineItem {
   height?: number;
 }
 
-export interface HighlightParam {
+interface HighlightParam {
   mouseX: number;
   mouseY: number;
   item: TimelineItem;
 }
 
+interface HighlightMask {
+  startMs: number;
+  endMs: number;
+}
+
 // Function to clone a sprite
 function cloneSprite(sprite: PIXI.Sprite) {
   const clonedSprite = new PIXI.Sprite(sprite.texture);
@@ -40,17 +46,70 @@ const TimelineWidget = (props: {
   onHighlight?: (param: HighlightParam) => any;
   onUnhighlight?: () => any;
   onTap?: (param: TimelineItem) => any;
+  highlightMask?: HighlightMask;
+  hoverMask?: HighlightMask;
 }): JSX.Element => {
   const domRef = useRef<HTMLDivElement>(null); // Should be HTMLDivElement not HTMLInputElement
   const app = useMemo<PIXI.Application>(() => new PIXI.Application(), []);
 
   const gridsContainer = useMemo(() => new PIXI.Container(), []);
   const screenshotsContainer = useMemo(() => new PIXI.Container(), []);
+  const highlightMaskContainer = useMemo(() => new PIXI.Container(), []);
+  const containerUpdaterRef = useRef(
+    // eslint-disable-next-line @typescript-eslint/no-empty-function
+    (_s: number | undefined, _e: number | undefined, _hs: number | undefined, _he: number | undefined) => {},
+  );
   const indicatorContainer = useMemo(() => new PIXI.Container(), []);
 
   const allScreenshots = props.screenshots || [];
   const maxTime = allScreenshots[allScreenshots.length - 1].timeOffset;
 
+  const sizeRatio = 2;
+
+  const titleBg = 0xdddddd; // @title-bg
+  const sideBg = 0xececec;
+  const gridTextColor = 0;
+  const shotBorderColor = 0x777777;
+  const gridLineColor = 0xcccccc; // @border-color
+  const gridHighlightColor = 0x06b1ab; // @main-blue
+  const timeContentFontSize = 20;
+  const commonPadding = 12;
+  const timeTextTop = commonPadding;
+  const timeTitleBottom = timeTextTop * 2 + timeContentFontSize;
+  const highlightMaskAlpha = 0.6;
+  const hoverMaskAlpha = 0.3;
+
+  const closestScreenshotItemOnXY = (x: number, _y: number) => {
+    // find out the screenshot that is closest to the mouse on the left
+    let closestScreenshot: TimelineItem | undefined; // already sorted
+    let closestIndex = -1;
+    for (let i = 0; i < allScreenshots.length; i++) {
+      const shot = allScreenshots[i];
+      if (shot.x! <= x) {
+        closestScreenshot = allScreenshots[i];
+        closestIndex = i;
+      } else {
+        break;
+      }
+    }
+    return {
+      closestScreenshot,
+      closestIndex,
+    };
+  };
+
+  useMemo(() => {
+    const { startMs, endMs } = props.highlightMask || {};
+    const { startMs: hoverStartMs, endMs: hoverEndMs } = props.hoverMask || {};
+    const fn = containerUpdaterRef.current;
+    fn(startMs, endMs, hoverStartMs, hoverEndMs);
+  }, [
+    props.highlightMask?.startMs,
+    props.highlightMask?.endMs,
+    props.hoverMask?.startMs,
+    props.hoverMask?.endMs,
+  ]);
+
   useEffect(() => {
     Promise.resolve(
       (async () => {
@@ -58,8 +117,6 @@ const TimelineWidget = (props: {
           return;
         }
 
-        const sizeRatio = 2;
-
         // width of domRef
         const { clientWidth, clientHeight } = domRef.current;
         const canvasWidth = clientWidth * sizeRatio;
@@ -67,7 +124,6 @@ const TimelineWidget = (props: {
 
         let singleGridWidth = 100 * sizeRatio;
         let gridCount = Math.floor(canvasWidth / singleGridWidth);
-        console.log('gridCount', gridCount, maxTime);
         const stepCandidate = [
           50, 100, 200, 300, 500, 1000, 2000, 3000, 5000, 6000, 8000, 9000, 10000, 20000, 30000, 40000, 60000,
           90000, 12000, 300000,
@@ -83,12 +139,7 @@ const TimelineWidget = (props: {
           singleGridWidth = Math.floor(singleGridWidth * (1 / gridRatio) * 0.9);
           gridCount = Math.floor(canvasWidth / singleGridWidth);
         }
-        // resize grid
-        // singleGridWidth = Math.floor((canvasWidth / maxTime) * timeStep);
 
-        console.log('timeStep', timeStep);
-
-        // const timeStep = Math.max(50, Math.round((maxTime - 50) / gridCount / 50) * 50);
         const leftForTimeOffset = (timeOffset: number) => {
           return Math.floor((singleGridWidth * timeOffset) / timeStep);
         };
@@ -96,22 +147,15 @@ const TimelineWidget = (props: {
           return Math.floor((left * timeStep) / singleGridWidth);
         };
 
-        const titleBg = 0xdddddd; // @title-bg
-        const sideBg = 0xececec;
-        const gridTextColor = 0;
-        const shotBorderColor = 0x777777;
-        const gridLineColor = 0xcccccc; // @border-color
-        const gridHighlightColor = 0x06b1ab; // @main-blue
-        const timeContentFontSize = 20;
-        const commonPadding = 12;
-        const timeTextTop = commonPadding;
-        const timeTitleBottom = timeTextTop * 2 + timeContentFontSize;
-
         await app.init({
           width: canvasWidth,
           height: canvasHeight,
-          background: sideBg,
+          backgroundColor: sideBg,
         });
+        if (!domRef.current) {
+          app.destroy();
+          return;
+        }
         domRef.current.replaceChildren(app.canvas);
 
         const pixiTextForNumber = (num: number) => {
@@ -200,24 +244,47 @@ const TimelineWidget = (props: {
           };
         });
 
-        const closestScreenshotItemOnXY = (x: number, _y: number) => {
-          // find out the screenshot that is closest to the mouse on the left
-          let closestScreenshot: TimelineItem | undefined; // already sorted
-          let closestIndex = -1;
-          for (let i = 0; i < allScreenshots.length; i++) {
-            const shot = allScreenshots[i];
-            if (shot.x! <= x) {
-              closestScreenshot = allScreenshots[i];
-              closestIndex = i;
-            } else {
-              break;
+        const highlightMaskUpdater = (
+          start: number | undefined,
+          end: number | undefined,
+          hoverStart: number | undefined,
+          hoverEnd: number | undefined,
+        ) => {
+          highlightMaskContainer.removeChildren();
+
+          const mask = (start: number | undefined, end: number | undefined, alpha: number) => {
+            if (typeof start === 'undefined' || typeof end === 'undefined' || end === 0) {
+              return;
             }
-          }
-          return {
-            closestScreenshot,
-            closestIndex,
+            const leftBorder = new PIXI.Graphics();
+            leftBorder.beginFill(gridHighlightColor, 1);
+            leftBorder.drawRect(leftForTimeOffset(start), 0, sizeRatio, canvasHeight);
+            leftBorder.endFill();
+            highlightMaskContainer.addChild(leftBorder);
+
+            const rightBorder = new PIXI.Graphics();
+            rightBorder.beginFill(gridHighlightColor, 1);
+            rightBorder.drawRect(leftForTimeOffset(end), 0, sizeRatio, canvasHeight);
+            rightBorder.endFill();
+            highlightMaskContainer.addChild(rightBorder);
+
+            const mask = new PIXI.Graphics();
+            mask.beginFill(gridHighlightColor, alpha);
+            mask.drawRect(
+              leftForTimeOffset(start),
+              0,
+              leftForTimeOffset(end) - leftForTimeOffset(start),
+              canvasHeight,
+            );
+            mask.endFill();
+            highlightMaskContainer.addChild(mask);
           };
+
+          mask(start, end, highlightMaskAlpha);
+          mask(hoverStart, hoverEnd, hoverMaskAlpha);
         };
+        highlightMaskUpdater(props.highlightMask?.startMs, props.highlightMask?.endMs, 0, 0);
+        containerUpdaterRef.current = highlightMaskUpdater;
 
         // keep tracking the position of the mouse moving above the canvas
         app.stage.interactive = true;
@@ -258,7 +325,7 @@ const TimelineWidget = (props: {
           // cursor line
           const indicator = new PIXI.Graphics();
           indicator.beginFill(gridHighlightColor, 1);
-          indicator.drawRect(x - 1, 0, 2, canvasHeight);
+          indicator.drawRect(x - 1, 0, 3, canvasHeight);
           indicator.endFill();
           indicatorContainer.addChild(indicator);
 
@@ -288,8 +355,8 @@ const TimelineWidget = (props: {
         };
 
         const onPointerTap = (event: PointerEvent) => {
-          const x = event.offsetX;
-          const y = event.offsetY;
+          const x = event.offsetX * sizeRatio;
+          const y = event.offsetY * sizeRatio;
           const { closestScreenshot } = closestScreenshotItemOnXY(x, y);
           if (closestScreenshot) {
             props.onTap?.(closestScreenshot);
@@ -297,6 +364,7 @@ const TimelineWidget = (props: {
         };
 
         app.stage.addChild(screenshotsContainer);
+        app.stage.addChild(highlightMaskContainer);
         app.stage.addChild(indicatorContainer);
 
         const canvas = app.view;
@@ -314,8 +382,11 @@ const Timeline = () => {
   const allTasks = useAllCurrentTasks();
   const wrapper = useRef<HTMLDivElement>(null);
   const setActiveTask = useExecutionDump((store) => store.setActiveTask);
-  const [highlightItem, setHighlightItem] = useState<TimelineItem | undefined>();
-  const [popupX, setPopupX] = useState(0);
+  const activeTask = useExecutionDump((store) => store.activeTask);
+  const hoverTask = useExecutionDump((store) => store.hoverTask);
+  const setHoverTask = useExecutionDump((store) => store.setHoverTask);
+  const setHoverPreviewConfig = useExecutionDump((store) => store.setHoverPreviewConfig);
+
   // should be first task time ?
   let startingTime = -1;
   let idCount = 1;
@@ -354,7 +425,6 @@ const Timeline = () => {
     .sort((a, b) => a.timeOffset - b.timeOffset);
 
   const itemOnTap = (item: TimelineItem) => {
-    console.log('onTap');
     const task = idTaskMap[item.id];
     if (task) {
       setActiveTask(task);
@@ -363,33 +433,53 @@ const Timeline = () => {
 
   const onHighlightItem = (param: HighlightParam) => {
     const { mouseX, item } = param;
-    setPopupX(mouseX);
-    setHighlightItem(item);
+    const refBounding = wrapper.current?.getBoundingClientRect();
+    const task = idTaskMap[item.id];
+    if (task) {
+      setHoverTask(task);
+      setHoverPreviewConfig({
+        x: mouseX + (refBounding?.left || 0),
+        y: (refBounding?.bottom || 1) - 1,
+      });
+    } else {
+      setHoverTask(null);
+      setHoverPreviewConfig(null);
+    }
   };
 
   const unhighlight = () => {
-    setHighlightItem(undefined);
+    setHoverTask(null);
+    setHoverPreviewConfig(null);
   };
 
-  const zoomViewWidth = '500px';
   // overall left of wrapper
-  const wrapperW = wrapper.current?.getBoundingClientRect().width || 0;
-  const left = Math.min(popupX, wrapperW - 500);
+
+  const maskConfigForTask = (task?: ExecutionTask | null): HighlightMask | undefined => {
+    if (!task) {
+      return undefined;
+    }
+    return task.timing?.start && task.timing?.end
+      ? {
+          startMs: task.timing.start - startingTime || 0,
+          endMs: task.timing.end - startingTime || 0,
+        }
+      : undefined;
+  };
+
+  const highlightMaskConfig = maskConfigForTask(activeTask);
+  const hoverMaskConfig = maskConfigForTask(hoverTask);
 
   return (
     <div className="timeline-wrapper" ref={wrapper}>
       <TimelineWidget
+        // key={dimensions.width}
         screenshots={allScreenshots}
         onTap={itemOnTap}
         onHighlight={onHighlightItem}
         onUnhighlight={unhighlight}
+        highlightMask={highlightMaskConfig}
+        hoverMask={hoverMaskConfig}
       />
-      <div
-        className="timeline-zoom-view"
-        style={{ width: zoomViewWidth, left, display: highlightItem ? 'block' : 'none' }}
-      >
-        {highlightItem ? <img src={highlightItem?.img} /> : null}
-      </div>
     </div>
   );
 };

packages/visualizer/src/index.less

@@ -7,6 +7,7 @@ body {
   padding: 0;
   margin: 0;
   font-family: PingFang SC, Hiragino Sans GB, Microsoft YaHei, Arial, sans-serif;
+  font-size: 14px;
 }
 
 // local debug
@@ -90,6 +91,7 @@ footer.mt-8{
     background: #F5F5F5;
     overflow-x: hidden;
     overflow-y: scroll;
+    border-left: 1px solid @border-color;
   }
     
   .main-side {
@@ -98,11 +100,10 @@ footer.mt-8{
     // margin: calc(@layout-space/2);
     // padding-left: @layout-padding;
     overflow-y: scroll;
-    border-right: 1px solid @border-color;
 
-    flex-basis: 380px; /* Set the fixed width */
-    flex-grow: 0;     /* Prevent it from growing */
-    flex-shrink: 0;   /* Prevent it from shrinking */
+    // flex-basis: 380px; /* Set the fixed width */
+    // flex-grow: 0;     /* Prevent it from growing */
+    // flex-shrink: 0;   /* Prevent it from shrinking */
   }
 
   .json-content {

packages/visualizer/src/index.tsx

@@ -1,22 +1,24 @@
 import './index.less';
-import { Layout, ConfigProvider, message, Upload, Button } from 'antd';
+import { ConfigProvider, message, Upload, Button } from 'antd';
 import type { UploadProps } from 'antd';
-import { useEffect } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { Helmet } from '@modern-js/runtime/head';
-import Sider from 'antd/es/layout/Sider';
+import { Panel, PanelGroup, PanelResizeHandle } from 'react-resizable-panels';
 import Timeline from './component/timeline';
 import DetailPanel from './component/detail-panel';
 import logo from './component/assets/logo-plain.svg';
+import GlobalHoverPreview from './component/global-hover-preview';
 import { useExecutionDump, useInsightDump } from '@/component/store';
 import DetailSide from '@/component/detail-side';
 import Sidebar from '@/component/sidebar';
 
-const { Content } = Layout;
 const { Dragger } = Upload;
 const Index = (): JSX.Element => {
   const executionDump = useExecutionDump((store) => store.dump);
   const setGroupedDump = useExecutionDump((store) => store.setGroupedDump);
   const reset = useExecutionDump((store) => store.reset);
+  const [mainLayoutChangeFlag, setMainLayoutChangeFlag] = useState(0);
+  const mainLayoutChangedRef = useRef(false);
 
   useEffect(() => {
     return () => {
@@ -24,6 +26,17 @@ const Index = (): JSX.Element => {
     };
   }, []);
 
+  useEffect(() => {
+    const onResize = () => {
+      setMainLayoutChangeFlag((prev) => prev + 1);
+    };
+    window.addEventListener('resize', onResize);
+
+    return () => {
+      window.removeEventListener('resize', onResize);
+    };
+  }, []);
+
   // TODO
   // const loadInsightDump = (dump: InsightDump) => {
   //   console.log('will convert insight dump to execution dump');
@@ -55,7 +68,7 @@ const Index = (): JSX.Element => {
         if (typeof result === 'string') {
           try {
             const data = JSON.parse(result);
-
+            // setMainLayoutChangeFlag((prev) => prev + 1);
             setGroupedDump(data);
             // if (ifActionFile) {
             // } else {
@@ -94,12 +107,12 @@ const Index = (): JSX.Element => {
           <p className="ant-upload-text">
             Click or drag the{' '}
             <b>
-              <i>.insight.json</i>
+              <i>.all-logs.json</i>
             </b>{' '}
-            or{' '}
+            {/* or{' '}
             <b>
               <i>.actions.json</i>
-            </b>{' '}
+            </b>{' '} */}
             file into this area.
           </p>
           <p className="ant-upload-text">
@@ -126,18 +139,49 @@ const Index = (): JSX.Element => {
     // dump
   } else {
     mainContent = (
-      <div className="main-right">
-        <Timeline />
-        <div className="main-content">
-          <div className="main-side">
-            <DetailSide />
-          </div>
+      <PanelGroup
+        autoSaveId="main-page-layout"
+        direction="horizontal"
+        onLayout={() => {
+          if (!mainLayoutChangedRef.current) {
+            setMainLayoutChangeFlag((prev) => prev + 1);
+          }
+        }}
+      >
+        <Panel maxSize={95}>
+          <Sidebar />
+        </Panel>
+        <PanelResizeHandle
+          onDragging={(isChanging) => {
+            if (mainLayoutChangedRef.current && !isChanging) {
+              // not changing anymore
+              setMainLayoutChangeFlag((prev) => prev + 1);
+            }
+            mainLayoutChangedRef.current = isChanging;
+          }}
+        />
+        <Panel defaultSize={80} maxSize={95}>
+          <div className="main-right">
+            <Timeline key={mainLayoutChangeFlag} />
+            <div className="main-content">
+              <PanelGroup autoSaveId="page-detail-layout" direction="horizontal">
+                <Panel maxSize={95}>
+                  <div className="main-side">
+                    <DetailSide />
+                  </div>
+                </Panel>
+                <PanelResizeHandle />
 
-          <div className="main-canvas-container">
-            <DetailPanel />
+                <Panel defaultSize={75} maxSize={95}>
+                  <div className="main-canvas-container">
+                    <DetailPanel />
+                  </div>
+                </Panel>
+              </PanelGroup>
+            </div>
           </div>
-        </div>
-      </div>
+        </Panel>
+      </PanelGroup>
     );
   }
 
@@ -157,16 +201,8 @@ const Index = (): JSX.Element => {
       <Helmet>
         <title>MidScene.js - Visualization Tool</title>
       </Helmet>
-      <div className="page-container">
-        <Layout style={{ height: '100' }}>
-          <Sider width={240} style={{ background: 'none', display: executionDump ? 'block' : 'none' }}>
-            <Sidebar />
-          </Sider>
-          <Layout>
-            <Content>{mainContent}</Content>
-          </Layout>
-        </Layout>
-      </div>
+      <div className="page-container">{mainContent}</div>
+      <GlobalHoverPreview />
     </ConfigProvider>
   );
 };

packages/visualizer/src/utils.ts

@@ -1,10 +1,10 @@
 /* eslint-disable @typescript-eslint/no-empty-function */
 import dayjs from 'dayjs';
-import type { ExecutionDump, ExecutionTaskInsight, InsightDump, ExecutionTask } from '@midscene/core';
+import type { ExecutionDump, ExecutionTaskInsightFind, InsightDump, ExecutionTask } from '@midscene/core';
 
 export function insightDumpToExecutionDump(insightDump: InsightDump | InsightDump[]): ExecutionDump {
-  const insightToTask = (insightDump: InsightDump): ExecutionTaskInsight => {
-    const task: ExecutionTaskInsight = {
+  const insightToTask = (insightDump: InsightDump): ExecutionTaskInsightFind => {
+    const task: ExecutionTaskInsightFind = {
       type: 'Insight',
       subType: insightDump.type === 'find' ? 'find' : 'extract',
       status: insightDump.error ? 'fail' : 'success',

packages/web-integration/src/playwright/actions.ts

@@ -2,13 +2,16 @@ import assert from 'assert';
 import type { Page as PlaywrightPage } from 'playwright';
 import Insight, {
   DumpSubscriber,
+  ExecutionDump,
   ExecutionRecorderItem,
   ExecutionTaskActionApply,
   ExecutionTaskApply,
   ExecutionTaskInsightFindApply,
+  ExecutionTaskInsightQueryApply,
   ExecutionTaskPlanningApply,
   Executor,
   InsightDump,
+  InsightExtractParam,
   PlanningAction,
   PlanningActionParamHover,
   PlanningActionParamInputOrKeyPress,
@@ -21,14 +24,14 @@ import { base64Encoded } from '@midscene/core/image';
 import { parseContextFromPlaywrightPage } from './utils';
 import { WebElementInfo } from './element';
 
-export class PlayWrightAI {
+export class PlayWrightActionAgent {
   page: PlaywrightPage;
 
   insight: Insight<WebElementInfo>;
 
   executor: Executor;
 
-  dumpPath?: string;
+  actionDump?: ExecutionDump;
 
   constructor(page: PlaywrightPage, opt?: { taskName?: string }) {
     this.page = page;
@@ -38,7 +41,7 @@ export class PlayWrightAI {
     this.executor = new Executor(opt?.taskName || 'MidScene - PlayWrightAI');
   }
 
-  private async screenshotTiming(timing: ExecutionRecorderItem['timing']) {
+  private async recordScreenshot(timing: ExecutionRecorderItem['timing']) {
     const file = getTmpFile('jpeg');
     await this.page.screenshot({
       ...commonScreenshotParam,
@@ -61,12 +64,12 @@ export class PlayWrightAI {
         const { task } = context;
         // set the recorder before executor in case of error
         task.recorder = recorder;
-        const shot = await this.screenshotTiming(`before ${task.type}`);
+        const shot = await this.recordScreenshot(`before ${task.type}`);
         recorder.push(shot);
         const result = await taskApply.executor(param, context, ...args);
         if (taskApply.type === 'Action') {
           await sleep(1000);
-          const shot2 = await this.screenshotTiming('after Action');
+          const shot2 = await this.recordScreenshot('after Action');
           recorder.push(shot2);
         }
         return result;
@@ -190,7 +193,6 @@ export class PlayWrightAI {
   }
 
   async action(userPrompt: string /* , actionInfo?: { actionType?: EventActions[number]['action'] } */) {
-    // TODO: what if multiple actions ?
     this.executor.description = userPrompt;
     const pageContext = await this.insight.contextRetrieverFn();
 
@@ -215,7 +217,7 @@ export class PlayWrightAI {
       // plan
       await this.executor.append(this.wrapExecutorWithScreenshot(planningTask));
       await this.executor.flush();
-      this.dumpPath = this.executor.dump();
+      this.actionDump = this.executor.dump();
 
       // append tasks
       const executables = await this.convertPlanToExecutable(plans);
@@ -223,7 +225,7 @@ export class PlayWrightAI {
 
       // flush actions
       await this.executor.flush();
-      this.executor.dump();
+      this.actionDump = this.executor.dump();
 
       assert(
         this.executor.status !== 'error',
@@ -231,9 +233,44 @@ export class PlayWrightAI {
       );
     } catch (e: any) {
       // keep the dump before throwing
-      this.dumpPath = this.executor.dump();
+      this.actionDump = this.executor.dump();
       const err = new Error(e.message, { cause: e });
       throw err;
     }
   }
+
+  async query(demand: InsightExtractParam) {
+    this.executor.description = JSON.stringify(demand);
+    let data: any;
+    const queryTask: ExecutionTaskInsightQueryApply = {
+      type: 'Insight',
+      subType: 'query',
+      param: {
+        dataDemand: demand,
+      },
+      executor: async (param) => {
+        let insightDump: InsightDump | undefined;
+        const dumpCollector: DumpSubscriber = (dump) => {
+          insightDump = dump;
+        };
+        this.insight.onceDumpUpdatedFn = dumpCollector;
+        data = await this.insight.extract<any>(param.dataDemand);
+        return {
+          output: data,
+          log: { dump: insightDump },
+        };
+      },
+    };
+    try {
+      await this.executor.append(this.wrapExecutorWithScreenshot(queryTask));
+      await this.executor.flush();
+      this.actionDump = this.executor.dump();
+    } catch (e: any) {
+      // keep the dump before throwing
+      this.actionDump = this.executor.dump();
+      const err = new Error(e.message, { cause: e });
+      throw err;
+    }
+    return data;
+  }
 }

packages/web-integration/src/playwright/index.ts

@@ -1,74 +1,120 @@
-import { readFileSync } from 'fs';
 import { TestInfo, TestType } from '@playwright/test';
-import { GroupedActionDump } from '@midscene/core';
+import { ExecutionDump, GroupedActionDump } from '@midscene/core';
 import { groupedActionDumpFileExt, writeDumpFile } from '@midscene/core/utils';
-import { PlayWrightAI } from './actions';
+import { PlayWrightActionAgent } from './actions';
 
-export { PlayWrightAI } from './actions';
+export { PlayWrightActionAgent } from './actions';
 
 export type APITestType = Pick<TestType<any, any>, 'step'>;
 
-const instanceKeyName = 'midscene-ai-instance';
-
-const actionDumps: GroupedActionDump[] = [];
-const writeOutActionDumps = () => {
-  writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(actionDumps));
-};
-
-/**
- * A helper function to generate a playwright fixture for ai(). Can be used in
- * a playwright setup
- */
-
 export const PlaywrightAiFixture = () => {
-  return {
-    ai: async ({ page }: any, use: any, testInfo: TestInfo) => {
-      let groupName: string;
-      let caseName: string;
-      const titlePath = [...testInfo.titlePath];
-      // use the last item of testInfo.titlePath() as the caseName, join the previous ones with ">" as groupName
-      if (titlePath.length > 1) {
-        caseName = titlePath.pop()!;
-        groupName = titlePath.join(' > ');
-      } else if (titlePath.length === 1) {
-        caseName = titlePath[0];
-        groupName = caseName;
-      } else {
-        caseName = 'unnamed';
-        groupName = 'unnamed';
-      }
+  const dumps: GroupedActionDump[] = [];
 
-      // find the GroupedActionDump in actionDumps or create a new one
-      let actionDump = actionDumps.find((dump) => dump.groupName === groupName);
-      if (!actionDump) {
-        actionDump = {
-          groupName,
-          executions: [],
-        };
-        actionDumps.push(actionDump);
-      }
-
-      const wrapped = async (task: string /* options: any */) => {
-        const aiInstance = page[instanceKeyName] ?? new PlayWrightAI(page, { taskName: caseName });
-        let error: Error | undefined;
-        try {
-          await aiInstance.action(task);
-        } catch (e: any) {
-          error = e;
-        }
-        if (aiInstance.dumpPath) {
-          actionDump!.executions.push(JSON.parse(readFileSync(aiInstance.dumpPath, 'utf8')));
-          writeOutActionDumps();
-        }
-        if (error) {
-          throw new Error(error.message, { cause: error });
-        }
+  const appendDump = (groupName: string, execution: ExecutionDump) => {
+    let currentDump = dumps.find((dump) => dump.groupName === groupName);
+    if (!currentDump) {
+      currentDump = {
+        groupName,
+        executions: [],
       };
-      await use(wrapped);
+      dumps.push(currentDump);
+    }
+    currentDump.executions.push(execution);
+  };
+
+  const writeOutActionDumps = () => {
+    writeDumpFile(`playwright-${process.pid}`, groupedActionDumpFileExt, JSON.stringify(dumps));
+  };
+
+  const groupAndCaseForTest = (testInfo: TestInfo) => {
+    let groupName: string;
+    let caseName: string;
+    const titlePath = [...testInfo.titlePath];
+
+    if (titlePath.length > 1) {
+      caseName = titlePath.pop()!;
+      groupName = titlePath.join(' > ');
+    } else if (titlePath.length === 1) {
+      caseName = titlePath[0];
+      groupName = caseName;
+    } else {
+      caseName = 'unnamed';
+      groupName = 'unnamed';
+    }
+    return { groupName, caseName };
+  };
+
+  const aiAction = async (page: any, testInfo: TestInfo, taskPrompt: string) => {
+    const { groupName, caseName } = groupAndCaseForTest(testInfo);
+
+    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+    let error: Error | undefined;
+    try {
+      await actionAgent.action(taskPrompt);
+    } catch (e: any) {
+      error = e;
+    }
+    if (actionAgent.actionDump) {
+      appendDump(groupName, actionAgent.actionDump);
+      writeOutActionDumps();
+    }
+    if (error) {
+      // playwright cli won't print error cause, so we print it here
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+  };
+
+  const aiQuery = async (page: any, testInfo: TestInfo, demand: any) => {
+    const { groupName, caseName } = groupAndCaseForTest(testInfo);
+
+    const actionAgent = new PlayWrightActionAgent(page, { taskName: caseName });
+    let error: Error | undefined;
+    let result: any;
+    try {
+      result = await actionAgent.query(demand);
+    } catch (e: any) {
+      error = e;
+    }
+    if (actionAgent.actionDump) {
+      appendDump(groupName, actionAgent.actionDump);
+      writeOutActionDumps();
+    }
+    if (error) {
+      // playwright cli won't print error cause, so we print it here
+      console.error(error);
+      throw new Error(error.message, { cause: error });
+    }
+    return result;
+  };
+
+  return {
+    // shortcut
+    ai: async ({ page }: any, use: any, testInfo: TestInfo) => {
+      await use(async (taskPrompt: string, type = 'action') => {
+        if (type === 'action') {
+          return aiAction(page, testInfo, taskPrompt);
+        } else if (type === 'query') {
+          return aiQuery(page, testInfo, taskPrompt);
+        }
+        throw new Error(`Unknown or Unsupported task type: ${type}, only support 'action' or 'query'`);
+      });
+    },
+    aiAction: async ({ page }: any, use: any, testInfo: TestInfo) => {
+      await use(async (taskPrompt: string) => {
+        await aiAction(page, testInfo, taskPrompt);
+      });
+    },
+    aiQuery: async ({ page }: any, use: any, testInfo: TestInfo) => {
+      await use(async function (demand: any) {
+        return aiQuery(page, testInfo, demand);
+      });
     },
   };
 };
 
 export type PlayWrightAiFixtureType = {
-  ai: (task: string | string[]) => ReturnType<PlayWrightAI['action']>;
+  ai: <T = any>(prompt: string, type?: 'action' | 'query') => Promise<T>;
+  aiAction: (taskPrompt: string) => ReturnType<PlayWrightActionAgent['action']>;
+  aiQuery: <T = any>(demand: any) => Promise<T>;
 };

packages/web-integration/src/playwright/utils.ts

@@ -5,7 +5,7 @@ import path from 'path';
 import type { Page as PlaywrightPage } from 'playwright';
 import { Page } from 'puppeteer';
 import { UIContext, PlaywrightParserOpt } from '@midscene/core';
-import { alignCoordByTrim, base64Encoded, imageInfo } from '@midscene/core/image';
+import { alignCoordByTrim, base64Encoded, imageInfo, imageInfoOfBase64 } from '@midscene/core/image';
 import { getTmpFile } from '@midscene/core/utils';
 import { WebElementInfo, WebElementInfoType } from './element';
 
@@ -14,16 +14,15 @@ export async function parseContextFromPlaywrightPage(
   _opt?: PlaywrightParserOpt,
 ): Promise<UIContext<WebElementInfo>> {
   assert(page, 'page is required');
-  const file = getTmpFile('jpeg');
+  const file = '/Users/bytedance/workspace/midscene/packages/midscene/tests/fixtures/heytea.jpeg'; // getTmpFile('jpeg');
   await page.screenshot({ path: file, type: 'jpeg', quality: 75 });
   const screenshotBuffer = readFileSync(file);
   const screenshotBase64 = base64Encoded(file);
   const captureElementSnapshot = await getElementInfosFromPage(page);
   // align element
   const elementsInfo = await alignElements(screenshotBuffer, captureElementSnapshot, page);
-  const size = await imageInfo(screenshotBase64);
+  const size = await imageInfoOfBase64(screenshotBase64);
 
-  
   return {
     content: elementsInfo,
     size,

packages/web-integration/tests/e2e/ai-auto-todo.spec.ts

@@ -1,10 +1,11 @@
+import { expect } from 'playwright/test';
 import { test } from './fixture';
 
 test.beforeEach(async ({ page }) => {
   await page.goto('https://todomvc.com/examples/react/dist/');
 });
 
-test('ai todo', async ({ ai }) => {
+test('ai todo', async ({ ai, aiQuery }) => {
   await ai('Enter "Learn JS today" in the task box, then press Enter to create');
   await ai('Enter "Learn Rust tomorrow" in the task box, then press Enter to create');
   await ai('Enter "Learning AI the day after tomorrow" in the task box, then press Enter to create');
@@ -13,4 +14,11 @@ test('ai todo', async ({ ai }) => {
   );
   await ai('Click the check button to the left of the second task');
   await ai('Click the completed Status button below the task list');
+
+  const taskList = await aiQuery<string[]>('string[], tasks in the list');
+  expect(taskList.length).toBe(1);
+  expect(taskList[0]).toBe('Learning AI the day after tomorrow');
+
+  const placeholder = await ai('string, return the placeholder text in the input box', 'query');
+  expect(placeholder).toBe('What needs to be done?');
 });

pnpm-lock.yaml

@@ -147,6 +147,9 @@ importers:
       react-dom:
         specifier: ~18.2.0
         version: 18.2.0(react@18.2.0)
+      react-resizable-panels:
+        specifier: 2.0.22
+        version: 2.0.22(react-dom@18.2.0(react@18.2.0))(react@18.2.0)
       zustand:
         specifier: 4.5.2
         version: 4.5.2(@types/react@18.2.79)(immer@9.0.21)(react@18.2.0)
@@ -7956,6 +7959,12 @@ packages:
     resolution: {integrity: sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==}
     engines: {node: '>=0.10.0'}
 
+  react-resizable-panels@2.0.22:
+    resolution: {integrity: sha512-G8x8o7wjQxCG+iF4x4ngKVBpe0CY+DAZ/SaiDoqBEt0yuKJe9OE/VVYMBMMugQ3GyQ65NnSJt23tujlaZZe75A==}
+    peerDependencies:
+      react: ^16.14.0 || ^17.0.0 || ^18.0.0
+      react-dom: ^16.14.0 || ^17.0.0 || ^18.0.0
+
   react-router-dom@6.11.1:
     resolution: {integrity: sha512-dPC2MhoPeTQ1YUOt5uIK376SMNWbwUxYRWk2ZmTT4fZfwlOvabF8uduRKKJIyfkCZvMgiF0GSCQckmkGGijIrg==}
     engines: {node: '>=14'}
@@ -19841,6 +19850,11 @@ snapshots:
 
   react-refresh@0.14.2: {}
 
+  react-resizable-panels@2.0.22(react-dom@18.2.0(react@18.2.0))(react@18.2.0):
+    dependencies:
+      react: 18.2.0
+      react-dom: 18.2.0(react@18.2.0)
+
   react-router-dom@6.11.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0):
     dependencies:
       '@remix-run/router': 1.6.1