fix(report): show highlight elment when cache hit (#723)

* fix(report): show highlight elment when cache hit

* fix(core): replay animation

---------

Co-authored-by: yutao <yutao.tao@bytedance.com>

apps/report/src/components/detail-panel.tsx

@@ -38,8 +38,7 @@ const DetailPanel = (): JSX.Element => {
     (store) => store._executionDumpLoadId,
   );
   const activeTask = useExecutionDump((store) => store.activeTask);
-  const blackboardViewAvailable =
-    Boolean(activeTask?.pageContext) && insightDump;
+  const blackboardViewAvailable = Boolean(activeTask?.pageContext);
   const [preferredViewType, setViewType] = useState(VIEW_TYPE_REPLAY);
   const animationScripts = useExecutionDump(
     (store) => store.activeExecutionAnimation,
@@ -87,12 +86,21 @@ const DetailPanel = (): JSX.Element => {
       </div>
     );
   } else if (viewType === VIEW_TYPE_BLACKBOARD) {
-    if (blackboardViewAvailable && insightDump) {
+    if (blackboardViewAvailable) {
+      let highlightElements;
+
+      if (insightDump?.matchedElement) {
+        highlightElements = insightDump?.matchedElement;
+      } else {
+        highlightElements = activeTask.output.element // hit cache
+          ? [activeTask.output.element]
+          : [];
+      }
       content = (
         <Blackboard
           uiContext={activeTask.pageContext}
-          highlightElements={insightDump.matchedElement}
-          highlightRect={insightDump!.taskInfo?.searchArea}
+          highlightElements={highlightElements}
+          highlightRect={insightDump?.taskInfo?.searchArea}
           key={`${dumpId}`}
         />
       );

packages/visualizer/src/component/player.tsx

@@ -9,7 +9,7 @@ import {
   DownloadOutlined,
   LoadingOutlined,
 } from '@ant-design/icons';
-import type { BaseElement, Rect } from '@midscene/core';
+import type { BaseElement, LocateResultElement, Rect } from '@midscene/core';
 import { Spin, Tooltip } from 'antd';
 import { rectMarkForItem } from './blackboard';
 import { getTextureFromCache, loadTexture } from './pixi-loader';
@@ -444,7 +444,7 @@ export function Player(props?: {
 
   const insightElementsAnimation = async (
     elements: BaseElement[],
-    highlightElements: BaseElement[],
+    highlightElements: (BaseElement | LocateResultElement)[],
     searchArea: Rect | undefined,
     duration: number,
     frame: FrameFn,
@@ -516,7 +516,7 @@ export function Player(props?: {
           highlightElements.map((element) => {
             const [insightMarkGraphic] = rectMarkForItem(
               element.rect,
-              element.content,
+              (element as BaseElement).content || '',
               'highlight',
             );
             insightMarkGraphic.alpha = 1;
@@ -615,18 +615,20 @@ export function Player(props?: {
           if (item.type === 'sleep') {
             await sleep(item.duration);
           } else if (item.type === 'insight') {
-            if (!item.insightDump || !item.img) {
-              throw new Error('insight dump or img is required');
+            if (!item.img) {
+              throw new Error('img is required');
             }
             currentImg.current = item.img;
             await repaintImage();
 
             const elements = item.context?.content || [];
-            const highlightElements = item.insightDump.matchedElement;
+            const highlightElements = item.highlightElement
+              ? [item.highlightElement]
+              : [];
             await insightElementsAnimation(
               elements,
               highlightElements,
-              item.insightDump.taskInfo?.searchArea,
+              item.searchArea,
               item.duration,
               frame,
             );

packages/visualizer/src/component/replay-scripts.tsx

@@ -9,7 +9,7 @@ import type {
   ExecutionTaskInsightLocate,
   ExecutionTaskPlanning,
   GroupedActionDump,
-  InsightDump,
+  LocateResultElement,
   Rect,
   UIContext,
 } from '@midscene/core';
@@ -38,8 +38,9 @@ export interface AnimationScript {
     | 'sleep';
   img?: string;
   camera?: TargetCameraState;
-  insightDump?: InsightDump;
   context?: UIContext;
+  highlightElement?: LocateResultElement;
+  searchArea?: Rect;
   duration: number;
   insightCameraDuration?: number;
   title?: string;
@@ -117,8 +118,8 @@ export const mergeTwoCameraState = (
 
 export interface ReplayScriptsInfo {
   scripts: AnimationScript[];
-  width: number;
-  height: number;
+  width?: number;
+  height?: number;
   sdkVersion?: string;
   modelName?: string;
   modelDescription?: string;
@@ -128,13 +129,25 @@ export const allScriptsFromDump = (
   dump: GroupedActionDump,
 ): ReplayScriptsInfo | null => {
   // find out the width and height of the screenshot
-  let width = 0;
-  let height = 0;
-  let sdkVersion = '';
-  let modelName = '';
-  let modelDescription = '';
+  let width: number | undefined = undefined;
+  let height: number | undefined = undefined;
+  let sdkVersion: string | undefined = undefined;
+  let modelName: string | undefined = undefined;
+  let modelDescription: string | undefined = undefined;
 
   dump.executions.forEach((execution) => {
+    if (execution.sdkVersion) {
+      sdkVersion = execution.sdkVersion;
+    }
+
+    if (execution.model_name) {
+      modelName = execution.model_name;
+    }
+
+    if (execution.model_description) {
+      modelDescription = execution.model_description;
+    }
+
     execution.tasks.forEach((task) => {
       const insightTask = task as ExecutionTaskInsightLocate;
       if (insightTask.pageContext?.size?.width) {
@@ -142,28 +155,33 @@ export const allScriptsFromDump = (
         height = insightTask.pageContext.size.height;
       }
 
-      if (insightTask.log?.dump?.sdkVersion) {
+      if (insightTask.log?.dump?.sdkVersion && !sdkVersion) {
         sdkVersion = insightTask.log.dump.sdkVersion;
       }
 
-      if (insightTask.log?.dump?.model_name) {
+      if (insightTask.log?.dump?.model_name && !modelName) {
         modelName = insightTask.log.dump.model_name;
       }
 
-      if (insightTask.log?.dump?.model_description) {
+      if (insightTask.log?.dump?.model_description && !modelDescription) {
         modelDescription = insightTask.log.dump.model_description;
       }
     });
   });
 
   if (!width || !height) {
-    console.error('width or height is missing in dump file');
-    return null;
+    console.warn('width or height is missing in dump file');
+    return {
+      scripts: [],
+      sdkVersion,
+      modelName,
+      modelDescription,
+    };
   }
 
   const allScripts: AnimationScript[] = [];
   dump.executions.forEach((execution) => {
-    const scripts = generateAnimationScripts(execution, -1, width, height);
+    const scripts = generateAnimationScripts(execution, -1, width!, height!);
     if (scripts) {
       allScripts.push(...scripts);
     }
@@ -297,8 +315,8 @@ export const generateAnimationScripts = (
         };
       }
       const context = insightTask.pageContext;
-      if (insightTask.log?.dump && context?.screenshotBase64) {
-        const insightDump = insightTask.log.dump;
+      if (context?.screenshotBase64) {
+        const insightDump = insightTask.log?.dump;
         const insightContentLength = context.content.length;
 
         if (context.screenshotBase64) {
@@ -328,8 +346,9 @@ export const generateAnimationScripts = (
           type: 'insight',
           img: context.screenshotBase64,
           context: context,
-          insightDump: insightDump,
           camera: cameraState,
+          highlightElement: insightTask.output?.element || undefined,
+          searchArea: insightDump?.taskInfo?.searchArea,
           duration:
             insightContentLength > 20 ? locateDuration : locateDuration * 0.5,
           insightCameraDuration: locateDuration,