fix(action): keyboard and div extracting (#304)

* fix: clearInput in browser

* fix: input issue on mac

* fix: playwright input

* fix: issue when collecting fixed item
---------

Co-authored-by: zhouxiao.shaw <zhouxiao.shaw@bytedance.com>

* fix: extract fixed style content

* fix: extract fixed style content

* fix: customize window size in yaml

packages/cli/src/index.ts

@@ -1,4 +1,6 @@
-import 'dotenv/config';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import dotenv from 'dotenv';
 import { matchYamlFiles, parseProcessArgs } from './cli-utils';
 import { playYamlFiles } from './yaml-runner';
 
@@ -9,6 +11,12 @@ Promise.resolve(
     const welcome = '\nWelcome to @midscene/cli\n';
     console.log(welcome);
 
+    const dotEnvConfigFile = join(process.cwd(), '.env');
+    if (existsSync(dotEnvConfigFile)) {
+      console.log(`loading .env file from ${dotEnvConfigFile}`);
+      dotenv.config({ path: dotEnvConfigFile });
+    }
+
     if (options.url) {
       console.error(
         'the cli mode is no longer supported, please use yaml file instead. See https://midscenejs.com/automate-with-scripts-in-yaml for more information. Sorry for the inconvenience.',

packages/web-integration/src/chrome-extension/page.ts

@@ -373,16 +373,13 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
     });
 
     await this.sendCommandToDebugger('Input.dispatchKeyEvent', {
-      type: 'keyDown',
-      key: 'Backspace',
-      code: 'Backspace',
+      type: 'keyUp',
+      commands: ['selectAll'],
     });
 
-    await this.sendCommandToDebugger('Input.dispatchKeyEvent', {
-      type: 'keyUp',
-      key: 'Backspace',
-      code: 'Backspace',
-    });
+    await sleep(100);
+
+    await this.keyboard.press('Backspace');
   }
 
   mouse = {

packages/web-integration/src/extractor/util.ts

@@ -295,6 +295,7 @@ export function visibleRect(
     return false;
   }
 
+  // check if the element is hidden by an ancestor
   let parent: HTMLElement | Node | null = el;
   while (parent && parent !== document.body) {
     if (!(parent instanceof HTMLElement)) {
@@ -319,6 +320,10 @@ export function visibleRect(
         return false;
       }
     }
+    // if the parent is a fixed element, stop the search
+    if (parentStyle.position === 'fixed') {
+      break;
+    }
     parent = parent.parentElement;
   }
 

packages/web-integration/src/puppeteer/agent-launcher.ts

@@ -80,6 +80,8 @@ export async function puppeteerAgentForTarget(
       ...(isWindows ? [] : ['--no-sandbox', '--disable-setuid-sandbox']),
       '--disable-features=PasswordLeakDetection',
       '--disable-save-password-bubble',
+      '--start-maximized',
+      `--window-size=${width},${height}`,
     ],
   });
   freeFn.push({

packages/web-integration/src/puppeteer/base-page.ts

@@ -1,5 +1,5 @@
 import type { Point, Size } from '@midscene/core';
-import { getTmpFile } from '@midscene/core/utils';
+import { getTmpFile, sleep } from '@midscene/core/utils';
 import { base64Encoded } from '@midscene/shared/img';
 import type { Page as PlaywrightPage } from 'playwright';
 import type { Page as PuppeteerPage } from 'puppeteer';
@@ -72,9 +72,14 @@ export class Page<
 
   get mouse() {
     return {
-      click: async (x: number, y: number, options?: { button: MouseButton }) =>
+      click: async (
+        x: number,
+        y: number,
+        options?: { button?: MouseButton; count?: number },
+      ) =>
         this.underlyingPage.mouse.click(x, y, {
           button: options?.button || 'left',
+          count: options?.count || 1,
         }),
       wheel: async (deltaX: number, deltaY: number) => {
         if (this.pageType === 'puppeteer') {
@@ -111,18 +116,26 @@ export class Page<
       return;
     }
 
-    await this.mouse.click(element.center[0], element.center[1]);
-
     const isMac = process.platform === 'darwin';
     if (isMac) {
-      await this.underlyingPage.keyboard.down('Meta');
-      await this.underlyingPage.keyboard.press('a');
-      await this.underlyingPage.keyboard.up('Meta');
+      if (this.pageType === 'puppeteer') {
+        // https://github.com/segment-boneyard/nightmare/issues/810#issuecomment-452669866
+        await this.mouse.click(element.center[0], element.center[1], {
+          count: 3,
+        });
+      } else {
+        await this.mouse.click(element.center[0], element.center[1]);
+        await this.underlyingPage.keyboard.down('Meta');
+        await this.underlyingPage.keyboard.press('a');
+        await this.underlyingPage.keyboard.up('Meta');
+      }
     } else {
+      await this.mouse.click(element.center[0], element.center[1]);
       await this.underlyingPage.keyboard.down('Control');
       await this.underlyingPage.keyboard.press('a');
       await this.underlyingPage.keyboard.up('Control');
     }
+    await sleep(100);
     await this.keyboard.press('Backspace');
   }
 

packages/web-integration/tests/ai/web/playwright/ai-auto-todo.spec.ts

@@ -12,7 +12,8 @@ test('ai todo', async ({ ai, aiQuery }) => {
     test.setTimeout(1000 * 50);
   }
 
-  await ai('Enter "Learn" in the task box, don\'t press enter');
+  await ai('Enter "Happy Birthday" in the task box');
+  await ai('Enter "Learn" in the task box');
 
   await ai(
     'Add "JS today" to base on the existing content(important) of the task box, then press enter',

packages/web-integration/tests/ai/web/puppeteer/showcase.test.ts

@@ -85,13 +85,14 @@ describe(
     });
 
     it('search engine', async () => {
-      const { originPage, reset } = await launchPage('https://www.bing.com/');
+      const { originPage, reset } = await launchPage('https://www.baidu.com/');
       const mid = new PuppeteerAgent(originPage);
+      await mid.aiAction('type "AI 101" in search box');
       await mid.aiAction(
-        'type "AI 101" in search box, hit Enter, wait 2s, click the second result, wait 4s',
+        'type "Hello world" in search box, hit Enter, wait 2s, click the second result, wait 4s',
       );
 
-      await mid.aiWaitFor('there are some search results');
+      await mid.aiWaitFor('there are some search results about "Hello world"');
 
       await reset();
     });

packages/web-integration/tests/unit-test/__snapshots__/web-extractor.test.ts.snap

@@ -559,6 +559,27 @@ exports[`extractor > basic 1`] = `
     },
     "content": "hidden label",
   },
+  {
+    "attributes": {
+      "htmlTagName": "<div>",
+      "nodeType": "TEXT Node",
+    },
+    "content": "i am fixed child content",
+  },
+  {
+    "attributes": {
+      "htmlTagName": "<span>",
+      "nodeType": "TEXT Node",
+    },
+    "content": "abcd efg",
+  },
+  {
+    "attributes": {
+      "htmlTagName": "<div>",
+      "nodeType": "TEXT Node",
+    },
+    "content": "content editable div content. We should collect the parent.",
+  },
   {
     "attributes": {
       "htmlTagName": "<div>",

packages/web-integration/tests/unit-test/fixtures/web-extractor/index.html

@@ -293,6 +293,23 @@
         <span>hidden label</span>
     </label>
 
+    <!-- element under position: fixed -->
+    <div style="position: absolute; top: -100px; left: -100px; width: 100px; height: 100px; background-color: #ccc; overflow: hidden;">
+        <div style="position: fixed; top: 200px; right: 40px; width: 200px; height: 100px; background-color: #EEE;">
+            <div>i am fixed child content</div>
+        </div>
+    </div>
+
+    <!-- div with content editable -->
+    <div contenteditable="true" class="content-editable-div" style="position: relative; width: 300px; height: 100px; background-color: #ccc;">
+        <!-- I am a content editable div. -->
+        <span>abcd efg</span>
+        <div style="position: absolute; left: 0px; bottom: 0; width: 100%; height: 50px; background-color: #EEE;">
+            content editable div content. We should collect the parent.
+        </div>
+    </div>
+
+
     <!-- absolute parent with width 0 -->
     <div style="position: absolute; top: 0px; left: 0; width: 100%; height: 0;">
         <div>absolute child content</div>