feat(workflow): Add readme, nx, deploy first release documentation (#1)

* docs(readme): add Added feature introduction

* docs(readme): optimize format

* chore: modify readme and doc content

* chore: modify pnpm version deps

* chore: fix lock file

* chore: fix lock file

* feat(workflow): add nx to project

.gitignore

@@ -98,4 +98,8 @@ playwright/.cache/
 # MidScene.js dump files
 midscene_run/
 
-__ai_responses__/
+__ai_responses__/
+
+
+.nx/cache
+.nx/workspace-data

README.ch.md

@@ -0,0 +1,25 @@
+<p align="center">
+  <img alt="MidScene.js"  width="260" src="https://github.com/user-attachments/assets/bff5e76f-ea5c-42b7-bd12-0143a04671cf">
+</p>
+
+<div align="center">
+
+[English](./README.md) | 简体中文
+
+</div>
+
+
+# MidScene.js
+
+欢迎使用 **Midscene**！这是一个全新的 AI 自动化页面操作和信息提取的框架，利用自然语言（NLQ）和人工智能（AI）技术，简化了复杂的数据查询和用户界面交互。通过 Midscene，您可以轻松地使用自然语言操作页面、定位元素、生成自定义数据结构，并自动分配类型，所有这些都无需进行任何自定义训练。
+
+## 特性 ✨
+
+- **自然语言页面控制**：使用自然语言操作页面，包括鼠标点击、滚动、键盘输入等人类行为 🗣️💻 
+- **JSON 响应**：通过提示 AI 生成所需的数据结构，确保 JSON 结构和值的可预测性。📊📋 
+- **TypeScript 友好**：自动分配类型，使用点表示法轻松访问数据。📝🔍 
+- **可视化工具**：使用可视化工具轻松调试提示和推理过程。🛠️👀 
+- **全新体验**：享受全新的自动化开发世界。🌟🚀 
+- **即用型 AI 模型**：使用 GPT-4o，无需进行任何自定义训练。🤖🔧 
+
+

README.md

@@ -1,2 +1,24 @@
+<p align="center">
+  <img alt="MidScene.js"  width="260" src="https://github.com/user-attachments/assets/bff5e76f-ea5c-42b7-bd12-0143a04671cf">
+</p>
+
+
+<div align="center">
+
+English | [简体中文](README.ch.md)
+
+</div>
+
 # MidScene.js
 
+Welcome to **Midscene**! This is a brand-new framework for AI automated page operations and information extraction, leveraging Natural Language Query (NLQ) and Artificial Intelligence (AI) technologies to simplify complex data queries and user interface interactions. With Midscene, you can easily operate pages, locate elements, generate custom data structures, and automatically assign types using natural language, all without any custom training.
+
+## Features ✨
+
+- **Natural Language Page Control**: Operate pages using natural language, including actions like clicking and typing 🗣️💻
+- **Natural Language Query**: Locate page elements using natural language, eliminating the need for DOM selectors 🔍🗂️
+- **JSON Responses**: Prompt AI to generate the required data structures, ensuring the predictability of JSON structures and values 📊📋
+- **TypeScript Friendly**: Automatically assign types and access data easily using dot notation 📝🔍
+- **Visualization Tools**: Easily debug prompts and reasoning processes with visualization tools 🛠️👀
+- **New Experience**: Enjoy a new world of automated development 🌟🚀
+- **Ready-to-Use AI Models**: Utilize GPT-4o without any custom training 🤖🔧

apps/site/package.json

@@ -7,10 +7,14 @@
     "build": "rspress build",
     "preview": "rspress preview"
   },
+  "engines": {
+    "pnpm": ">=9.3.0",
+    "node": ">=18.19.0"
+  },
+  "packageManager": "pnpm@9.3.0",
   "dependencies": {
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
-    "@arco-design/web-react": "2.63.1",
     "querystring": "0.2.1",
     "rspress": "^1.24.0",
     "midscene-visualizer": "workspace:*"

nx.json

@@ -0,0 +1,38 @@
+{
+  "$schema": "./node_modules/nx/schemas/nx-schema.json",
+  "targetDefaults": {
+    "dev": {
+      "dependsOn": [
+        "^dev"
+      ]
+    },
+    "build": {
+      "dependsOn": [
+        "^build"
+      ],
+      "cache": true
+    },
+    "build:watch": {
+      "dependsOn": [
+        "^build"
+      ]
+    },
+    "test": {
+      "dependsOn": [
+        "^build"
+      ],
+      "cache": true
+    },
+    "e2e": {
+      "dependsOn": [
+        "^build"
+      ]
+    },
+    "e2e:ui": {
+      "dependsOn": [
+        "^build"
+      ]
+    }
+  },
+  "defaultBase": "main"
+}

package.json

@@ -17,6 +17,7 @@
     "pnpm": ">=9.3.0",
     "node": ">=18.19.0"
   },
+  "packageManager": "pnpm@9.3.0",
   "devDependencies": {
     "@commitlint/cli": "^17.0.3",
     "@commitlint/config-conventional": "^17.0.3",
@@ -27,11 +28,12 @@
     "prettier": "^2.7.1",
     "pretty-quick": "3.1.3",
     "@changesets/cli": "2.24.1",
-    "commitizen": "4.2.5"
+    "commitizen": "4.2.5",
+    "nx": "19.5.2"
   },
   "config": {
     "commitizen": {
       "path": "cz-conventional-changelog"
     }
   }
-}
+}

packages/midscene/src/types.ts

@@ -80,6 +80,8 @@ export abstract class UIContext<ElementType extends BaseElement = BaseElement> {
 
   abstract content: ElementType[];
 
+  abstract size: Size;
+
   // abstract describer: () => Promise<{
   //   description: string;
   //   elementById: (id: string) => ElementType;

packages/web-integration/src/playwright/utils.ts

@@ -5,7 +5,7 @@ import path from 'path';
 import type { Page as PlaywrightPage } from 'playwright';
 import { Page } from 'puppeteer';
 import { UIContext, PlaywrightParserOpt } from '@midscene/core';
-import { alignCoordByTrim, base64Encoded } from '@midscene/core/image';
+import { alignCoordByTrim, base64Encoded, imageInfo } from '@midscene/core/image';
 import { getTmpFile } from '@midscene/core/utils';
 import { WebElementInfo, WebElementInfoType } from './element';
 
@@ -21,12 +21,13 @@ export async function parseContextFromPlaywrightPage(
   const captureElementSnapshot = await getElementInfosFromPage(page);
   // align element
   const elementsInfo = await alignElements(screenshotBuffer, captureElementSnapshot, page);
-  const baseContext = {
-    content: elementsInfo,
-    screenshotBase64,
-  };
+  const size = await imageInfo(screenshotBase64);
+
+  
   return {
-    ...baseContext,
+    content: elementsInfo,
+    size,
+    screenshotBase64,
   };
 }