mirror of
https://github.com/web-infra-dev/midscene.git
synced 2025-12-27 15:10:20 +00:00
fix: keypress issue in chrome extension (#201)
* fix: keypress issue in chrome extension * fix: keypress issue in chrome extension * fix: connectivity * doc: update readme
This commit is contained in:
yuyutaotao
GitHub
parent
21b3574280
commit
e6343b63f4
@ -6,8 +6,6 @@ Introducing Midscene.js, an innovative SDK designed to bring joy back to automat
|
||||
|
||||
Midscene.js leverages a multimodal Large Language Model (LLM) to intuitively “understand” your user interface and carry out the necessary actions. You can simply describe the interaction steps or expected data formats, and the AI will handle the execution for you.
|
||||
|
||||
Currently, the model we are using by default is the OpenAI GPT-4o model, while you can [customize it to a different multimodal model](./model-provider.html) if needed.
|
||||
|
||||
<div style={{"width": "100%", "display": "flex", justifyContent: "center"}}>
|
||||
<iframe
|
||||
style={{"maxWidth": "100%", "width": "800px", "height": "450px"}}
|
||||
@ -68,7 +66,9 @@ Midscene will provide a visual report after each run. With this report, you can
|
||||
|
||||
Midscene.js is an open-source project (GitHub: [Midscene](https://github.com/web-infra-dev/midscene/)) under the MIT license. You can run it in your own environment. All data gathered from pages will be sent directly to OpenAI or the custom model provider according to your configuration. Therefore, only you and the model provider will have access to the data. No third-party platform will access the data.
|
||||
|
||||
For custom model, you can refer to [Customize Model and Provider](./model-provider.html) document.
|
||||
## Customize Model
|
||||
|
||||
Currently, the model we are using by default is the OpenAI GPT-4o model, while you can [customize it to a different multimodal model](./model-provider.html) if needed.
|
||||
|
||||
## Start with Chrome Extension
|
||||
|
||||
|
||||
@ -1,13 +1,11 @@
|
||||
# Midscene.js - AI 加持，带来愉悦的 UI 自动化体验
|
||||
|
||||
UI 自动化太难维护了。UI 自动化脚本里往往到处都是选择器，比如 `#ids`、`data-test`、`.selectors`。在需要重构的时候，这可能会让人感到非常头疼，尽管在这种情况下，理论上UI自动化应该能够发挥作用。
|
||||
UI 自动化太难维护了。UI 自动化脚本里往往到处都是选择器，比如 `#ids`、`data-test`、`.selectors`。在需要重构的时候，这可能会让人感到非常头疼，尽管在这种情况下，UI 自动化应该能够发挥作用。
|
||||
|
||||
我们在这里推出 Midscene.js，助你重拾编码的乐趣。
|
||||
|
||||
Midscene.js 采用了多模态大语言模型（LLM），能够直观地“理解”你的用户界面并执行必要的操作。你只需描述交互步骤或期望的数据格式，AI 就能为你完成任务。
|
||||
|
||||
目前我们默认选择的是 OpenAI GPT-4o 作为模型，你也可以[自定义为其他多模态模型](./model-provider.html)。
|
||||
|
||||
<video src="/introduction/Midscene.mp4" controls/>
|
||||
|
||||
## 通过 AI 执行交互、提取数据和断言
|
||||
@ -57,7 +55,9 @@ console.log("headphones in stock", items);
|
||||
|
||||
Midscene.js 是一个采用 MIT 许可证的开源项目 (GitHub: [Midscene](https://github.com/web-infra-dev/midscene/)) 。项目代码运行在用户的自有环境中，所有从页面收集的数据会依照用户的配置，直接传送到 OpenAI 或指定的自定义模型。因此，数据仅用户和指定的模型服务商可访问，任何第三方平台均无法获取这些数据。
|
||||
|
||||
关于自定义模型，可以参考 [自定义模型和服务商](./model-provider.html) 文档。
|
||||
## 自定义模型
|
||||
|
||||
目前我们默认选择的是 OpenAI GPT-4o 作为模型，你也可以[自定义为其他多模态模型](./model-provider.html)。
|
||||
|
||||
## 从 Chrome插件开始快速体验
|
||||
|
||||
|
||||
@ -37,7 +37,7 @@ export MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON='{"apiVersion": "2024-11-01-previe
|
||||
|
||||
## 选用 `gpt-4o` 以外的其他模型
|
||||
|
||||
我们发现 `gpt-4o` 是目前表现最佳的模型。其他已知支持的模型有：`claude-3-opus-20240229`, `gemini-1.5-pro`, `qwen-vl-max-latest` (千问）, `doubao-vision-pro-32k` (豆包）
|
||||
我们发现 `gpt-4o` 是目前表现最佳的模型。其他已知支持的模型有：`claude-3-opus-20240229`, `gemini-1.5-pro`, `qwen-vl-max-latest`（千问）, `doubao-vision-pro-32k`（豆包）
|
||||
|
||||
如果你想要使用其他模型，请遵循以下步骤：
|
||||
|
||||
|
||||
@ -23,76 +23,79 @@ vi.setConfig({
|
||||
return;
|
||||
}
|
||||
|
||||
describe.skipIf(process.env.CI)(
|
||||
`LLM service connectivity: ${envFile}`,
|
||||
() => {
|
||||
beforeAll(() => {
|
||||
const result = dotenv.config({
|
||||
debug: true,
|
||||
path: configPath,
|
||||
override: true,
|
||||
});
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
describe(`LLM service connectivity: ${envFile}`, () => {
|
||||
beforeAll(() => {
|
||||
const result = dotenv.config({
|
||||
debug: true,
|
||||
path: configPath,
|
||||
override: true,
|
||||
});
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
});
|
||||
|
||||
it('text only', async () => {
|
||||
const result = await call([
|
||||
it('text only', async () => {
|
||||
const result = await call([
|
||||
{
|
||||
role: 'system',
|
||||
content: 'Answer the question',
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content:
|
||||
'鲁迅认识周树人吗？回答我：1. 分析原因 2.回答：是/否/无效问题',
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.content.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('call to get json result', async () => {
|
||||
const result = await callToGetJSONObject<{ answer: number }>(
|
||||
[
|
||||
{
|
||||
role: 'system',
|
||||
content: 'Answer the question',
|
||||
content: 'Answer the question with JSON: {answer: number}',
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content:
|
||||
'鲁迅认识周树人吗？回答我：1. 分析原因 2.回答：是/否/无效问题',
|
||||
content: '3 x 5 = ?',
|
||||
},
|
||||
]);
|
||||
],
|
||||
AIActionType.EXTRACT_DATA,
|
||||
);
|
||||
expect(result.content).toEqual({ answer: 15 });
|
||||
});
|
||||
|
||||
expect(result.content.length).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it('call to get json result', async () => {
|
||||
const result = await callToGetJSONObject<{ answer: number }>(
|
||||
[
|
||||
it('image input', async () => {
|
||||
const imagePath = getFixture('baidu.png');
|
||||
const result = await call([
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
role: 'system',
|
||||
content: 'Answer the question with JSON: {answer: number}',
|
||||
type: 'text',
|
||||
text: 'Describe this image in one sentence.',
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: '3 x 5 = ?',
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: base64Encoded(imagePath),
|
||||
detail: 'high',
|
||||
},
|
||||
},
|
||||
],
|
||||
AIActionType.EXTRACT_DATA,
|
||||
);
|
||||
expect(result.content).toEqual({ answer: 15 });
|
||||
});
|
||||
},
|
||||
]);
|
||||
|
||||
it('image input', async () => {
|
||||
const imagePath = getFixture('baidu.png');
|
||||
const result = await call([
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'Describe this image in one sentence.',
|
||||
},
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: {
|
||||
url: base64Encoded(imagePath),
|
||||
detail: 'high',
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(result.content.length).toBeGreaterThan(10);
|
||||
});
|
||||
},
|
||||
);
|
||||
expect(result.content.length).toBeGreaterThan(10);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('keep at least one test in each suite', () => {
|
||||
it('test', () => {
|
||||
expect(1).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
@ -0,0 +1,671 @@
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2017 Google Inc.
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
/**
|
||||
* @internal
|
||||
*/
|
||||
export interface KeyDefinition {
|
||||
keyCode?: number;
|
||||
shiftKeyCode?: number;
|
||||
key?: string;
|
||||
shiftKey?: string;
|
||||
code?: string;
|
||||
text?: string;
|
||||
shiftText?: string;
|
||||
location?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* All the valid keys that can be passed to functions that take user input, such
|
||||
* as {@link Keyboard.press | keyboard.press }
|
||||
*
|
||||
* @public
|
||||
*/
|
||||
export type KeyInput =
|
||||
| '0'
|
||||
| '1'
|
||||
| '2'
|
||||
| '3'
|
||||
| '4'
|
||||
| '5'
|
||||
| '6'
|
||||
| '7'
|
||||
| '8'
|
||||
| '9'
|
||||
| 'Power'
|
||||
| 'Eject'
|
||||
| 'Abort'
|
||||
| 'Help'
|
||||
| 'Backspace'
|
||||
| 'Tab'
|
||||
| 'Numpad5'
|
||||
| 'NumpadEnter'
|
||||
| 'Enter'
|
||||
| '\r'
|
||||
| '\n'
|
||||
| 'ShiftLeft'
|
||||
| 'ShiftRight'
|
||||
| 'ControlLeft'
|
||||
| 'ControlRight'
|
||||
| 'AltLeft'
|
||||
| 'AltRight'
|
||||
| 'Pause'
|
||||
| 'CapsLock'
|
||||
| 'Escape'
|
||||
| 'Convert'
|
||||
| 'NonConvert'
|
||||
| 'Space'
|
||||
| 'Numpad9'
|
||||
| 'PageUp'
|
||||
| 'Numpad3'
|
||||
| 'PageDown'
|
||||
| 'End'
|
||||
| 'Numpad1'
|
||||
| 'Home'
|
||||
| 'Numpad7'
|
||||
| 'ArrowLeft'
|
||||
| 'Numpad4'
|
||||
| 'Numpad8'
|
||||
| 'ArrowUp'
|
||||
| 'ArrowRight'
|
||||
| 'Numpad6'
|
||||
| 'Numpad2'
|
||||
| 'ArrowDown'
|
||||
| 'Select'
|
||||
| 'Open'
|
||||
| 'PrintScreen'
|
||||
| 'Insert'
|
||||
| 'Numpad0'
|
||||
| 'Delete'
|
||||
| 'NumpadDecimal'
|
||||
| 'Digit0'
|
||||
| 'Digit1'
|
||||
| 'Digit2'
|
||||
| 'Digit3'
|
||||
| 'Digit4'
|
||||
| 'Digit5'
|
||||
| 'Digit6'
|
||||
| 'Digit7'
|
||||
| 'Digit8'
|
||||
| 'Digit9'
|
||||
| 'KeyA'
|
||||
| 'KeyB'
|
||||
| 'KeyC'
|
||||
| 'KeyD'
|
||||
| 'KeyE'
|
||||
| 'KeyF'
|
||||
| 'KeyG'
|
||||
| 'KeyH'
|
||||
| 'KeyI'
|
||||
| 'KeyJ'
|
||||
| 'KeyK'
|
||||
| 'KeyL'
|
||||
| 'KeyM'
|
||||
| 'KeyN'
|
||||
| 'KeyO'
|
||||
| 'KeyP'
|
||||
| 'KeyQ'
|
||||
| 'KeyR'
|
||||
| 'KeyS'
|
||||
| 'KeyT'
|
||||
| 'KeyU'
|
||||
| 'KeyV'
|
||||
| 'KeyW'
|
||||
| 'KeyX'
|
||||
| 'KeyY'
|
||||
| 'KeyZ'
|
||||
| 'MetaLeft'
|
||||
| 'MetaRight'
|
||||
| 'ContextMenu'
|
||||
| 'NumpadMultiply'
|
||||
| 'NumpadAdd'
|
||||
| 'NumpadSubtract'
|
||||
| 'NumpadDivide'
|
||||
| 'F1'
|
||||
| 'F2'
|
||||
| 'F3'
|
||||
| 'F4'
|
||||
| 'F5'
|
||||
| 'F6'
|
||||
| 'F7'
|
||||
| 'F8'
|
||||
| 'F9'
|
||||
| 'F10'
|
||||
| 'F11'
|
||||
| 'F12'
|
||||
| 'F13'
|
||||
| 'F14'
|
||||
| 'F15'
|
||||
| 'F16'
|
||||
| 'F17'
|
||||
| 'F18'
|
||||
| 'F19'
|
||||
| 'F20'
|
||||
| 'F21'
|
||||
| 'F22'
|
||||
| 'F23'
|
||||
| 'F24'
|
||||
| 'NumLock'
|
||||
| 'ScrollLock'
|
||||
| 'AudioVolumeMute'
|
||||
| 'AudioVolumeDown'
|
||||
| 'AudioVolumeUp'
|
||||
| 'MediaTrackNext'
|
||||
| 'MediaTrackPrevious'
|
||||
| 'MediaStop'
|
||||
| 'MediaPlayPause'
|
||||
| 'Semicolon'
|
||||
| 'Equal'
|
||||
| 'NumpadEqual'
|
||||
| 'Comma'
|
||||
| 'Minus'
|
||||
| 'Period'
|
||||
| 'Slash'
|
||||
| 'Backquote'
|
||||
| 'BracketLeft'
|
||||
| 'Backslash'
|
||||
| 'BracketRight'
|
||||
| 'Quote'
|
||||
| 'AltGraph'
|
||||
| 'Props'
|
||||
| 'Cancel'
|
||||
| 'Clear'
|
||||
| 'Shift'
|
||||
| 'Control'
|
||||
| 'Alt'
|
||||
| 'Accept'
|
||||
| 'ModeChange'
|
||||
| ' '
|
||||
| 'Print'
|
||||
| 'Execute'
|
||||
| '\u0000'
|
||||
| 'a'
|
||||
| 'b'
|
||||
| 'c'
|
||||
| 'd'
|
||||
| 'e'
|
||||
| 'f'
|
||||
| 'g'
|
||||
| 'h'
|
||||
| 'i'
|
||||
| 'j'
|
||||
| 'k'
|
||||
| 'l'
|
||||
| 'm'
|
||||
| 'n'
|
||||
| 'o'
|
||||
| 'p'
|
||||
| 'q'
|
||||
| 'r'
|
||||
| 's'
|
||||
| 't'
|
||||
| 'u'
|
||||
| 'v'
|
||||
| 'w'
|
||||
| 'x'
|
||||
| 'y'
|
||||
| 'z'
|
||||
| 'Meta'
|
||||
| '*'
|
||||
| '+'
|
||||
| '-'
|
||||
| '/'
|
||||
| ';'
|
||||
| '='
|
||||
| ','
|
||||
| '.'
|
||||
| '`'
|
||||
| '['
|
||||
| '\\'
|
||||
| ']'
|
||||
| "'"
|
||||
| 'Attn'
|
||||
| 'CrSel'
|
||||
| 'ExSel'
|
||||
| 'EraseEof'
|
||||
| 'Play'
|
||||
| 'ZoomOut'
|
||||
| ')'
|
||||
| '!'
|
||||
| '@'
|
||||
| '#'
|
||||
| '$'
|
||||
| '%'
|
||||
| '^'
|
||||
| '&'
|
||||
| '('
|
||||
| 'A'
|
||||
| 'B'
|
||||
| 'C'
|
||||
| 'D'
|
||||
| 'E'
|
||||
| 'F'
|
||||
| 'G'
|
||||
| 'H'
|
||||
| 'I'
|
||||
| 'J'
|
||||
| 'K'
|
||||
| 'L'
|
||||
| 'M'
|
||||
| 'N'
|
||||
| 'O'
|
||||
| 'P'
|
||||
| 'Q'
|
||||
| 'R'
|
||||
| 'S'
|
||||
| 'T'
|
||||
| 'U'
|
||||
| 'V'
|
||||
| 'W'
|
||||
| 'X'
|
||||
| 'Y'
|
||||
| 'Z'
|
||||
| ':'
|
||||
| '<'
|
||||
| '_'
|
||||
| '>'
|
||||
| '?'
|
||||
| '~'
|
||||
| '{'
|
||||
| '|'
|
||||
| '}'
|
||||
| '"'
|
||||
| 'SoftLeft'
|
||||
| 'SoftRight'
|
||||
| 'Camera'
|
||||
| 'Call'
|
||||
| 'EndCall'
|
||||
| 'VolumeDown'
|
||||
| 'VolumeUp';
|
||||
|
||||
/**
|
||||
* @internal
|
||||
*/
|
||||
export const _keyDefinitions: Readonly<Record<KeyInput, KeyDefinition>> = {
|
||||
'0': { keyCode: 48, key: '0', code: 'Digit0' },
|
||||
'1': { keyCode: 49, key: '1', code: 'Digit1' },
|
||||
'2': { keyCode: 50, key: '2', code: 'Digit2' },
|
||||
'3': { keyCode: 51, key: '3', code: 'Digit3' },
|
||||
'4': { keyCode: 52, key: '4', code: 'Digit4' },
|
||||
'5': { keyCode: 53, key: '5', code: 'Digit5' },
|
||||
'6': { keyCode: 54, key: '6', code: 'Digit6' },
|
||||
'7': { keyCode: 55, key: '7', code: 'Digit7' },
|
||||
'8': { keyCode: 56, key: '8', code: 'Digit8' },
|
||||
'9': { keyCode: 57, key: '9', code: 'Digit9' },
|
||||
Power: { key: 'Power', code: 'Power' },
|
||||
Eject: { key: 'Eject', code: 'Eject' },
|
||||
Abort: { keyCode: 3, code: 'Abort', key: 'Cancel' },
|
||||
Help: { keyCode: 6, code: 'Help', key: 'Help' },
|
||||
Backspace: { keyCode: 8, code: 'Backspace', key: 'Backspace' },
|
||||
Tab: { keyCode: 9, code: 'Tab', key: 'Tab' },
|
||||
Numpad5: {
|
||||
keyCode: 12,
|
||||
shiftKeyCode: 101,
|
||||
key: 'Clear',
|
||||
code: 'Numpad5',
|
||||
shiftKey: '5',
|
||||
location: 3,
|
||||
},
|
||||
NumpadEnter: {
|
||||
keyCode: 13,
|
||||
code: 'NumpadEnter',
|
||||
key: 'Enter',
|
||||
text: '\r',
|
||||
location: 3,
|
||||
},
|
||||
Enter: { keyCode: 13, code: 'Enter', key: 'Enter', text: '\r' },
|
||||
'\r': { keyCode: 13, code: 'Enter', key: 'Enter', text: '\r' },
|
||||
'\n': { keyCode: 13, code: 'Enter', key: 'Enter', text: '\r' },
|
||||
ShiftLeft: { keyCode: 16, code: 'ShiftLeft', key: 'Shift', location: 1 },
|
||||
ShiftRight: { keyCode: 16, code: 'ShiftRight', key: 'Shift', location: 2 },
|
||||
ControlLeft: {
|
||||
keyCode: 17,
|
||||
code: 'ControlLeft',
|
||||
key: 'Control',
|
||||
location: 1,
|
||||
},
|
||||
ControlRight: {
|
||||
keyCode: 17,
|
||||
code: 'ControlRight',
|
||||
key: 'Control',
|
||||
location: 2,
|
||||
},
|
||||
AltLeft: { keyCode: 18, code: 'AltLeft', key: 'Alt', location: 1 },
|
||||
AltRight: { keyCode: 18, code: 'AltRight', key: 'Alt', location: 2 },
|
||||
Pause: { keyCode: 19, code: 'Pause', key: 'Pause' },
|
||||
CapsLock: { keyCode: 20, code: 'CapsLock', key: 'CapsLock' },
|
||||
Escape: { keyCode: 27, code: 'Escape', key: 'Escape' },
|
||||
Convert: { keyCode: 28, code: 'Convert', key: 'Convert' },
|
||||
NonConvert: { keyCode: 29, code: 'NonConvert', key: 'NonConvert' },
|
||||
Space: { keyCode: 32, code: 'Space', key: ' ' },
|
||||
Numpad9: {
|
||||
keyCode: 33,
|
||||
shiftKeyCode: 105,
|
||||
key: 'PageUp',
|
||||
code: 'Numpad9',
|
||||
shiftKey: '9',
|
||||
location: 3,
|
||||
},
|
||||
PageUp: { keyCode: 33, code: 'PageUp', key: 'PageUp' },
|
||||
Numpad3: {
|
||||
keyCode: 34,
|
||||
shiftKeyCode: 99,
|
||||
key: 'PageDown',
|
||||
code: 'Numpad3',
|
||||
shiftKey: '3',
|
||||
location: 3,
|
||||
},
|
||||
PageDown: { keyCode: 34, code: 'PageDown', key: 'PageDown' },
|
||||
End: { keyCode: 35, code: 'End', key: 'End' },
|
||||
Numpad1: {
|
||||
keyCode: 35,
|
||||
shiftKeyCode: 97,
|
||||
key: 'End',
|
||||
code: 'Numpad1',
|
||||
shiftKey: '1',
|
||||
location: 3,
|
||||
},
|
||||
Home: { keyCode: 36, code: 'Home', key: 'Home' },
|
||||
Numpad7: {
|
||||
keyCode: 36,
|
||||
shiftKeyCode: 103,
|
||||
key: 'Home',
|
||||
code: 'Numpad7',
|
||||
shiftKey: '7',
|
||||
location: 3,
|
||||
},
|
||||
ArrowLeft: { keyCode: 37, code: 'ArrowLeft', key: 'ArrowLeft' },
|
||||
Numpad4: {
|
||||
keyCode: 37,
|
||||
shiftKeyCode: 100,
|
||||
key: 'ArrowLeft',
|
||||
code: 'Numpad4',
|
||||
shiftKey: '4',
|
||||
location: 3,
|
||||
},
|
||||
Numpad8: {
|
||||
keyCode: 38,
|
||||
shiftKeyCode: 104,
|
||||
key: 'ArrowUp',
|
||||
code: 'Numpad8',
|
||||
shiftKey: '8',
|
||||
location: 3,
|
||||
},
|
||||
ArrowUp: { keyCode: 38, code: 'ArrowUp', key: 'ArrowUp' },
|
||||
ArrowRight: { keyCode: 39, code: 'ArrowRight', key: 'ArrowRight' },
|
||||
Numpad6: {
|
||||
keyCode: 39,
|
||||
shiftKeyCode: 102,
|
||||
key: 'ArrowRight',
|
||||
code: 'Numpad6',
|
||||
shiftKey: '6',
|
||||
location: 3,
|
||||
},
|
||||
Numpad2: {
|
||||
keyCode: 40,
|
||||
shiftKeyCode: 98,
|
||||
key: 'ArrowDown',
|
||||
code: 'Numpad2',
|
||||
shiftKey: '2',
|
||||
location: 3,
|
||||
},
|
||||
ArrowDown: { keyCode: 40, code: 'ArrowDown', key: 'ArrowDown' },
|
||||
Select: { keyCode: 41, code: 'Select', key: 'Select' },
|
||||
Open: { keyCode: 43, code: 'Open', key: 'Execute' },
|
||||
PrintScreen: { keyCode: 44, code: 'PrintScreen', key: 'PrintScreen' },
|
||||
Insert: { keyCode: 45, code: 'Insert', key: 'Insert' },
|
||||
Numpad0: {
|
||||
keyCode: 45,
|
||||
shiftKeyCode: 96,
|
||||
key: 'Insert',
|
||||
code: 'Numpad0',
|
||||
shiftKey: '0',
|
||||
location: 3,
|
||||
},
|
||||
Delete: { keyCode: 46, code: 'Delete', key: 'Delete' },
|
||||
NumpadDecimal: {
|
||||
keyCode: 46,
|
||||
shiftKeyCode: 110,
|
||||
code: 'NumpadDecimal',
|
||||
key: '\u0000',
|
||||
shiftKey: '.',
|
||||
location: 3,
|
||||
},
|
||||
Digit0: { keyCode: 48, code: 'Digit0', shiftKey: ')', key: '0' },
|
||||
Digit1: { keyCode: 49, code: 'Digit1', shiftKey: '!', key: '1' },
|
||||
Digit2: { keyCode: 50, code: 'Digit2', shiftKey: '@', key: '2' },
|
||||
Digit3: { keyCode: 51, code: 'Digit3', shiftKey: '#', key: '3' },
|
||||
Digit4: { keyCode: 52, code: 'Digit4', shiftKey: '$', key: '4' },
|
||||
Digit5: { keyCode: 53, code: 'Digit5', shiftKey: '%', key: '5' },
|
||||
Digit6: { keyCode: 54, code: 'Digit6', shiftKey: '^', key: '6' },
|
||||
Digit7: { keyCode: 55, code: 'Digit7', shiftKey: '&', key: '7' },
|
||||
Digit8: { keyCode: 56, code: 'Digit8', shiftKey: '*', key: '8' },
|
||||
Digit9: { keyCode: 57, code: 'Digit9', shiftKey: '(', key: '9' },
|
||||
KeyA: { keyCode: 65, code: 'KeyA', shiftKey: 'A', key: 'a' },
|
||||
KeyB: { keyCode: 66, code: 'KeyB', shiftKey: 'B', key: 'b' },
|
||||
KeyC: { keyCode: 67, code: 'KeyC', shiftKey: 'C', key: 'c' },
|
||||
KeyD: { keyCode: 68, code: 'KeyD', shiftKey: 'D', key: 'd' },
|
||||
KeyE: { keyCode: 69, code: 'KeyE', shiftKey: 'E', key: 'e' },
|
||||
KeyF: { keyCode: 70, code: 'KeyF', shiftKey: 'F', key: 'f' },
|
||||
KeyG: { keyCode: 71, code: 'KeyG', shiftKey: 'G', key: 'g' },
|
||||
KeyH: { keyCode: 72, code: 'KeyH', shiftKey: 'H', key: 'h' },
|
||||
KeyI: { keyCode: 73, code: 'KeyI', shiftKey: 'I', key: 'i' },
|
||||
KeyJ: { keyCode: 74, code: 'KeyJ', shiftKey: 'J', key: 'j' },
|
||||
KeyK: { keyCode: 75, code: 'KeyK', shiftKey: 'K', key: 'k' },
|
||||
KeyL: { keyCode: 76, code: 'KeyL', shiftKey: 'L', key: 'l' },
|
||||
KeyM: { keyCode: 77, code: 'KeyM', shiftKey: 'M', key: 'm' },
|
||||
KeyN: { keyCode: 78, code: 'KeyN', shiftKey: 'N', key: 'n' },
|
||||
KeyO: { keyCode: 79, code: 'KeyO', shiftKey: 'O', key: 'o' },
|
||||
KeyP: { keyCode: 80, code: 'KeyP', shiftKey: 'P', key: 'p' },
|
||||
KeyQ: { keyCode: 81, code: 'KeyQ', shiftKey: 'Q', key: 'q' },
|
||||
KeyR: { keyCode: 82, code: 'KeyR', shiftKey: 'R', key: 'r' },
|
||||
KeyS: { keyCode: 83, code: 'KeyS', shiftKey: 'S', key: 's' },
|
||||
KeyT: { keyCode: 84, code: 'KeyT', shiftKey: 'T', key: 't' },
|
||||
KeyU: { keyCode: 85, code: 'KeyU', shiftKey: 'U', key: 'u' },
|
||||
KeyV: { keyCode: 86, code: 'KeyV', shiftKey: 'V', key: 'v' },
|
||||
KeyW: { keyCode: 87, code: 'KeyW', shiftKey: 'W', key: 'w' },
|
||||
KeyX: { keyCode: 88, code: 'KeyX', shiftKey: 'X', key: 'x' },
|
||||
KeyY: { keyCode: 89, code: 'KeyY', shiftKey: 'Y', key: 'y' },
|
||||
KeyZ: { keyCode: 90, code: 'KeyZ', shiftKey: 'Z', key: 'z' },
|
||||
MetaLeft: { keyCode: 91, code: 'MetaLeft', key: 'Meta', location: 1 },
|
||||
MetaRight: { keyCode: 92, code: 'MetaRight', key: 'Meta', location: 2 },
|
||||
ContextMenu: { keyCode: 93, code: 'ContextMenu', key: 'ContextMenu' },
|
||||
NumpadMultiply: {
|
||||
keyCode: 106,
|
||||
code: 'NumpadMultiply',
|
||||
key: '*',
|
||||
location: 3,
|
||||
},
|
||||
NumpadAdd: { keyCode: 107, code: 'NumpadAdd', key: '+', location: 3 },
|
||||
NumpadSubtract: {
|
||||
keyCode: 109,
|
||||
code: 'NumpadSubtract',
|
||||
key: '-',
|
||||
location: 3,
|
||||
},
|
||||
NumpadDivide: { keyCode: 111, code: 'NumpadDivide', key: '/', location: 3 },
|
||||
F1: { keyCode: 112, code: 'F1', key: 'F1' },
|
||||
F2: { keyCode: 113, code: 'F2', key: 'F2' },
|
||||
F3: { keyCode: 114, code: 'F3', key: 'F3' },
|
||||
F4: { keyCode: 115, code: 'F4', key: 'F4' },
|
||||
F5: { keyCode: 116, code: 'F5', key: 'F5' },
|
||||
F6: { keyCode: 117, code: 'F6', key: 'F6' },
|
||||
F7: { keyCode: 118, code: 'F7', key: 'F7' },
|
||||
F8: { keyCode: 119, code: 'F8', key: 'F8' },
|
||||
F9: { keyCode: 120, code: 'F9', key: 'F9' },
|
||||
F10: { keyCode: 121, code: 'F10', key: 'F10' },
|
||||
F11: { keyCode: 122, code: 'F11', key: 'F11' },
|
||||
F12: { keyCode: 123, code: 'F12', key: 'F12' },
|
||||
F13: { keyCode: 124, code: 'F13', key: 'F13' },
|
||||
F14: { keyCode: 125, code: 'F14', key: 'F14' },
|
||||
F15: { keyCode: 126, code: 'F15', key: 'F15' },
|
||||
F16: { keyCode: 127, code: 'F16', key: 'F16' },
|
||||
F17: { keyCode: 128, code: 'F17', key: 'F17' },
|
||||
F18: { keyCode: 129, code: 'F18', key: 'F18' },
|
||||
F19: { keyCode: 130, code: 'F19', key: 'F19' },
|
||||
F20: { keyCode: 131, code: 'F20', key: 'F20' },
|
||||
F21: { keyCode: 132, code: 'F21', key: 'F21' },
|
||||
F22: { keyCode: 133, code: 'F22', key: 'F22' },
|
||||
F23: { keyCode: 134, code: 'F23', key: 'F23' },
|
||||
F24: { keyCode: 135, code: 'F24', key: 'F24' },
|
||||
NumLock: { keyCode: 144, code: 'NumLock', key: 'NumLock' },
|
||||
ScrollLock: { keyCode: 145, code: 'ScrollLock', key: 'ScrollLock' },
|
||||
AudioVolumeMute: {
|
||||
keyCode: 173,
|
||||
code: 'AudioVolumeMute',
|
||||
key: 'AudioVolumeMute',
|
||||
},
|
||||
AudioVolumeDown: {
|
||||
keyCode: 174,
|
||||
code: 'AudioVolumeDown',
|
||||
key: 'AudioVolumeDown',
|
||||
},
|
||||
AudioVolumeUp: { keyCode: 175, code: 'AudioVolumeUp', key: 'AudioVolumeUp' },
|
||||
MediaTrackNext: {
|
||||
keyCode: 176,
|
||||
code: 'MediaTrackNext',
|
||||
key: 'MediaTrackNext',
|
||||
},
|
||||
MediaTrackPrevious: {
|
||||
keyCode: 177,
|
||||
code: 'MediaTrackPrevious',
|
||||
key: 'MediaTrackPrevious',
|
||||
},
|
||||
MediaStop: { keyCode: 178, code: 'MediaStop', key: 'MediaStop' },
|
||||
MediaPlayPause: {
|
||||
keyCode: 179,
|
||||
code: 'MediaPlayPause',
|
||||
key: 'MediaPlayPause',
|
||||
},
|
||||
Semicolon: { keyCode: 186, code: 'Semicolon', shiftKey: ':', key: ';' },
|
||||
Equal: { keyCode: 187, code: 'Equal', shiftKey: '+', key: '=' },
|
||||
NumpadEqual: { keyCode: 187, code: 'NumpadEqual', key: '=', location: 3 },
|
||||
Comma: { keyCode: 188, code: 'Comma', shiftKey: '<', key: ',' },
|
||||
Minus: { keyCode: 189, code: 'Minus', shiftKey: '_', key: '-' },
|
||||
Period: { keyCode: 190, code: 'Period', shiftKey: '>', key: '.' },
|
||||
Slash: { keyCode: 191, code: 'Slash', shiftKey: '?', key: '/' },
|
||||
Backquote: { keyCode: 192, code: 'Backquote', shiftKey: '~', key: '`' },
|
||||
BracketLeft: { keyCode: 219, code: 'BracketLeft', shiftKey: '{', key: '[' },
|
||||
Backslash: { keyCode: 220, code: 'Backslash', shiftKey: '|', key: '\\' },
|
||||
BracketRight: { keyCode: 221, code: 'BracketRight', shiftKey: '}', key: ']' },
|
||||
Quote: { keyCode: 222, code: 'Quote', shiftKey: '"', key: "'" },
|
||||
AltGraph: { keyCode: 225, code: 'AltGraph', key: 'AltGraph' },
|
||||
Props: { keyCode: 247, code: 'Props', key: 'CrSel' },
|
||||
Cancel: { keyCode: 3, key: 'Cancel', code: 'Abort' },
|
||||
Clear: { keyCode: 12, key: 'Clear', code: 'Numpad5', location: 3 },
|
||||
Shift: { keyCode: 16, key: 'Shift', code: 'ShiftLeft', location: 1 },
|
||||
Control: { keyCode: 17, key: 'Control', code: 'ControlLeft', location: 1 },
|
||||
Alt: { keyCode: 18, key: 'Alt', code: 'AltLeft', location: 1 },
|
||||
Accept: { keyCode: 30, key: 'Accept' },
|
||||
ModeChange: { keyCode: 31, key: 'ModeChange' },
|
||||
' ': { keyCode: 32, key: ' ', code: 'Space' },
|
||||
Print: { keyCode: 42, key: 'Print' },
|
||||
Execute: { keyCode: 43, key: 'Execute', code: 'Open' },
|
||||
'\u0000': { keyCode: 46, key: '\u0000', code: 'NumpadDecimal', location: 3 },
|
||||
a: { keyCode: 65, key: 'a', code: 'KeyA' },
|
||||
b: { keyCode: 66, key: 'b', code: 'KeyB' },
|
||||
c: { keyCode: 67, key: 'c', code: 'KeyC' },
|
||||
d: { keyCode: 68, key: 'd', code: 'KeyD' },
|
||||
e: { keyCode: 69, key: 'e', code: 'KeyE' },
|
||||
f: { keyCode: 70, key: 'f', code: 'KeyF' },
|
||||
g: { keyCode: 71, key: 'g', code: 'KeyG' },
|
||||
h: { keyCode: 72, key: 'h', code: 'KeyH' },
|
||||
i: { keyCode: 73, key: 'i', code: 'KeyI' },
|
||||
j: { keyCode: 74, key: 'j', code: 'KeyJ' },
|
||||
k: { keyCode: 75, key: 'k', code: 'KeyK' },
|
||||
l: { keyCode: 76, key: 'l', code: 'KeyL' },
|
||||
m: { keyCode: 77, key: 'm', code: 'KeyM' },
|
||||
n: { keyCode: 78, key: 'n', code: 'KeyN' },
|
||||
o: { keyCode: 79, key: 'o', code: 'KeyO' },
|
||||
p: { keyCode: 80, key: 'p', code: 'KeyP' },
|
||||
q: { keyCode: 81, key: 'q', code: 'KeyQ' },
|
||||
r: { keyCode: 82, key: 'r', code: 'KeyR' },
|
||||
s: { keyCode: 83, key: 's', code: 'KeyS' },
|
||||
t: { keyCode: 84, key: 't', code: 'KeyT' },
|
||||
u: { keyCode: 85, key: 'u', code: 'KeyU' },
|
||||
v: { keyCode: 86, key: 'v', code: 'KeyV' },
|
||||
w: { keyCode: 87, key: 'w', code: 'KeyW' },
|
||||
x: { keyCode: 88, key: 'x', code: 'KeyX' },
|
||||
y: { keyCode: 89, key: 'y', code: 'KeyY' },
|
||||
z: { keyCode: 90, key: 'z', code: 'KeyZ' },
|
||||
Meta: { keyCode: 91, key: 'Meta', code: 'MetaLeft', location: 1 },
|
||||
'*': { keyCode: 106, key: '*', code: 'NumpadMultiply', location: 3 },
|
||||
'+': { keyCode: 107, key: '+', code: 'NumpadAdd', location: 3 },
|
||||
'-': { keyCode: 109, key: '-', code: 'NumpadSubtract', location: 3 },
|
||||
'/': { keyCode: 111, key: '/', code: 'NumpadDivide', location: 3 },
|
||||
';': { keyCode: 186, key: ';', code: 'Semicolon' },
|
||||
'=': { keyCode: 187, key: '=', code: 'Equal' },
|
||||
',': { keyCode: 188, key: ',', code: 'Comma' },
|
||||
'.': { keyCode: 190, key: '.', code: 'Period' },
|
||||
'`': { keyCode: 192, key: '`', code: 'Backquote' },
|
||||
'[': { keyCode: 219, key: '[', code: 'BracketLeft' },
|
||||
'\\': { keyCode: 220, key: '\\', code: 'Backslash' },
|
||||
']': { keyCode: 221, key: ']', code: 'BracketRight' },
|
||||
"'": { keyCode: 222, key: "'", code: 'Quote' },
|
||||
Attn: { keyCode: 246, key: 'Attn' },
|
||||
CrSel: { keyCode: 247, key: 'CrSel', code: 'Props' },
|
||||
ExSel: { keyCode: 248, key: 'ExSel' },
|
||||
EraseEof: { keyCode: 249, key: 'EraseEof' },
|
||||
Play: { keyCode: 250, key: 'Play' },
|
||||
ZoomOut: { keyCode: 251, key: 'ZoomOut' },
|
||||
')': { keyCode: 48, key: ')', code: 'Digit0' },
|
||||
'!': { keyCode: 49, key: '!', code: 'Digit1' },
|
||||
'@': { keyCode: 50, key: '@', code: 'Digit2' },
|
||||
'#': { keyCode: 51, key: '#', code: 'Digit3' },
|
||||
$: { keyCode: 52, key: '$', code: 'Digit4' },
|
||||
'%': { keyCode: 53, key: '%', code: 'Digit5' },
|
||||
'^': { keyCode: 54, key: '^', code: 'Digit6' },
|
||||
'&': { keyCode: 55, key: '&', code: 'Digit7' },
|
||||
'(': { keyCode: 57, key: '(', code: 'Digit9' },
|
||||
A: { keyCode: 65, key: 'A', code: 'KeyA' },
|
||||
B: { keyCode: 66, key: 'B', code: 'KeyB' },
|
||||
C: { keyCode: 67, key: 'C', code: 'KeyC' },
|
||||
D: { keyCode: 68, key: 'D', code: 'KeyD' },
|
||||
E: { keyCode: 69, key: 'E', code: 'KeyE' },
|
||||
F: { keyCode: 70, key: 'F', code: 'KeyF' },
|
||||
G: { keyCode: 71, key: 'G', code: 'KeyG' },
|
||||
H: { keyCode: 72, key: 'H', code: 'KeyH' },
|
||||
I: { keyCode: 73, key: 'I', code: 'KeyI' },
|
||||
J: { keyCode: 74, key: 'J', code: 'KeyJ' },
|
||||
K: { keyCode: 75, key: 'K', code: 'KeyK' },
|
||||
L: { keyCode: 76, key: 'L', code: 'KeyL' },
|
||||
M: { keyCode: 77, key: 'M', code: 'KeyM' },
|
||||
N: { keyCode: 78, key: 'N', code: 'KeyN' },
|
||||
O: { keyCode: 79, key: 'O', code: 'KeyO' },
|
||||
P: { keyCode: 80, key: 'P', code: 'KeyP' },
|
||||
Q: { keyCode: 81, key: 'Q', code: 'KeyQ' },
|
||||
R: { keyCode: 82, key: 'R', code: 'KeyR' },
|
||||
S: { keyCode: 83, key: 'S', code: 'KeyS' },
|
||||
T: { keyCode: 84, key: 'T', code: 'KeyT' },
|
||||
U: { keyCode: 85, key: 'U', code: 'KeyU' },
|
||||
V: { keyCode: 86, key: 'V', code: 'KeyV' },
|
||||
W: { keyCode: 87, key: 'W', code: 'KeyW' },
|
||||
X: { keyCode: 88, key: 'X', code: 'KeyX' },
|
||||
Y: { keyCode: 89, key: 'Y', code: 'KeyY' },
|
||||
Z: { keyCode: 90, key: 'Z', code: 'KeyZ' },
|
||||
':': { keyCode: 186, key: ':', code: 'Semicolon' },
|
||||
'<': { keyCode: 188, key: '<', code: 'Comma' },
|
||||
_: { keyCode: 189, key: '_', code: 'Minus' },
|
||||
'>': { keyCode: 190, key: '>', code: 'Period' },
|
||||
'?': { keyCode: 191, key: '?', code: 'Slash' },
|
||||
'~': { keyCode: 192, key: '~', code: 'Backquote' },
|
||||
'{': { keyCode: 219, key: '{', code: 'BracketLeft' },
|
||||
'|': { keyCode: 220, key: '|', code: 'Backslash' },
|
||||
'}': { keyCode: 221, key: '}', code: 'BracketRight' },
|
||||
'"': { keyCode: 222, key: '"', code: 'Quote' },
|
||||
SoftLeft: { key: 'SoftLeft', code: 'SoftLeft', location: 4 },
|
||||
SoftRight: { key: 'SoftRight', code: 'SoftRight', location: 4 },
|
||||
Camera: { keyCode: 44, key: 'Camera', code: 'Camera', location: 4 },
|
||||
Call: { key: 'Call', code: 'Call', location: 4 },
|
||||
EndCall: { keyCode: 95, key: 'EndCall', code: 'EndCall', location: 4 },
|
||||
VolumeDown: {
|
||||
keyCode: 182,
|
||||
key: 'VolumeDown',
|
||||
code: 'VolumeDown',
|
||||
location: 4,
|
||||
},
|
||||
VolumeUp: { keyCode: 183, key: 'VolumeUp', code: 'VolumeUp', location: 4 },
|
||||
};
|
||||
222
packages/web-integration/src/chrome-extension/cdpInput.ts
Normal file
222
packages/web-integration/src/chrome-extension/cdpInput.ts
Normal file
@ -0,0 +1,222 @@
|
||||
// From https://github.com/puppeteer/puppeteer/blob/15abcc390862fd08cc3475532f2d9a11284aee6b/packages/puppeteer-core/src/cdp/Input.ts#L55
|
||||
// with some modifications to fit the session type
|
||||
/**
|
||||
* @license
|
||||
* Copyright 2017 Google Inc.
|
||||
* SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
import assert from 'node:assert';
|
||||
import {
|
||||
type KeyDefinition,
|
||||
type KeyInput,
|
||||
_keyDefinitions,
|
||||
} from './USKeyboardLayout';
|
||||
|
||||
type KeyDescription = Required<
|
||||
Pick<KeyDefinition, 'keyCode' | 'key' | 'text' | 'code' | 'location'>
|
||||
>;
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export interface KeyDownOptions {
|
||||
/**
|
||||
* @deprecated Do not use. This is automatically handled.
|
||||
*/
|
||||
text?: string;
|
||||
/**
|
||||
* @deprecated Do not use. This is automatically handled.
|
||||
*/
|
||||
commands?: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export interface KeyboardTypeOptions {
|
||||
delay?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* @public
|
||||
*/
|
||||
export type KeyPressOptions = KeyDownOptions & KeyboardTypeOptions;
|
||||
|
||||
type InternalCDPSession = {
|
||||
send: (command: string, params: any) => Promise<void>;
|
||||
};
|
||||
|
||||
/**
|
||||
* @internal
|
||||
*/
|
||||
export class CdpKeyboard {
|
||||
#pressedKeys = new Set<string>();
|
||||
|
||||
#client: InternalCDPSession;
|
||||
|
||||
_modifiers = 0;
|
||||
|
||||
constructor(client: InternalCDPSession) {
|
||||
this.#client = client;
|
||||
}
|
||||
|
||||
updateClient(client: InternalCDPSession): void {
|
||||
this.#client = client;
|
||||
}
|
||||
|
||||
async down(
|
||||
key: KeyInput,
|
||||
options: Readonly<KeyDownOptions> = {
|
||||
text: undefined,
|
||||
commands: [],
|
||||
},
|
||||
): Promise<void> {
|
||||
const description = this.#keyDescriptionForString(key);
|
||||
|
||||
const autoRepeat = this.#pressedKeys.has(description.code);
|
||||
this.#pressedKeys.add(description.code);
|
||||
this._modifiers |= this.#modifierBit(description.key);
|
||||
|
||||
const text = options.text === undefined ? description.text : options.text;
|
||||
await this.#client.send('Input.dispatchKeyEvent', {
|
||||
type: text ? 'keyDown' : 'rawKeyDown',
|
||||
modifiers: this._modifiers,
|
||||
windowsVirtualKeyCode: description.keyCode,
|
||||
code: description.code,
|
||||
key: description.key,
|
||||
text: text,
|
||||
unmodifiedText: text,
|
||||
autoRepeat,
|
||||
location: description.location,
|
||||
isKeypad: description.location === 3,
|
||||
commands: options.commands,
|
||||
});
|
||||
}
|
||||
|
||||
#modifierBit(key: string): number {
|
||||
if (key === 'Alt') {
|
||||
return 1;
|
||||
}
|
||||
if (key === 'Control') {
|
||||
return 2;
|
||||
}
|
||||
if (key === 'Meta') {
|
||||
return 4;
|
||||
}
|
||||
if (key === 'Shift') {
|
||||
return 8;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
#keyDescriptionForString(keyString: KeyInput): KeyDescription {
|
||||
const shift = this._modifiers & 8;
|
||||
const description = {
|
||||
key: '',
|
||||
keyCode: 0,
|
||||
code: '',
|
||||
text: '',
|
||||
location: 0,
|
||||
};
|
||||
|
||||
const definition = _keyDefinitions[keyString];
|
||||
assert(definition, `Unknown key: "${keyString}"`);
|
||||
|
||||
if (definition.key) {
|
||||
description.key = definition.key;
|
||||
}
|
||||
if (shift && definition.shiftKey) {
|
||||
description.key = definition.shiftKey;
|
||||
}
|
||||
|
||||
if (definition.keyCode) {
|
||||
description.keyCode = definition.keyCode;
|
||||
}
|
||||
if (shift && definition.shiftKeyCode) {
|
||||
description.keyCode = definition.shiftKeyCode;
|
||||
}
|
||||
|
||||
if (definition.code) {
|
||||
description.code = definition.code;
|
||||
}
|
||||
|
||||
if (definition.location) {
|
||||
description.location = definition.location;
|
||||
}
|
||||
|
||||
if (description.key.length === 1) {
|
||||
description.text = description.key;
|
||||
}
|
||||
|
||||
if (definition.text) {
|
||||
description.text = definition.text;
|
||||
}
|
||||
if (shift && definition.shiftText) {
|
||||
description.text = definition.shiftText;
|
||||
}
|
||||
|
||||
// if any modifiers besides shift are pressed, no text should be sent
|
||||
if (this._modifiers & ~8) {
|
||||
description.text = '';
|
||||
}
|
||||
|
||||
return description;
|
||||
}
|
||||
|
||||
async up(key: KeyInput): Promise<void> {
|
||||
const description = this.#keyDescriptionForString(key);
|
||||
|
||||
this._modifiers &= ~this.#modifierBit(description.key);
|
||||
this.#pressedKeys.delete(description.code);
|
||||
await this.#client.send('Input.dispatchKeyEvent', {
|
||||
type: 'keyUp',
|
||||
modifiers: this._modifiers,
|
||||
key: description.key,
|
||||
windowsVirtualKeyCode: description.keyCode,
|
||||
code: description.code,
|
||||
location: description.location,
|
||||
});
|
||||
}
|
||||
|
||||
async sendCharacter(char: string): Promise<void> {
|
||||
await this.#client.send('Input.insertText', { text: char });
|
||||
}
|
||||
|
||||
private charIsKey(char: string): char is KeyInput {
|
||||
return !!_keyDefinitions[char as KeyInput];
|
||||
}
|
||||
|
||||
async type(
|
||||
text: string,
|
||||
options: Readonly<KeyboardTypeOptions> = {},
|
||||
): Promise<void> {
|
||||
const delay = options.delay || undefined;
|
||||
for (const char of text) {
|
||||
if (this.charIsKey(char)) {
|
||||
await this.press(char, { delay });
|
||||
} else {
|
||||
if (delay) {
|
||||
await new Promise((f) => {
|
||||
return setTimeout(f, delay);
|
||||
});
|
||||
}
|
||||
await this.sendCharacter(char);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async press(
|
||||
key: KeyInput,
|
||||
options: Readonly<KeyPressOptions> = {},
|
||||
): Promise<void> {
|
||||
const { delay = null } = options;
|
||||
await this.down(key, options);
|
||||
if (delay) {
|
||||
await new Promise((f) => {
|
||||
return setTimeout(f, options.delay);
|
||||
});
|
||||
}
|
||||
await this.up(key);
|
||||
}
|
||||
}
|
||||
@ -12,6 +12,7 @@ import type { AbstractPage } from '@/page';
|
||||
import type { Point, Rect, Size } from '@midscene/core/.';
|
||||
import { ifInBrowser } from '@midscene/shared/utils';
|
||||
import type { Protocol as CDPTypes } from 'devtools-protocol';
|
||||
import { CdpKeyboard } from './cdpInput';
|
||||
|
||||
// remember to include this file into extension's package
|
||||
const scriptFileToRetrieve = './lib/htmlElement.js';
|
||||
@ -324,35 +325,16 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
|
||||
|
||||
keyboard = {
|
||||
type: async (text: string) => {
|
||||
for (const char of text) {
|
||||
await this.sendCommandToDebugger('Input.insertText', {
|
||||
text: char,
|
||||
});
|
||||
|
||||
// sleep 50ms
|
||||
await new Promise((resolve) => setTimeout(resolve, 50));
|
||||
}
|
||||
const cdpKeyboard = new CdpKeyboard({
|
||||
send: this.sendCommandToDebugger.bind(this),
|
||||
});
|
||||
await cdpKeyboard.type(text);
|
||||
},
|
||||
press: async (key: WebKeyInput) => {
|
||||
await this.sendCommandToDebugger('Input.dispatchKeyEvent', {
|
||||
type: 'rawKeyDown',
|
||||
code: key,
|
||||
key: key,
|
||||
});
|
||||
|
||||
// Dispatch 'char' event
|
||||
await this.sendCommandToDebugger('Input.dispatchKeyEvent', {
|
||||
type: 'char',
|
||||
code: key,
|
||||
key: key,
|
||||
});
|
||||
|
||||
// Dispatch 'keyUp' event
|
||||
await this.sendCommandToDebugger('Input.dispatchKeyEvent', {
|
||||
type: 'keyUp',
|
||||
code: key,
|
||||
key: key,
|
||||
const cdpKeyboard = new CdpKeyboard({
|
||||
send: this.sendCommandToDebugger.bind(this),
|
||||
});
|
||||
await cdpKeyboard.press(key);
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user