yujunjun/playwright
1
0
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-06-26 21:40:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: add playwright-mdd experiment (#36430)

Browse Source
This commit is contained in:
Pavel Feldman 2025-06-24 16:43:12 -07:00 committed by GitHub
parent 25e64e976e
commit 6b231cbf79
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
15 changed files with 1234 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
package-lock.json generated
View File

@ -1373,6 +1373,10 @@
"resolved": "packages/playwright-ct-vue",
"link": true
},
"node_modules/@playwright/mdd": {
"resolved": "packages/playwright-mdd",
"link": true
},
"node_modules/@playwright/test": {
"resolved": "packages/playwright-test",
"link": true
@ -1754,6 +1758,16 @@
"@types/tern": "*"
}
},
"node_modules/@types/debug": {
"version": "4.1.12",
"resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
"integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",
"dev": true,
"license": "MIT",
"dependencies": {
"@types/ms": "*"
}
},
"node_modules/@types/estree": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz",
@ -1812,6 +1826,13 @@
"@types/node": "*"
}
},
"node_modules/@types/ms": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz",
"integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==",
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": {
"version": "18.19.76",
"resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.76.tgz",
@ -2978,6 +2999,15 @@
"node": ">=0.1.90"
}
},
"node_modules/commander": {
"version": "13.1.0",
"resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz",
"integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==",
"license": "MIT",
"engines": {
"node": ">=18"
}
},
"node_modules/concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@ -3139,9 +3169,9 @@
}
},
"node_modules/debug": {
"version": "4.4.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
"integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
"version": "4.4.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
"integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
"license": "MIT",
"dependencies": {
"ms": "^2.1.3"
@ -3305,10 +3335,9 @@
}
},
"node_modules/dotenv": {
"version": "16.4.7",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz",
"integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==",
"dev": true,
"version": "16.5.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz",
"integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==",
"license": "BSD-2-Clause",
"engines": {
"node": ">=12"
@ -5933,6 +5962,27 @@
"wrappy": "1"
}
},
"node_modules/openai": {
"version": "5.7.0",
"resolved": "https://registry.npmjs.org/openai/-/openai-5.7.0.tgz",
"integrity": "sha512-zXWawZl6J/P5Wz57/nKzVT3kJQZvogfuyuNVCdEp4/XU2UNrjL7SsuNpWAyLZbo6HVymwmnfno9toVzBhelygA==",
"license": "Apache-2.0",
"bin": {
"openai": "bin/cli"
},
"peerDependencies": {
"ws": "^8.18.0",
"zod": "^3.23.8"
},
"peerDependenciesMeta": {
"ws": {
"optional": true
},
"zod": {
"optional": true
}
}
},
"node_modules/optionator": {
"version": "0.9.4",
"resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@ -7809,10 +7859,10 @@
"license": "ISC"
},
"node_modules/ws": {
"version": "8.18.0",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
"dev": true,
"version": "8.18.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz",
"integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==",
"devOptional": true,
"license": "MIT",
"engines": {
"node": ">=10.0.0"
@ -7956,12 +8006,20 @@
"version": "3.24.2",
"resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz",
"integrity": "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==",
"dev": true,
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/colinhacks"
}
},
"node_modules/zod-to-json-schema": {
"version": "3.24.5",
"resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz",
"integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==",
"license": "ISC",
"peerDependencies": {
"zod": "^3.24.1"
}
},
"packages/html-reporter": {
"version": "0.0.0"
},
@ -8706,6 +8764,44 @@
"node": ">=18"
}
},
"packages/playwright-mdd": {
"name": "@playwright/mdd",
"version": "0.0.1",
"license": "Apache-2.0",
"dependencies": {
"commander": "^13.1.0",
"debug": "^4.4.1",
"dotenv": "^16.5.0",
"mime": "^4.0.7",
"openai": "^5.7.0",
"playwright-core": "1.54.0-next",
"zod-to-json-schema": "^3.24.4"
},
"bin": {
"playwright-mdd": "cli.js"
},
"devDependencies": {
"@types/debug": "^4.1.7"
},
"engines": {
"node": ">=18"
}
},
"packages/playwright-mdd/node_modules/mime": {
"version": "4.0.7",
"resolved": "https://registry.npmjs.org/mime/-/mime-4.0.7.tgz",
"integrity": "sha512-2OfDPL+e03E0LrXaGYOtTFIYhiuzep94NSsuhrNULq+stylcJedcHdzHtz0atMUuGwJfFYs0YL5xeC/Ca2x0eQ==",
"funding": [
"https://github.com/sponsors/broofa"
],
"license": "MIT",
"bin": {
"mime": "bin/cli.js"
},
"engines": {
"node": ">=16"
}
},
"packages/playwright-test": {
"name": "@playwright/test",
"version": "1.54.0-next",

19
packages/playwright-mdd/cli.js Executable file
View File

@ -0,0 +1,19 @@
#!/usr/bin/env node
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
const { program } = require('./lib/program');
void program.parseAsync(process.argv);

33
packages/playwright-mdd/package.json Normal file
View File

@ -0,0 +1,33 @@
{
"name": "@playwright/mdd",
"version": "0.0.1",
"description": "Playwright MDD",
"private": true,
"repository": {
"type": "git",
"url": "git+https://github.com/microsoft/playwright.git"
},
"homepage": "https://playwright.dev",
"engines": {
"node": ">=18"
},
"author": {
"name": "Microsoft Corporation"
},
"license": "Apache-2.0",
"dependencies": {
"commander": "^13.1.0",
"debug": "^4.4.1",
"dotenv": "^16.5.0",
"mime": "^4.0.7",
"openai": "^5.7.0",
"playwright-core": "1.54.0-next",
"zod-to-json-schema": "^3.24.4"
},
"devDependencies": {
"@types/debug": "^4.1.7"
},
"bin": {
"playwright-mdd": "cli.js"
}
}

232
packages/playwright-mdd/src/context.ts Normal file
View File

@ -0,0 +1,232 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import debug from 'debug';
import * as playwright from 'playwright';
import { callOnPageNoTrace, waitForCompletion } from './tools/utils';
import { ManualPromise } from './manualPromise';
import type { ModalState, Tool, ToolActionResult } from './tools/tool';
type PendingAction = {
dialogShown: ManualPromise<void>;
};
type PageEx = playwright.Page & {
_snapshotForAI: () => Promise<string>;
};
const testDebug = debug('pw:mcp:test');
export class Context {
readonly browser: playwright.Browser;
readonly page: playwright.Page;
readonly tools: Tool[];
private _modalStates: ModalState[] = [];
private _pendingAction: PendingAction | undefined;
private _downloads: { download: playwright.Download, finished: boolean, outputFile: string }[] = [];
constructor(browser: playwright.Browser, page: playwright.Page, tools: Tool[]) {
this.browser = browser;
this.page = page;
this.tools = tools;
testDebug('create context');
}
static async create(tools: Tool[]): Promise<Context> {
const browser = await playwright.chromium.launch({
headless: false,
});
const context = await browser.newContext();
const page = await context.newPage();
return new Context(browser, page, tools);
}
async close() {
await this.browser.close();
}
modalStates(): ModalState[] {
return this._modalStates;
}
setModalState(modalState: ModalState) {
this._modalStates.push(modalState);
}
clearModalState(modalState: ModalState) {
this._modalStates = this._modalStates.filter(state => state !== modalState);
}
modalStatesMarkdown(): string[] {
const result: string[] = ['### Modal state'];
if (this._modalStates.length === 0)
result.push('- There is no modal state present');
for (const state of this._modalStates) {
const tool = this.tools.find(tool => tool.clearsModalState === state.type);
result.push(`- [${state.description}]: can be handled by the "${tool?.schema.name}" tool`);
}
return result;
}
async run(tool: Tool, params: Record<string, unknown> | undefined): Promise<{ content: string, code: string[] }> {
const toolResult = await tool.handle(this, tool.schema.inputSchema.parse(params || {}));
const { code, action, waitForNetwork, captureSnapshot } = toolResult;
const racingAction = action ? () => this._raceAgainstModalDialogs(action) : undefined;
if (waitForNetwork)
await waitForCompletion(this, async () => racingAction?.());
else
await racingAction?.();
const result: string[] = [];
if (this.modalStates().length) {
result.push(...this.modalStatesMarkdown());
return {
code,
content: result.join('\n'),
};
}
if (this._downloads.length) {
result.push('', '### Downloads');
for (const entry of this._downloads) {
if (entry.finished)
result.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`);
else
result.push(`- Downloading file ${entry.download.suggestedFilename()} ...`);
}
result.push('');
}
result.push(
`- Page URL: ${this.page.url()}`,
`- Page Title: ${await this.title()}`
);
if (captureSnapshot && !this._javaScriptBlocked())
result.push(await this._snapshot());
return {
code,
content: result.join('\n'),
};
}
async title(): Promise<string> {
return await callOnPageNoTrace(this.page, page => page.title());
}
async waitForTimeout(time: number) {
if (this._javaScriptBlocked()) {
await new Promise(f => setTimeout(f, time));
return;
}
await callOnPageNoTrace(this.page, page => {
return page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
});
}
async waitForLoadState(state: 'load', options?: { timeout?: number }): Promise<void> {
await callOnPageNoTrace(this.page, page => page.waitForLoadState(state, options).catch(() => {}));
}
async navigate(url: string) {
const downloadEvent = callOnPageNoTrace(this.page, page => page.waitForEvent('download').catch(() => {}));
try {
await this.page.goto(url, { waitUntil: 'domcontentloaded' });
} catch (_e: unknown) {
const e = _e as Error;
const mightBeDownload =
e.message.includes('net::ERR_ABORTED') // chromium
|| e.message.includes('Download is starting'); // firefox + webkit
if (!mightBeDownload)
throw e;
// on chromium, the download event is fired *after* page.goto rejects, so we wait a lil bit
const download = await Promise.race([
downloadEvent,
new Promise(resolve => setTimeout(resolve, 1000)),
]);
if (!download)
throw e;
}
// Cap load event to 5 seconds, the page is operational at this point.
await this.waitForLoadState('load', { timeout: 5000 });
}
refLocator(params: { element: string, ref: string }): playwright.Locator {
return this.page.locator(`aria-ref=${params.ref}`).describe(params.element);
}
private async _raceAgainstModalDialogs(action: () => Promise<ToolActionResult>): Promise<ToolActionResult> {
this._pendingAction = {
dialogShown: new ManualPromise(),
};
let result: ToolActionResult | undefined;
try {
await Promise.race([
action().then(r => result = r),
this._pendingAction.dialogShown,
]);
} finally {
this._pendingAction = undefined;
}
return result;
}
private _javaScriptBlocked(): boolean {
return this._modalStates.some(state => state.type === 'dialog');
}
dialogShown(dialog: playwright.Dialog) {
this.setModalState({
type: 'dialog',
description: `"${dialog.type()}" dialog with message "${dialog.message()}"`,
dialog,
});
this._pendingAction?.dialogShown.resolve();
}
async downloadStarted(download: playwright.Download) {
const entry = {
download,
finished: false,
outputFile: this._outputFile(download.suggestedFilename())
};
this._downloads.push(entry);
await download.saveAs(entry.outputFile);
entry.finished = true;
}
private async _snapshot() {
const snapshot = await callOnPageNoTrace(this.page, page => (page as PageEx)._snapshotForAI());
return [
`- Page Snapshot`,
'```yaml',
snapshot,
'```',
].join('\n');
}
private _outputFile(filename: string) {
return filename;
}
}

53
packages/playwright-mdd/src/format.ts Normal file
View File

@ -0,0 +1,53 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
// adapted from:
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts
// NOTE: this function should not be used to escape any selectors.
export function escapeWithQuotes(text: string, char: string = '\'') {
const stringified = JSON.stringify(text);
const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');
if (char === '\'')
return char + escapedText.replace(/[']/g, '\\\'') + char;
if (char === '"')
return char + escapedText.replace(/["]/g, '\\"') + char;
if (char === '`')
return char + escapedText.replace(/[`]/g, '`') + char;
throw new Error('Invalid escape char');
}
export function quote(text: string) {
return escapeWithQuotes(text, '\'');
}
export function formatObject(value: any, indent = ' '): string {
if (typeof value === 'string')
return quote(value);
if (Array.isArray(value))
return `[${value.map(o => formatObject(o)).join(', ')}]`;
if (typeof value === 'object') {
const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();
if (!keys.length)
return '{}';
const tokens: string[] = [];
for (const key of keys)
tokens.push(`${key}: ${formatObject(value[key])}`);
return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;
}
return String(value);
}

108
packages/playwright-mdd/src/loop.ts Normal file
View File

@ -0,0 +1,108 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import OpenAI from 'openai';
import debug from 'debug';
import { zodToJsonSchema } from 'zod-to-json-schema';
import { Tool } from './tools/tool';
import { Context } from './context';
/* eslint-disable no-console */
export async function runTasks(context: Context, tasks: string[]): Promise<string> {
const openai = new OpenAI();
const allCode: string[] = [
`test('generated code', async ({ page }) => {`,
];
for (const task of tasks) {
const { taskCode } = await runTask(openai, context, task);
if (taskCode.length)
allCode.push('', ...taskCode.map(code => ` ${code}`));
}
allCode.push('});');
return allCode.join('\n');
}
async function runTask(openai: OpenAI, context: Context, task: string): Promise<{ taskCode: string[] }> {
console.log('Perform task:', task);
const taskCode: string[] = [
`// ${task}`,
];
const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
{
role: 'user',
content: `Peform following task: ${task}. Once the task is complete, call the "done" tool.`
}
];
for (let iteration = 0; iteration < 5; ++iteration) {
debug('history')(messages);
const response = await openai.chat.completions.create({
model: 'gpt-4.1',
messages,
tools: context.tools.map(asOpenAIDeclaration),
tool_choice: 'auto'
});
const message = response.choices[0].message;
if (!message.tool_calls?.length)
throw new Error('Unexpected response from LLM: ' + message.content);
messages.push({
role: 'assistant',
tool_calls: message.tool_calls
});
for (const toolCall of message.tool_calls) {
const functionCall = toolCall.function;
console.log('Call tool:', functionCall.name, functionCall.arguments);
if (functionCall.name === 'done')
return { taskCode };
const tool = context.tools.find(tool => tool.schema.name === functionCall.name);
if (!tool)
throw new Error('Unknown tool: ' + functionCall.name);
const { code, content } = await context.run(tool, JSON.parse(functionCall.arguments));
taskCode.push(...code);
messages.push({
role: 'tool',
tool_call_id: toolCall.id,
content,
});
}
}
throw new Error('Failed to perform step, max attempts reached');
}
function asOpenAIDeclaration(tool: Tool<any>): OpenAI.Chat.Completions.ChatCompletionTool {
const parameters = zodToJsonSchema(tool.schema.inputSchema);
delete parameters.$schema;
delete (parameters as any).additionalProperties;
return {
type: 'function',
function: {
name: tool.schema.name,
description: tool.schema.description,
parameters,
},
};
}

127
packages/playwright-mdd/src/manualPromise.ts Normal file
View File

@ -0,0 +1,127 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
export class ManualPromise<T = void> extends Promise<T> {
private _resolve!: (t: T) => void;
private _reject!: (e: Error) => void;
private _isDone: boolean;
constructor() {
let resolve: (t: T) => void;
let reject: (e: Error) => void;
super((f, r) => {
resolve = f;
reject = r;
});
this._isDone = false;
this._resolve = resolve!;
this._reject = reject!;
}
isDone() {
return this._isDone;
}
resolve(t: T) {
this._isDone = true;
this._resolve(t);
}
reject(e: Error) {
this._isDone = true;
this._reject(e);
}
static override get [Symbol.species]() {
return Promise;
}
override get [Symbol.toStringTag]() {
return 'ManualPromise';
}
}
export class LongStandingScope {
private _terminateError: Error | undefined;
private _closeError: Error | undefined;
private _terminatePromises = new Map<ManualPromise<Error>, string[]>();
private _isClosed = false;
reject(error: Error) {
this._isClosed = true;
this._terminateError = error;
for (const p of this._terminatePromises.keys())
p.resolve(error);
}
close(error: Error) {
this._isClosed = true;
this._closeError = error;
for (const [p, frames] of this._terminatePromises)
p.resolve(cloneError(error, frames));
}
isClosed() {
return this._isClosed;
}
static async raceMultiple<T>(scopes: LongStandingScope[], promise: Promise<T>): Promise<T> {
return Promise.race(scopes.map(s => s.race(promise)));
}
async race<T>(promise: Promise<T> | Promise<T>[]): Promise<T> {
return this._race(Array.isArray(promise) ? promise : [promise], false) as Promise<T>;
}
async safeRace<T>(promise: Promise<T>, defaultValue?: T): Promise<T> {
return this._race([promise], true, defaultValue);
}
private async _race(promises: Promise<any>[], safe: boolean, defaultValue?: any): Promise<any> {
const terminatePromise = new ManualPromise<Error>();
const frames = captureRawStack();
if (this._terminateError)
terminatePromise.resolve(this._terminateError);
if (this._closeError)
terminatePromise.resolve(cloneError(this._closeError, frames));
this._terminatePromises.set(terminatePromise, frames);
try {
return await Promise.race([
terminatePromise.then(e => safe ? defaultValue : Promise.reject(e)),
...promises
]);
} finally {
this._terminatePromises.delete(terminatePromise);
}
}
}
function cloneError(error: Error, frames: string[]) {
const clone = new Error();
clone.name = error.name;
clone.message = error.message;
clone.stack = [error.name + ':' + error.message, ...frames].join('\n');
return clone;
}
function captureRawStack(): string[] {
const stackTraceLimit = Error.stackTraceLimit;
Error.stackTraceLimit = 50;
const error = new Error();
const stack = error.stack || '';
Error.stackTraceLimit = stackTraceLimit;
return stack.split('\n');
}

50
packages/playwright-mdd/src/program.ts Normal file
View File

@ -0,0 +1,50 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import dotenv from 'dotenv';
import { program } from 'commander';
import { runTasks } from './loop';
import { Context } from './context';
import { tools } from './tools';
/* eslint-disable no-console */
dotenv.config();
const packageJSON = require('../package.json');
program
.version('Version ' + packageJSON.version)
.name(packageJSON.name)
.action(async () => {
const context = await Context.create(tools);
const code = await runTasks(context, script);
console.log('Output code:');
console.log('```javascript');
console.log(code);
console.log('```');
await context.close();
});
const script = [
'Navigate to https://debs-obrien.github.io/playwright-movies-app',
'Click search icon',
'Type "Twister" in the search field and hit Enter',
'Click on the link for the movie "Twisters"',
];
export { program };

27
packages/playwright-mdd/src/tools.ts Normal file
View File

@ -0,0 +1,27 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import snapshot from './tools/snapshot';
import done from './tools/done';
import navigate from './tools/navigate';
import type { Tool } from './tools/tool.js';
export const tools: Tool<any>[] = [
...navigate,
...snapshot,
...done,
];

39
packages/playwright-mdd/src/tools/done.ts Normal file
View File

@ -0,0 +1,39 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { z } from 'zod';
import { defineTool } from './tool';
const doneTool = defineTool({
schema: {
name: 'done',
description: 'Call this tool to indicate that the task is complete',
inputSchema: z.object({}),
},
handle: async () => {
return {
code: [],
captureSnapshot: false,
waitForNetwork: false,
};
},
});
export default [
doneTool,
];

88
packages/playwright-mdd/src/tools/navigate.ts Normal file
View File

@ -0,0 +1,88 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { z } from 'zod';
import { defineTool } from './tool.js';
const navigate = defineTool({
schema: {
name: 'browser_navigate',
description: 'Navigate to a URL',
inputSchema: z.object({
url: z.string().describe('The URL to navigate to'),
}),
},
handle: async (context, params) => {
const code = [
`await page.goto('${params.url}');`,
];
await context.navigate(params.url);
return {
code,
captureSnapshot: true,
waitForNetwork: false,
};
},
});
const goBack = defineTool({
schema: {
name: 'browser_navigate_back',
description: 'Go back to the previous page',
inputSchema: z.object({}),
},
handle: async context => {
await context.page.goBack();
const code = [
`await page.goBack();`,
];
return {
code,
captureSnapshot: true,
waitForNetwork: false,
};
},
});
const goForward = defineTool({
schema: {
name: 'browser_navigate_forward',
description: 'Go forward to the next page',
inputSchema: z.object({}),
},
handle: async context => {
await context.page.goForward();
const code = [
`await page.goForward();`,
];
return {
code,
captureSnapshot: true,
waitForNetwork: false,
};
},
});
export default [
navigate,
goBack,
goForward,
];

194
packages/playwright-mdd/src/tools/snapshot.ts Normal file
View File

@ -0,0 +1,194 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import { z } from 'zod';
import { defineTool } from './tool';
import * as format from '../format';
import { generateLocator } from './utils';
const snapshot = defineTool({
schema: {
name: 'browser_snapshot',
description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
inputSchema: z.object({}),
},
handle: async () => {
return {
code: [],
captureSnapshot: true,
waitForNetwork: false,
};
},
});
const elementSchema = z.object({
element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
ref: z.string().describe('Exact target element reference from the page snapshot'),
});
const click = defineTool({
schema: {
name: 'browser_click',
description: 'Perform click on a web page',
inputSchema: elementSchema,
},
handle: async (context, params) => {
const locator = context.refLocator(params);
const code = [
`await page.${await generateLocator(locator)}.click();`
];
return {
code,
action: () => locator.click(),
captureSnapshot: true,
waitForNetwork: true,
};
},
});
const drag = defineTool({
schema: {
name: 'browser_drag',
description: 'Perform drag and drop between two elements',
inputSchema: z.object({
startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),
startRef: z.string().describe('Exact source element reference from the page snapshot'),
endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),
endRef: z.string().describe('Exact target element reference from the page snapshot'),
}),
},
handle: async (context, params) => {
const startLocator = context.refLocator({ ref: params.startRef, element: params.startElement });
const endLocator = context.refLocator({ ref: params.endRef, element: params.endElement });
const code = [
`await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`
];
return {
code,
action: () => startLocator.dragTo(endLocator),
captureSnapshot: true,
waitForNetwork: true,
};
},
});
const hover = defineTool({
schema: {
name: 'browser_hover',
description: 'Hover over element on page',
inputSchema: elementSchema,
},
handle: async (context, params) => {
const locator = context.refLocator(params);
const code = [
`await page.${await generateLocator(locator)}.hover();`
];
return {
code,
action: () => locator.hover(),
captureSnapshot: true,
waitForNetwork: true,
};
},
});
const typeSchema = elementSchema.extend({
text: z.string().describe('Text to type into the element'),
submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),
slowly: z.boolean().optional().describe('Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.'),
});
const type = defineTool({
schema: {
name: 'browser_type',
description: 'Type text into editable element',
inputSchema: typeSchema,
},
handle: async (context, params) => {
const locator = context.refLocator(params);
const code: string[] = [];
const steps: (() => Promise<void>)[] = [];
if (params.slowly) {
code.push(`await page.${await generateLocator(locator)}.pressSequentially(${format.quote(params.text)});`);
steps.push(() => locator.pressSequentially(params.text));
} else {
code.push(`await page.${await generateLocator(locator)}.fill(${format.quote(params.text)});`);
steps.push(() => locator.fill(params.text));
}
if (params.submit) {
code.push(`await page.${await generateLocator(locator)}.press('Enter');`);
steps.push(() => locator.press('Enter'));
}
return {
code,
action: () => steps.reduce((acc, step) => acc.then(step), Promise.resolve()),
captureSnapshot: true,
waitForNetwork: true,
};
},
});
const selectOptionSchema = elementSchema.extend({
values: z.array(z.string()).describe('Array of values to select in the dropdown. This can be a single value or multiple values.'),
});
const selectOption = defineTool({
schema: {
name: 'browser_select_option',
description: 'Select an option in a dropdown',
inputSchema: selectOptionSchema,
},
handle: async (context, params) => {
const locator = context.refLocator(params);
const code = [
`await page.${await generateLocator(locator)}.selectOption(${format.formatObject(params.values)});`
];
return {
code,
action: () => locator.selectOption(params.values).then(() => {}),
captureSnapshot: true,
waitForNetwork: true,
};
},
});
export default [
snapshot,
click,
drag,
hover,
type,
selectOption,
];

60
packages/playwright-mdd/src/tools/tool.ts Normal file
View File

@ -0,0 +1,60 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type { z } from 'zod';
import type * as playwright from 'playwright-core';
import type { Context } from '../context';
export type ToolSchema<Input extends InputType> = {
name: string;
description: string;
inputSchema: Input;
};
type InputType = z.Schema;
export type FileUploadModalState = {
type: 'fileChooser';
description: string;
fileChooser: playwright.FileChooser;
};
export type DialogModalState = {
type: 'dialog';
description: string;
dialog: playwright.Dialog;
};
export type ModalState = FileUploadModalState | DialogModalState;
export type ToolActionResult = string | undefined | void;
export type ToolResult = {
code: string[];
action?: () => Promise<void>;
captureSnapshot: boolean;
waitForNetwork: boolean;
};
export type Tool<Input extends InputType = InputType> = {
schema: ToolSchema<Input>;
clearsModalState?: ModalState['type'];
handle: (context: Context, params: z.output<Input>) => Promise<ToolResult>;
};
export function defineTool<Input extends InputType>(tool: Tool<Input>): Tool<Input> {
return tool;
}

91
packages/playwright-mdd/src/tools/utils.ts Normal file
View File

@ -0,0 +1,91 @@
/**
* Copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import type * as playwright from 'playwright';
import type { Context } from '../context';
export async function waitForCompletion<R>(context: Context, callback: () => Promise<R>): Promise<R> {
const requests = new Set<playwright.Request>();
let frameNavigated = false;
let waitCallback: () => void = () => {};
const waitBarrier = new Promise<void>(f => { waitCallback = f; });
const requestListener = (request: playwright.Request) => requests.add(request);
const requestFinishedListener = (request: playwright.Request) => {
requests.delete(request);
if (!requests.size)
waitCallback();
};
const frameNavigateListener = (frame: playwright.Frame) => {
if (frame.parentFrame())
return;
frameNavigated = true;
dispose();
clearTimeout(timeout);
void context.waitForLoadState('load').then(waitCallback);
};
const onTimeout = () => {
dispose();
waitCallback();
};
context.page.on('request', requestListener);
context.page.on('requestfinished', requestFinishedListener);
context.page.on('framenavigated', frameNavigateListener);
const timeout = setTimeout(onTimeout, 10000);
const dispose = () => {
context.page.off('request', requestListener);
context.page.off('requestfinished', requestFinishedListener);
context.page.off('framenavigated', frameNavigateListener);
clearTimeout(timeout);
};
try {
const result = await callback();
if (!requests.size && !frameNavigated)
waitCallback();
await waitBarrier;
await context.page.waitForTimeout(1000);
return result;
} finally {
dispose();
}
}
export function sanitizeForFilePath(s: string) {
const sanitize = (s: string) => s.replace(/[\x00-\x2C\x2E-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+/g, '-');
const separator = s.lastIndexOf('.');
if (separator === -1)
return sanitize(s);
return sanitize(s.substring(0, separator)) + '.' + sanitize(s.substring(separator + 1));
}
export async function generateLocator(locator: playwright.Locator): Promise<string> {
try {
return await (locator as any)._generateLocatorString();
} catch (e) {
if (e instanceof Error && /locator._generateLocatorString: Timeout .* exceeded/.test(e.message))
throw new Error('Ref not found, likely because element was removed. Use browser_snapshot to see what elements are currently on the page.');
throw e;
}
}
export async function callOnPageNoTrace<T>(page: playwright.Page, callback: (page: playwright.Page) => Promise<T>): Promise<T> {
return await (page as any)._wrapApiCall(() => callback(page), { internal: true });
}

5
utils/workspace.js
View File

@ -219,6 +219,11 @@ const workspace = new Workspace(ROOT_PATH, [
path: path.join(ROOT_PATH, 'packages', 'playwright-ct-vue'),
files: ['LICENSE'],
}),
new PWPackage({
name: 'playwright-mdd',
path: path.join(ROOT_PATH, 'packages', 'playwright-mdd'),
files: ['LICENSE'],
}),
]);
if (require.main === module) {