chore: add playwright-mdd experiment (#36430)

package-lock.json

@@ -1373,6 +1373,10 @@
       "resolved": "packages/playwright-ct-vue",
       "link": true
     },
+    "node_modules/@playwright/mdd": {
+      "resolved": "packages/playwright-mdd",
+      "link": true
+    },
     "node_modules/@playwright/test": {
       "resolved": "packages/playwright-test",
       "link": true
@@ -1754,6 +1758,16 @@
         "@types/tern": "*"
       }
     },
+    "node_modules/@types/debug": {
+      "version": "4.1.12",
+      "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz",
+      "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "@types/ms": "*"
+      }
+    },
     "node_modules/@types/estree": {
       "version": "1.0.7",
       "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz",
@@ -1812,6 +1826,13 @@
         "@types/node": "*"
       }
     },
+    "node_modules/@types/ms": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz",
+      "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/node": {
       "version": "18.19.76",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.76.tgz",
@@ -2978,6 +2999,15 @@
         "node": ">=0.1.90"
       }
     },
+    "node_modules/commander": {
+      "version": "13.1.0",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-13.1.0.tgz",
+      "integrity": "sha512-/rFeCpNJQbhSZjGVwO9RFV3xPqbnERS8MmIQzCtD/zl6gpJuV/bMLuN92oG3F7d8oDEHHRrujSXNUr8fpjntKw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3139,9 +3169,9 @@
       }
     },
     "node_modules/debug": {
-      "version": "4.4.0",
+      "version": "4.4.1",
-      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz",
-      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+      "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==",
       "license": "MIT",
       "dependencies": {
         "ms": "^2.1.3"
@@ -3305,10 +3335,9 @@
       }
     },
     "node_modules/dotenv": {
-      "version": "16.4.7",
+      "version": "16.5.0",
-      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.4.7.tgz",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.5.0.tgz",
-      "integrity": "sha512-47qPchRCykZC03FhkYAhrvwU4xDBFIj1QPqaarj6mdM/hgUzfPHcpkHJOn3mJAufFeeAxAzeGsr5X0M4k6fLZQ==",
+      "integrity": "sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==",
-      "dev": true,
       "license": "BSD-2-Clause",
       "engines": {
         "node": ">=12"
@@ -5933,6 +5962,27 @@
         "wrappy": "1"
       }
     },
+    "node_modules/openai": {
+      "version": "5.7.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-5.7.0.tgz",
+      "integrity": "sha512-zXWawZl6J/P5Wz57/nKzVT3kJQZvogfuyuNVCdEp4/XU2UNrjL7SsuNpWAyLZbo6HVymwmnfno9toVzBhelygA==",
+      "license": "Apache-2.0",
+      "bin": {
+        "openai": "bin/cli"
+      },
+      "peerDependencies": {
+        "ws": "^8.18.0",
+        "zod": "^3.23.8"
+      },
+      "peerDependenciesMeta": {
+        "ws": {
+          "optional": true
+        },
+        "zod": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/optionator": {
       "version": "0.9.4",
       "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz",
@@ -7809,10 +7859,10 @@
       "license": "ISC"
     },
     "node_modules/ws": {
-      "version": "8.18.0",
+      "version": "8.18.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.0.tgz",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.2.tgz",
-      "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
+      "integrity": "sha512-DMricUmwGZUVr++AEAe2uiVM7UoO9MAVZMDu05UQOaUII0lp+zOzLLU4Xqh/JvTqklB1T4uELaaPBKyjE1r4fQ==",
-      "dev": true,
+      "devOptional": true,
       "license": "MIT",
       "engines": {
         "node": ">=10.0.0"
@@ -7956,12 +8006,20 @@
       "version": "3.24.2",
       "resolved": "https://registry.npmjs.org/zod/-/zod-3.24.2.tgz",
       "integrity": "sha512-lY7CDW43ECgW9u1TcT3IoXHflywfVqDYze4waEz812jR/bZ8FHDsl7pFQoSZTz5N+2NqRXs8GBwnAwo3ZNxqhQ==",
-      "dev": true,
       "license": "MIT",
       "funding": {
         "url": "https://github.com/sponsors/colinhacks"
       }
     },
+    "node_modules/zod-to-json-schema": {
+      "version": "3.24.5",
+      "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz",
+      "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==",
+      "license": "ISC",
+      "peerDependencies": {
+        "zod": "^3.24.1"
+      }
+    },
     "packages/html-reporter": {
       "version": "0.0.0"
     },
@@ -8706,6 +8764,44 @@
         "node": ">=18"
       }
     },
+    "packages/playwright-mdd": {
+      "name": "@playwright/mdd",
+      "version": "0.0.1",
+      "license": "Apache-2.0",
+      "dependencies": {
+        "commander": "^13.1.0",
+        "debug": "^4.4.1",
+        "dotenv": "^16.5.0",
+        "mime": "^4.0.7",
+        "openai": "^5.7.0",
+        "playwright-core": "1.54.0-next",
+        "zod-to-json-schema": "^3.24.4"
+      },
+      "bin": {
+        "playwright-mdd": "cli.js"
+      },
+      "devDependencies": {
+        "@types/debug": "^4.1.7"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
+    "packages/playwright-mdd/node_modules/mime": {
+      "version": "4.0.7",
+      "resolved": "https://registry.npmjs.org/mime/-/mime-4.0.7.tgz",
+      "integrity": "sha512-2OfDPL+e03E0LrXaGYOtTFIYhiuzep94NSsuhrNULq+stylcJedcHdzHtz0atMUuGwJfFYs0YL5xeC/Ca2x0eQ==",
+      "funding": [
+        "https://github.com/sponsors/broofa"
+      ],
+      "license": "MIT",
+      "bin": {
+        "mime": "bin/cli.js"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
     "packages/playwright-test": {
       "name": "@playwright/test",
       "version": "1.54.0-next",

packages/playwright-mdd/cli.js

@@ -0,0 +1,19 @@
+#!/usr/bin/env node
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const { program } = require('./lib/program');
+void program.parseAsync(process.argv);

packages/playwright-mdd/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@playwright/mdd",
+  "version": "0.0.1",
+  "description": "Playwright MDD",
+  "private": true,
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/microsoft/playwright.git"
+  },
+  "homepage": "https://playwright.dev",
+  "engines": {
+    "node": ">=18"
+  },
+  "author": {
+    "name": "Microsoft Corporation"
+  },
+  "license": "Apache-2.0",
+  "dependencies": {
+    "commander": "^13.1.0",
+    "debug": "^4.4.1",
+    "dotenv": "^16.5.0",
+    "mime": "^4.0.7",
+    "openai": "^5.7.0",
+    "playwright-core": "1.54.0-next",
+    "zod-to-json-schema": "^3.24.4"
+  },
+  "devDependencies": {
+    "@types/debug": "^4.1.7"
+  },
+  "bin": {
+    "playwright-mdd": "cli.js"
+  }
+}

packages/playwright-mdd/src/context.ts

@@ -0,0 +1,232 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import debug from 'debug';
+import * as playwright from 'playwright';
+
+import { callOnPageNoTrace, waitForCompletion } from './tools/utils';
+import { ManualPromise } from './manualPromise';
+
+import type { ModalState, Tool, ToolActionResult } from './tools/tool';
+
+type PendingAction = {
+  dialogShown: ManualPromise<void>;
+};
+
+type PageEx = playwright.Page & {
+  _snapshotForAI: () => Promise<string>;
+};
+
+const testDebug = debug('pw:mcp:test');
+
+export class Context {
+  readonly browser: playwright.Browser;
+  readonly page: playwright.Page;
+  readonly tools: Tool[];
+  private _modalStates: ModalState[] = [];
+  private _pendingAction: PendingAction | undefined;
+  private _downloads: { download: playwright.Download, finished: boolean, outputFile: string }[] = [];
+
+  constructor(browser: playwright.Browser, page: playwright.Page, tools: Tool[]) {
+    this.browser = browser;
+    this.page = page;
+    this.tools = tools;
+    testDebug('create context');
+  }
+
+  static async create(tools: Tool[]): Promise<Context> {
+    const browser = await playwright.chromium.launch({
+      headless: false,
+    });
+    const context = await browser.newContext();
+    const page = await context.newPage();
+    return new Context(browser, page, tools);
+  }
+
+  async close() {
+    await this.browser.close();
+  }
+
+  modalStates(): ModalState[] {
+    return this._modalStates;
+  }
+
+  setModalState(modalState: ModalState) {
+    this._modalStates.push(modalState);
+  }
+
+  clearModalState(modalState: ModalState) {
+    this._modalStates = this._modalStates.filter(state => state !== modalState);
+  }
+
+  modalStatesMarkdown(): string[] {
+    const result: string[] = ['### Modal state'];
+    if (this._modalStates.length === 0)
+      result.push('- There is no modal state present');
+    for (const state of this._modalStates) {
+      const tool = this.tools.find(tool => tool.clearsModalState === state.type);
+      result.push(`- [${state.description}]: can be handled by the "${tool?.schema.name}" tool`);
+    }
+    return result;
+  }
+
+  async run(tool: Tool, params: Record<string, unknown> | undefined): Promise<{ content: string, code: string[] }> {
+    const toolResult = await tool.handle(this, tool.schema.inputSchema.parse(params || {}));
+    const { code, action, waitForNetwork, captureSnapshot } = toolResult;
+    const racingAction = action ? () => this._raceAgainstModalDialogs(action) : undefined;
+
+    if (waitForNetwork)
+      await waitForCompletion(this, async () => racingAction?.());
+    else
+      await racingAction?.();
+
+    const result: string[] = [];
+
+    if (this.modalStates().length) {
+      result.push(...this.modalStatesMarkdown());
+      return {
+        code,
+        content: result.join('\n'),
+      };
+    }
+
+    if (this._downloads.length) {
+      result.push('', '### Downloads');
+      for (const entry of this._downloads) {
+        if (entry.finished)
+          result.push(`- Downloaded file ${entry.download.suggestedFilename()} to ${entry.outputFile}`);
+        else
+          result.push(`- Downloading file ${entry.download.suggestedFilename()} ...`);
+      }
+      result.push('');
+    }
+
+    result.push(
+        `- Page URL: ${this.page.url()}`,
+        `- Page Title: ${await this.title()}`
+    );
+
+    if (captureSnapshot && !this._javaScriptBlocked())
+      result.push(await this._snapshot());
+
+    return {
+      code,
+      content: result.join('\n'),
+    };
+  }
+
+  async title(): Promise<string> {
+    return await callOnPageNoTrace(this.page, page => page.title());
+  }
+
+  async waitForTimeout(time: number) {
+    if (this._javaScriptBlocked()) {
+      await new Promise(f => setTimeout(f, time));
+      return;
+    }
+
+    await callOnPageNoTrace(this.page, page => {
+      return page.evaluate(() => new Promise(f => setTimeout(f, 1000)));
+    });
+  }
+
+  async waitForLoadState(state: 'load', options?: { timeout?: number }): Promise<void> {
+    await callOnPageNoTrace(this.page, page => page.waitForLoadState(state, options).catch(() => {}));
+  }
+
+  async navigate(url: string) {
+    const downloadEvent = callOnPageNoTrace(this.page, page => page.waitForEvent('download').catch(() => {}));
+    try {
+      await this.page.goto(url, { waitUntil: 'domcontentloaded' });
+    } catch (_e: unknown) {
+      const e = _e as Error;
+      const mightBeDownload =
+        e.message.includes('net::ERR_ABORTED') // chromium
+        || e.message.includes('Download is starting'); // firefox + webkit
+      if (!mightBeDownload)
+        throw e;
+      // on chromium, the download event is fired *after* page.goto rejects, so we wait a lil bit
+      const download = await Promise.race([
+        downloadEvent,
+        new Promise(resolve => setTimeout(resolve, 1000)),
+      ]);
+      if (!download)
+        throw e;
+    }
+
+    // Cap load event to 5 seconds, the page is operational at this point.
+    await this.waitForLoadState('load', { timeout: 5000 });
+  }
+
+  refLocator(params: { element: string, ref: string }): playwright.Locator {
+    return this.page.locator(`aria-ref=${params.ref}`).describe(params.element);
+  }
+
+  private async _raceAgainstModalDialogs(action: () => Promise<ToolActionResult>): Promise<ToolActionResult> {
+    this._pendingAction = {
+      dialogShown: new ManualPromise(),
+    };
+
+    let result: ToolActionResult | undefined;
+    try {
+      await Promise.race([
+        action().then(r => result = r),
+        this._pendingAction.dialogShown,
+      ]);
+    } finally {
+      this._pendingAction = undefined;
+    }
+    return result;
+  }
+
+  private _javaScriptBlocked(): boolean {
+    return this._modalStates.some(state => state.type === 'dialog');
+  }
+
+  dialogShown(dialog: playwright.Dialog) {
+    this.setModalState({
+      type: 'dialog',
+      description: `"${dialog.type()}" dialog with message "${dialog.message()}"`,
+      dialog,
+    });
+    this._pendingAction?.dialogShown.resolve();
+  }
+
+  async downloadStarted(download: playwright.Download) {
+    const entry = {
+      download,
+      finished: false,
+      outputFile: this._outputFile(download.suggestedFilename())
+    };
+    this._downloads.push(entry);
+    await download.saveAs(entry.outputFile);
+    entry.finished = true;
+  }
+
+  private async _snapshot() {
+    const snapshot = await callOnPageNoTrace(this.page, page => (page as PageEx)._snapshotForAI());
+    return [
+      `- Page Snapshot`,
+      '```yaml',
+      snapshot,
+      '```',
+    ].join('\n');
+  }
+
+  private _outputFile(filename: string) {
+    return filename;
+  }
+}

packages/playwright-mdd/src/format.ts

@@ -0,0 +1,53 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// adapted from:
+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/utils/isomorphic/stringUtils.ts
+// - https://github.com/microsoft/playwright/blob/76ee48dc9d4034536e3ec5b2c7ce8be3b79418a8/packages/playwright-core/src/server/codegen/javascript.ts
+
+// NOTE: this function should not be used to escape any selectors.
+export function escapeWithQuotes(text: string, char: string = '\'') {
+  const stringified = JSON.stringify(text);
+  const escapedText = stringified.substring(1, stringified.length - 1).replace(/\\"/g, '"');
+  if (char === '\'')
+    return char + escapedText.replace(/[']/g, '\\\'') + char;
+  if (char === '"')
+    return char + escapedText.replace(/["]/g, '\\"') + char;
+  if (char === '`')
+    return char + escapedText.replace(/[`]/g, '`') + char;
+  throw new Error('Invalid escape char');
+}
+
+export function quote(text: string) {
+  return escapeWithQuotes(text, '\'');
+}
+
+export function formatObject(value: any, indent = '  '): string {
+  if (typeof value === 'string')
+    return quote(value);
+  if (Array.isArray(value))
+    return `[${value.map(o => formatObject(o)).join(', ')}]`;
+  if (typeof value === 'object') {
+    const keys = Object.keys(value).filter(key => value[key] !== undefined).sort();
+    if (!keys.length)
+      return '{}';
+    const tokens: string[] = [];
+    for (const key of keys)
+      tokens.push(`${key}: ${formatObject(value[key])}`);
+    return `{\n${indent}${tokens.join(`,\n${indent}`)}\n}`;
+  }
+  return String(value);
+}

packages/playwright-mdd/src/loop.ts

@@ -0,0 +1,108 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import OpenAI from 'openai';
+import debug from 'debug';
+import { zodToJsonSchema } from 'zod-to-json-schema';
+
+import { Tool } from './tools/tool';
+import { Context } from './context';
+
+/* eslint-disable no-console */
+
+export async function runTasks(context: Context, tasks: string[]): Promise<string> {
+  const openai = new OpenAI();
+  const allCode: string[] = [
+    `test('generated code', async ({ page }) => {`,
+  ];
+  for (const task of tasks) {
+    const { taskCode } = await runTask(openai, context, task);
+    if (taskCode.length)
+      allCode.push('', ...taskCode.map(code => `  ${code}`));
+  }
+  allCode.push('});');
+  return allCode.join('\n');
+}
+
+async function runTask(openai: OpenAI, context: Context, task: string): Promise<{ taskCode: string[] }> {
+  console.log('Perform task:', task);
+
+  const taskCode: string[] = [
+    `// ${task}`,
+  ];
+
+  const messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[] = [
+    {
+      role: 'user',
+      content: `Peform following task: ${task}. Once the task is complete, call the "done" tool.`
+    }
+  ];
+
+  for (let iteration = 0; iteration < 5; ++iteration) {
+    debug('history')(messages);
+    const response = await openai.chat.completions.create({
+      model: 'gpt-4.1',
+      messages,
+      tools: context.tools.map(asOpenAIDeclaration),
+      tool_choice: 'auto'
+    });
+
+    const message = response.choices[0].message;
+    if (!message.tool_calls?.length)
+      throw new Error('Unexpected response from LLM: ' + message.content);
+
+    messages.push({
+      role: 'assistant',
+      tool_calls: message.tool_calls
+    });
+
+    for (const toolCall of message.tool_calls) {
+      const functionCall = toolCall.function;
+      console.log('Call tool:', functionCall.name, functionCall.arguments);
+
+      if (functionCall.name === 'done')
+        return { taskCode };
+
+      const tool = context.tools.find(tool => tool.schema.name === functionCall.name);
+      if (!tool)
+        throw new Error('Unknown tool: ' + functionCall.name);
+
+      const { code, content } = await context.run(tool, JSON.parse(functionCall.arguments));
+      taskCode.push(...code);
+
+      messages.push({
+        role: 'tool',
+        tool_call_id: toolCall.id,
+        content,
+      });
+    }
+  }
+  throw new Error('Failed to perform step, max attempts reached');
+}
+
+function asOpenAIDeclaration(tool: Tool<any>): OpenAI.Chat.Completions.ChatCompletionTool {
+  const parameters = zodToJsonSchema(tool.schema.inputSchema);
+  delete parameters.$schema;
+  delete (parameters as any).additionalProperties;
+  return {
+    type: 'function',
+    function: {
+      name: tool.schema.name,
+      description: tool.schema.description,
+      parameters,
+    },
+  };
+}

packages/playwright-mdd/src/manualPromise.ts

@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export class ManualPromise<T = void> extends Promise<T> {
+  private _resolve!: (t: T) => void;
+  private _reject!: (e: Error) => void;
+  private _isDone: boolean;
+
+  constructor() {
+    let resolve: (t: T) => void;
+    let reject: (e: Error) => void;
+    super((f, r) => {
+      resolve = f;
+      reject = r;
+    });
+    this._isDone = false;
+    this._resolve = resolve!;
+    this._reject = reject!;
+  }
+
+  isDone() {
+    return this._isDone;
+  }
+
+  resolve(t: T) {
+    this._isDone = true;
+    this._resolve(t);
+  }
+
+  reject(e: Error) {
+    this._isDone = true;
+    this._reject(e);
+  }
+
+  static override get [Symbol.species]() {
+    return Promise;
+  }
+
+  override get [Symbol.toStringTag]() {
+    return 'ManualPromise';
+  }
+}
+
+export class LongStandingScope {
+  private _terminateError: Error | undefined;
+  private _closeError: Error | undefined;
+  private _terminatePromises = new Map<ManualPromise<Error>, string[]>();
+  private _isClosed = false;
+
+  reject(error: Error) {
+    this._isClosed = true;
+    this._terminateError = error;
+    for (const p of this._terminatePromises.keys())
+      p.resolve(error);
+  }
+
+  close(error: Error) {
+    this._isClosed = true;
+    this._closeError = error;
+    for (const [p, frames] of this._terminatePromises)
+      p.resolve(cloneError(error, frames));
+  }
+
+  isClosed() {
+    return this._isClosed;
+  }
+
+  static async raceMultiple<T>(scopes: LongStandingScope[], promise: Promise<T>): Promise<T> {
+    return Promise.race(scopes.map(s => s.race(promise)));
+  }
+
+  async race<T>(promise: Promise<T> | Promise<T>[]): Promise<T> {
+    return this._race(Array.isArray(promise) ? promise : [promise], false) as Promise<T>;
+  }
+
+  async safeRace<T>(promise: Promise<T>, defaultValue?: T): Promise<T> {
+    return this._race([promise], true, defaultValue);
+  }
+
+  private async _race(promises: Promise<any>[], safe: boolean, defaultValue?: any): Promise<any> {
+    const terminatePromise = new ManualPromise<Error>();
+    const frames = captureRawStack();
+    if (this._terminateError)
+      terminatePromise.resolve(this._terminateError);
+    if (this._closeError)
+      terminatePromise.resolve(cloneError(this._closeError, frames));
+    this._terminatePromises.set(terminatePromise, frames);
+    try {
+      return await Promise.race([
+        terminatePromise.then(e => safe ? defaultValue : Promise.reject(e)),
+        ...promises
+      ]);
+    } finally {
+      this._terminatePromises.delete(terminatePromise);
+    }
+  }
+}
+
+function cloneError(error: Error, frames: string[]) {
+  const clone = new Error();
+  clone.name = error.name;
+  clone.message = error.message;
+  clone.stack = [error.name + ':' + error.message, ...frames].join('\n');
+  return clone;
+}
+
+function captureRawStack(): string[] {
+  const stackTraceLimit = Error.stackTraceLimit;
+  Error.stackTraceLimit = 50;
+  const error = new Error();
+  const stack = error.stack || '';
+  Error.stackTraceLimit = stackTraceLimit;
+  return stack.split('\n');
+}

packages/playwright-mdd/src/program.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import dotenv from 'dotenv';
+import { program } from 'commander';
+
+import { runTasks } from './loop';
+import { Context } from './context';
+import { tools } from './tools';
+
+/* eslint-disable no-console */
+
+dotenv.config();
+
+const packageJSON = require('../package.json');
+
+program
+    .version('Version ' + packageJSON.version)
+    .name(packageJSON.name)
+    .action(async () => {
+      const context = await Context.create(tools);
+      const code = await runTasks(context, script);
+      console.log('Output code:');
+      console.log('```javascript');
+      console.log(code);
+      console.log('```');
+      await context.close();
+    });
+
+const script = [
+  'Navigate to https://debs-obrien.github.io/playwright-movies-app',
+  'Click search icon',
+  'Type "Twister" in the search field and hit Enter',
+  'Click on the link for the movie "Twisters"',
+];
+
+export { program };

packages/playwright-mdd/src/tools.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import snapshot from './tools/snapshot';
+import done from './tools/done';
+import navigate from './tools/navigate';
+
+import type { Tool } from './tools/tool.js';
+
+export const tools: Tool<any>[] = [
+  ...navigate,
+  ...snapshot,
+  ...done,
+];

packages/playwright-mdd/src/tools/done.ts

@@ -0,0 +1,39 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTool } from './tool';
+
+const doneTool = defineTool({
+  schema: {
+    name: 'done',
+    description: 'Call this tool to indicate that the task is complete',
+    inputSchema: z.object({}),
+  },
+
+  handle: async () => {
+    return {
+      code: [],
+      captureSnapshot: false,
+      waitForNetwork: false,
+    };
+  },
+});
+
+export default [
+  doneTool,
+];

packages/playwright-mdd/src/tools/navigate.ts

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTool } from './tool.js';
+
+const navigate = defineTool({
+  schema: {
+    name: 'browser_navigate',
+    description: 'Navigate to a URL',
+    inputSchema: z.object({
+      url: z.string().describe('The URL to navigate to'),
+    }),
+  },
+
+  handle: async (context, params) => {
+    const code = [
+      `await page.goto('${params.url}');`,
+    ];
+    await context.navigate(params.url);
+
+    return {
+      code,
+      captureSnapshot: true,
+      waitForNetwork: false,
+    };
+  },
+});
+
+const goBack = defineTool({
+  schema: {
+    name: 'browser_navigate_back',
+    description: 'Go back to the previous page',
+    inputSchema: z.object({}),
+  },
+
+  handle: async context => {
+    await context.page.goBack();
+    const code = [
+      `await page.goBack();`,
+    ];
+
+    return {
+      code,
+      captureSnapshot: true,
+      waitForNetwork: false,
+    };
+  },
+});
+
+const goForward = defineTool({
+  schema: {
+    name: 'browser_navigate_forward',
+    description: 'Go forward to the next page',
+    inputSchema: z.object({}),
+  },
+  handle: async context => {
+    await context.page.goForward();
+    const code = [
+      `await page.goForward();`,
+    ];
+    return {
+      code,
+      captureSnapshot: true,
+      waitForNetwork: false,
+    };
+  },
+});
+
+export default [
+  navigate,
+  goBack,
+  goForward,
+];

packages/playwright-mdd/src/tools/snapshot.ts

@@ -0,0 +1,194 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+
+import { defineTool } from './tool';
+import * as format from '../format';
+import { generateLocator } from './utils';
+
+const snapshot = defineTool({
+  schema: {
+    name: 'browser_snapshot',
+    description: 'Capture accessibility snapshot of the current page, this is better than screenshot',
+    inputSchema: z.object({}),
+  },
+
+  handle: async () => {
+    return {
+      code: [],
+      captureSnapshot: true,
+      waitForNetwork: false,
+    };
+  },
+});
+
+const elementSchema = z.object({
+  element: z.string().describe('Human-readable element description used to obtain permission to interact with the element'),
+  ref: z.string().describe('Exact target element reference from the page snapshot'),
+});
+
+const click = defineTool({
+  schema: {
+    name: 'browser_click',
+    description: 'Perform click on a web page',
+    inputSchema: elementSchema,
+  },
+
+  handle: async (context, params) => {
+    const locator = context.refLocator(params);
+
+    const code = [
+      `await page.${await generateLocator(locator)}.click();`
+    ];
+
+    return {
+      code,
+      action: () => locator.click(),
+      captureSnapshot: true,
+      waitForNetwork: true,
+    };
+  },
+});
+
+const drag = defineTool({
+  schema: {
+    name: 'browser_drag',
+    description: 'Perform drag and drop between two elements',
+    inputSchema: z.object({
+      startElement: z.string().describe('Human-readable source element description used to obtain the permission to interact with the element'),
+      startRef: z.string().describe('Exact source element reference from the page snapshot'),
+      endElement: z.string().describe('Human-readable target element description used to obtain the permission to interact with the element'),
+      endRef: z.string().describe('Exact target element reference from the page snapshot'),
+    }),
+  },
+
+  handle: async (context, params) => {
+    const startLocator = context.refLocator({ ref: params.startRef, element: params.startElement });
+    const endLocator = context.refLocator({ ref: params.endRef, element: params.endElement });
+
+    const code = [
+      `await page.${await generateLocator(startLocator)}.dragTo(page.${await generateLocator(endLocator)});`
+    ];
+
+    return {
+      code,
+      action: () => startLocator.dragTo(endLocator),
+      captureSnapshot: true,
+      waitForNetwork: true,
+    };
+  },
+});
+
+const hover = defineTool({
+  schema: {
+    name: 'browser_hover',
+    description: 'Hover over element on page',
+    inputSchema: elementSchema,
+  },
+
+  handle: async (context, params) => {
+    const locator = context.refLocator(params);
+
+    const code = [
+      `await page.${await generateLocator(locator)}.hover();`
+    ];
+
+    return {
+      code,
+      action: () => locator.hover(),
+      captureSnapshot: true,
+      waitForNetwork: true,
+    };
+  },
+});
+
+const typeSchema = elementSchema.extend({
+  text: z.string().describe('Text to type into the element'),
+  submit: z.boolean().optional().describe('Whether to submit entered text (press Enter after)'),
+  slowly: z.boolean().optional().describe('Whether to type one character at a time. Useful for triggering key handlers in the page. By default entire text is filled in at once.'),
+});
+
+const type = defineTool({
+  schema: {
+    name: 'browser_type',
+    description: 'Type text into editable element',
+    inputSchema: typeSchema,
+  },
+
+  handle: async (context, params) => {
+    const locator = context.refLocator(params);
+
+    const code: string[] = [];
+    const steps: (() => Promise<void>)[] = [];
+
+    if (params.slowly) {
+      code.push(`await page.${await generateLocator(locator)}.pressSequentially(${format.quote(params.text)});`);
+      steps.push(() => locator.pressSequentially(params.text));
+    } else {
+      code.push(`await page.${await generateLocator(locator)}.fill(${format.quote(params.text)});`);
+      steps.push(() => locator.fill(params.text));
+    }
+
+    if (params.submit) {
+      code.push(`await page.${await generateLocator(locator)}.press('Enter');`);
+      steps.push(() => locator.press('Enter'));
+    }
+
+    return {
+      code,
+      action: () => steps.reduce((acc, step) => acc.then(step), Promise.resolve()),
+      captureSnapshot: true,
+      waitForNetwork: true,
+    };
+  },
+});
+
+const selectOptionSchema = elementSchema.extend({
+  values: z.array(z.string()).describe('Array of values to select in the dropdown. This can be a single value or multiple values.'),
+});
+
+const selectOption = defineTool({
+  schema: {
+    name: 'browser_select_option',
+    description: 'Select an option in a dropdown',
+    inputSchema: selectOptionSchema,
+  },
+
+  handle: async (context, params) => {
+    const locator = context.refLocator(params);
+
+    const code = [
+      `await page.${await generateLocator(locator)}.selectOption(${format.formatObject(params.values)});`
+    ];
+
+    return {
+      code,
+      action: () => locator.selectOption(params.values).then(() => {}),
+      captureSnapshot: true,
+      waitForNetwork: true,
+    };
+  },
+});
+
+export default [
+  snapshot,
+  click,
+  drag,
+  hover,
+  type,
+  selectOption,
+];

packages/playwright-mdd/src/tools/tool.ts

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { z } from 'zod';
+import type * as playwright from 'playwright-core';
+import type { Context } from '../context';
+
+export type ToolSchema<Input extends InputType> = {
+  name: string;
+  description: string;
+  inputSchema: Input;
+};
+
+type InputType = z.Schema;
+
+export type FileUploadModalState = {
+  type: 'fileChooser';
+  description: string;
+  fileChooser: playwright.FileChooser;
+};
+
+export type DialogModalState = {
+  type: 'dialog';
+  description: string;
+  dialog: playwright.Dialog;
+};
+
+export type ModalState = FileUploadModalState | DialogModalState;
+
+export type ToolActionResult = string | undefined | void;
+
+export type ToolResult = {
+  code: string[];
+  action?: () => Promise<void>;
+  captureSnapshot: boolean;
+  waitForNetwork: boolean;
+};
+
+export type Tool<Input extends InputType = InputType> = {
+  schema: ToolSchema<Input>;
+  clearsModalState?: ModalState['type'];
+  handle: (context: Context, params: z.output<Input>) => Promise<ToolResult>;
+};
+
+export function defineTool<Input extends InputType>(tool: Tool<Input>): Tool<Input> {
+  return tool;
+}

packages/playwright-mdd/src/tools/utils.ts

@@ -0,0 +1,91 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type * as playwright from 'playwright';
+import type { Context } from '../context';
+
+export async function waitForCompletion<R>(context: Context, callback: () => Promise<R>): Promise<R> {
+  const requests = new Set<playwright.Request>();
+  let frameNavigated = false;
+  let waitCallback: () => void = () => {};
+  const waitBarrier = new Promise<void>(f => { waitCallback = f; });
+
+  const requestListener = (request: playwright.Request) => requests.add(request);
+  const requestFinishedListener = (request: playwright.Request) => {
+    requests.delete(request);
+    if (!requests.size)
+      waitCallback();
+  };
+
+  const frameNavigateListener = (frame: playwright.Frame) => {
+    if (frame.parentFrame())
+      return;
+    frameNavigated = true;
+    dispose();
+    clearTimeout(timeout);
+    void context.waitForLoadState('load').then(waitCallback);
+  };
+
+  const onTimeout = () => {
+    dispose();
+    waitCallback();
+  };
+
+  context.page.on('request', requestListener);
+  context.page.on('requestfinished', requestFinishedListener);
+  context.page.on('framenavigated', frameNavigateListener);
+  const timeout = setTimeout(onTimeout, 10000);
+
+  const dispose = () => {
+    context.page.off('request', requestListener);
+    context.page.off('requestfinished', requestFinishedListener);
+    context.page.off('framenavigated', frameNavigateListener);
+    clearTimeout(timeout);
+  };
+
+  try {
+    const result = await callback();
+    if (!requests.size && !frameNavigated)
+      waitCallback();
+    await waitBarrier;
+    await context.page.waitForTimeout(1000);
+    return result;
+  } finally {
+    dispose();
+  }
+}
+
+export function sanitizeForFilePath(s: string) {
+  const sanitize = (s: string) => s.replace(/[\x00-\x2C\x2E-\x2F\x3A-\x40\x5B-\x60\x7B-\x7F]+/g, '-');
+  const separator = s.lastIndexOf('.');
+  if (separator === -1)
+    return sanitize(s);
+  return sanitize(s.substring(0, separator)) + '.' + sanitize(s.substring(separator + 1));
+}
+
+export async function generateLocator(locator: playwright.Locator): Promise<string> {
+  try {
+    return await (locator as any)._generateLocatorString();
+  } catch (e) {
+    if (e instanceof Error && /locator._generateLocatorString: Timeout .* exceeded/.test(e.message))
+      throw new Error('Ref not found, likely because element was removed. Use browser_snapshot to see what elements are currently on the page.');
+    throw e;
+  }
+}
+
+export async function callOnPageNoTrace<T>(page: playwright.Page, callback: (page: playwright.Page) => Promise<T>): Promise<T> {
+  return await (page as any)._wrapApiCall(() => callback(page), { internal: true });
+}

utils/workspace.js

@@ -219,6 +219,11 @@ const workspace = new Workspace(ROOT_PATH, [
     path: path.join(ROOT_PATH, 'packages', 'playwright-ct-vue'),
     files: ['LICENSE'],
   }),
+  new PWPackage({
+    name: 'playwright-mdd',
+    path: path.join(ROOT_PATH, 'packages', 'playwright-mdd'),
+    files: ['LICENSE'],
+  }),
 ]);
 
 if (require.main === module) {