feat(prompt): link to error prompt in terminal (#35341)

packages/html-reporter/src/testErrorView.tsx

@@ -19,11 +19,12 @@ import * as React from 'react';
 import './testErrorView.css';
 import type { ImageDiff } from '@web/shared/imageDiffView';
 import { ImageDiffView } from '@web/shared/imageDiffView';
+import { TestAttachment } from './types';
 
 export const TestErrorView: React.FC<{
   error: string;
   testId?: string;
-  prompt?: string;
+  prompt?: TestAttachment;
 }> = ({ error, testId, prompt }) => {
   return (
     <CodeSnippet code={error} testId={testId}>
@@ -46,13 +47,14 @@ export const CodeSnippet = ({ code, children, testId }: React.PropsWithChildren<
   );
 };
 
-const PromptButton: React.FC<{ prompt: string }> = ({ prompt }) => {
+const PromptButton: React.FC<{ prompt: TestAttachment }> = ({ prompt }) => {
   const [copied, setCopied] = React.useState(false);
   return <button
     className='button'
     style={{ minWidth: 100 }}
     onClick={async () => {
-      await navigator.clipboard.writeText(prompt);
+      const text = prompt.body ? prompt.body : await fetch(prompt.path!).then(r => r.text());
+      await navigator.clipboard.writeText(text);
       setCopied(true);
       setTimeout(() => {
         setCopied(false);

packages/html-reporter/src/testResultView.tsx

@@ -165,7 +165,7 @@ function classifyErrors(testErrors: string[], diffs: ImageDiff[], attachments: T
       }
     }
 
-    const prompt = attachments.find(a => a.name === `_prompt-${i}`)?.body;
+    const prompt = attachments.find(a => a.name === `_prompt-${i}`);
     return { type: 'regular', error, prompt };
   });
 }

packages/playwright/src/prompt.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import * as fs from 'fs';
+import * as fs from 'fs/promises';
 import * as path from 'path';
 
 import { parseErrorStack } from 'playwright-core/lib/utils';
@@ -38,17 +38,21 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st
     }
   }
 
-  for (const [index, error] of testInfo.errors.entries()) {
+  const errors = [...testInfo.errors.entries()].filter(([, error]) => {
     if (!error.message)
-      return;
+      return false;
-    if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))
-      continue;
 
     // Skip errors that are just a single line - they are likely to already be the error message.
     if (!error.message.includes('\n') && !meaningfulSingleLineErrors.has(error.message))
-      continue;
+      return false;
 
+    return true;
+  });
+
+  for (const [index, error] of errors) {
     const metadata = testInfo.config.metadata as MetadataWithCommitInfo;
+    if (testInfo.attachments.find(a => a.name === `_prompt-${index}`))
+      continue;
 
     const promptParts = [
       `# Instructions`,
@@ -119,10 +123,13 @@ export async function attachErrorPrompts(testInfo: TestInfo, sourceCache: Map<st
       );
     }
 
+    const promptPath = testInfo.outputPath(errors.length === 1 ? `prompt.md` : `prompt-${index}.md`);
+    await fs.writeFile(promptPath, promptParts.join('\n'), 'utf8');
+
     (testInfo as TestInfoImpl)._attach({
       name: `_prompt-${index}`,
       contentType: 'text/markdown',
-      body: Buffer.from(promptParts.join('\n')),
+      path: promptPath,
     }, undefined);
   }
 }
@@ -144,7 +151,7 @@ async function loadSource(file: string, sourceCache: Map<string, string>) {
   let source = sourceCache.get(file);
   if (!source) {
     // A mild race is Ok here.
-    source = await fs.promises.readFile(file, 'utf8');
+    source = await fs.readFile(file, 'utf8');
     sourceCache.set(file, source);
   }
   return source;

packages/playwright/src/reporters/base.ts

@@ -350,15 +350,18 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCase
     const errors = formatResultFailure(screen, test, result, '    ');
     if (!errors.length)
       continue;
-    const retryLines = [];
     if (result.retry) {
-      retryLines.push('');
+      resultLines.push('');
-      retryLines.push(screen.colors.gray(separator(screen, `    Retry #${result.retry}`)));
+      resultLines.push(screen.colors.gray(separator(screen, `    Retry #${result.retry}`)));
     }
-    resultLines.push(...retryLines);
     resultLines.push(...errors.map(error => '\n' + error.message));
     for (let i = 0; i < result.attachments.length; ++i) {
       const attachment = result.attachments[i];
+      if (attachment.name.startsWith('_prompt') && attachment.path) {
+        resultLines.push('');
+        resultLines.push(screen.colors.dim(`    Error Prompt: ${relativeFilePath(screen, config, attachment.path)}`));
+        continue;
+      }
       if (attachment.name.startsWith('_'))
         continue;
       const hasPrintableContent = attachment.contentType.startsWith('text/');

tests/playwright-test/playwright.artifacts.spec.ts

@@ -130,7 +130,7 @@ test('should work with screenshot: on', async ({ runInlineTest }, testInfo) => {
     'playwright.config.ts': `
       module.exports = { use: { screenshot: 'on' } };
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(5);
@@ -168,7 +168,7 @@ test('should work with screenshot: only-on-failure', async ({ runInlineTest }, t
     'playwright.config.ts': `
       module.exports = { use: { screenshot: 'only-on-failure' } };
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(5);
@@ -204,7 +204,7 @@ test('should work with screenshot: on-first-failure', async ({ runInlineTest },
         use: { screenshot: 'on-first-failure' }
       };
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(0);
@@ -230,7 +230,7 @@ test('should work with screenshot: only-on-failure & fullPage', async ({ runInli
     'playwright.config.ts': `
       module.exports = { use: { screenshot: { mode: 'only-on-failure', fullPage: true } } };
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(0);
   expect(result.failed).toBe(1);
@@ -263,7 +263,7 @@ test('should capture a single screenshot on failure when afterAll fails', async
         await page.setContent('this is test');
       });
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(0);
@@ -282,7 +282,7 @@ test('should work with trace: on', async ({ runInlineTest }, testInfo) => {
     'playwright.config.ts': `
       module.exports = { use: { trace: 'on' } };
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(5);
@@ -318,7 +318,7 @@ test('should work with trace: retain-on-failure', async ({ runInlineTest }, test
     'playwright.config.ts': `
       module.exports = { use: { trace: 'retain-on-failure' } };
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(5);
@@ -344,7 +344,7 @@ test('should work with trace: on-first-retry', async ({ runInlineTest }, testInf
     'playwright.config.ts': `
       module.exports = { use: { trace: 'on-first-retry' } };
     `,
-  }, { workers: 1, retries: 1 });
+  }, { workers: 1, retries: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(5);
@@ -370,7 +370,7 @@ test('should work with trace: on-all-retries', async ({ runInlineTest }, testInf
     'playwright.config.ts': `
       module.exports = { use: { trace: 'on-all-retries' } };
     `,
-  }, { workers: 1, retries: 2 });
+  }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(5);
@@ -406,7 +406,7 @@ test('should work with trace: retain-on-first-failure', async ({ runInlineTest }
     'playwright.config.ts': `
       module.exports = { use: { trace: 'retain-on-first-failure' } };
     `,
-  }, { workers: 1, retries: 2 });
+  }, { workers: 1, retries: 2 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.passed).toBe(5);
@@ -442,7 +442,7 @@ test('should take screenshot when page is closed in afterEach', async ({ runInli
         expect(1).toBe(2);
       });
     `,
-  }, { workers: 1 });
+  }, { workers: 1 }, { PLAYWRIGHT_NO_COPY_PROMPT: 'true' });
 
   expect(result.exitCode).toBe(1);
   expect(result.failed).toBe(1);

tests/playwright-test/playwright.spec.ts

@@ -510,7 +510,7 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => {
   expect(fs.existsSync(dirPass)).toBeFalsy();
 
   const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');
-  expect(fs.existsSync(dirFail)).toBeFalsy();
+  expect(fs.readdirSync(dirFail)).toEqual(['prompt.md']);
 
   const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1');
   const videoFailRetry = fs.readdirSync(dirRetry).find(file => file.endsWith('webm'));

tests/playwright-test/reporter-line.spec.ts

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import path from 'path';
 import { test, expect } from './playwright-test-fixtures';
 
 for (const useIntermediateMergeReport of [false, true] as const) {
@@ -188,5 +189,22 @@ for (const useIntermediateMergeReport of [false, true] as const) {
       expect(text).toContain('1) a.test.ts:3:15 › passes ──');
       expect(result.exitCode).toBe(1);
     });
+
+    test('should show error prompt with relative path', async ({ runInlineTest, useIntermediateMergeReport }) => {
+      const result = await runInlineTest({
+        'a.test.js': `
+          const { test, expect } = require('@playwright/test');
+          test('one', async ({}) => {
+            expect(1).toBe(0);
+          });
+        `,
+      }, { reporter: 'line' });
+      const text = result.output;
+      if (useIntermediateMergeReport)
+        expect(text).toContain(`Error Prompt: ${path.join('blob-report', 'resources')}`);
+      else
+        expect(text).toContain(`Error Prompt: ${path.join('test-results', 'a-one', 'prompt.md')}`);
+      expect(result.exitCode).toBe(1);
+    });
   });
 }