= new Map();
- private _attachErrorContext: boolean;
+ private _errorContext: ErrorContextOption;
- constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption, attachErrorContext: boolean) {
+ constructor(playwright: PlaywrightImpl, artifactsDir: string, screenshot: ScreenshotOption, errorContext: ErrorContextOption) {
this._playwright = playwright;
this._artifactsDir = artifactsDir;
- this._attachErrorContext = attachErrorContext;
+ this._errorContext = errorContext;
const screenshotOptions = typeof screenshot === 'string' ? undefined : screenshot;
this._startedCollectingArtifacts = Symbol('startedCollectingArtifacts');
@@ -671,7 +673,7 @@ class ArtifactsRecorder {
}
private async _takePageSnapshot(context: BrowserContext) {
- if (process.env.PLAYWRIGHT_NO_COPY_PROMPT)
+ if (!this._errorContext)
return;
if (this._testInfo.errors.length === 0)
return;
@@ -719,10 +721,8 @@ class ArtifactsRecorder {
if (context)
await this._takePageSnapshot(context);
- if (this._attachErrorContext)
- await attachErrorContext(this._testInfo, this._pageSnapshot);
- else
- await attachErrorPrompts(this._testInfo, this._sourceCache, this._pageSnapshot);
+ if (this._errorContext)
+ await attachErrorContext(this._testInfo, this._errorContext.format, this._sourceCache, this._pageSnapshot);
}
private async _startTraceChunkOnContextCreation(tracing: Tracing) {
diff --git a/packages/playwright/src/isomorphic/testServerInterface.ts b/packages/playwright/src/isomorphic/testServerInterface.ts
index e327b8b11a..5849168542 100644
--- a/packages/playwright/src/isomorphic/testServerInterface.ts
+++ b/packages/playwright/src/isomorphic/testServerInterface.ts
@@ -102,7 +102,7 @@ export interface TestServerInterface {
projects?: string[];
reuseContext?: boolean;
connectWsEndpoint?: string;
- attachErrorContext?: boolean;
+ errorContext?: { format: 'json' | 'markdown' };
}): Promise<{
status: reporterTypes.FullResult['status'];
}>;
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index 75807c1b67..d7f6a2a825 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -337,9 +337,9 @@ export function formatFailure(screen: Screen, config: FullConfig, test: TestCase
// }
for (let i = 0; i < result.attachments.length; ++i) {
const attachment = result.attachments[i];
- if (attachment.name.startsWith('_prompt') && attachment.path) {
+ if (attachment.name.startsWith('_error-context') && attachment.path) {
resultLines.push('');
- resultLines.push(screen.colors.dim(` Error Prompt: ${relativeFilePath(screen, config, attachment.path)}`));
+ resultLines.push(screen.colors.dim(` Error Context: ${relativeFilePath(screen, config, attachment.path)}`));
continue;
}
if (attachment.name.startsWith('_'))
diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.ts
index 4ff1dd55ce..a6f8b0b7ca 100644
--- a/packages/playwright/src/runner/testServer.ts
+++ b/packages/playwright/src/runner/testServer.ts
@@ -314,7 +314,7 @@ export class TestServerDispatcher implements TestServerInterface {
...(params.headed !== undefined ? { headless: !params.headed } : {}),
_optionContextReuseMode: params.reuseContext ? 'when-possible' : undefined,
_optionConnectOptions: params.connectWsEndpoint ? { wsEndpoint: params.connectWsEndpoint } : undefined,
- _optionAttachErrorContext: params.attachErrorContext,
+ _optionErrorContext: params.errorContext,
},
...(params.updateSnapshots ? { updateSnapshots: params.updateSnapshots } : {}),
...(params.updateSourceMethod ? { updateSourceMethod: params.updateSourceMethod } : {}),
diff --git a/packages/trace-viewer/src/ui/errorsTab.tsx b/packages/trace-viewer/src/ui/errorsTab.tsx
index 7ade3e2dbf..a5ea191e1f 100644
--- a/packages/trace-viewer/src/ui/errorsTab.tsx
+++ b/packages/trace-viewer/src/ui/errorsTab.tsx
@@ -26,6 +26,7 @@ import { ToolbarButton } from '@web/components/toolbarButton';
import { useIsLLMAvailable, useLLMChat } from './llm';
import { useAsyncMemo } from '@web/uiUtils';
import { attachmentURL } from './attachmentsTab';
+import { fixTestInstructions } from '@web/prompts';
const CopyPromptButton: React.FC<{ prompt: string }> = ({ prompt }) => {
return (
@@ -67,10 +68,10 @@ function Error({ message, error, errorId, sdkLanguage, revealInSource }: { messa
}
const prompt = useAsyncMemo(async () => {
- if (!error.prompt)
+ if (!error.context)
return;
- const response = await fetch(attachmentURL(error.prompt));
- return await response.text();
+ const response = await fetch(attachmentURL(error.context));
+ return fixTestInstructions + await response.text();
}, [error], undefined);
return
diff --git a/packages/trace-viewer/src/ui/modelUtil.ts b/packages/trace-viewer/src/ui/modelUtil.ts
index c75c722419..ea8a15676d 100644
--- a/packages/trace-viewer/src/ui/modelUtil.ts
+++ b/packages/trace-viewer/src/ui/modelUtil.ts
@@ -56,7 +56,7 @@ export type ErrorDescription = {
action?: ActionTraceEventInContext;
stack?: StackFrame[];
message: string;
- prompt?: trace.AfterActionTraceEventAttachment & { traceUrl: string };
+ context?: trace.AfterActionTraceEventAttachment & { traceUrl: string };
};
export type Attachment = trace.AfterActionTraceEventAttachment & { traceUrl: string };
@@ -141,7 +141,7 @@ export class MultiTraceModel {
return this.errors.filter(e => !!e.message).map((error, i) => ({
stack: error.stack,
message: error.message,
- prompt: this.attachments.find(a => a.name === `_prompt-${i}`),
+ context: this.attachments.find(a => a.name === `_error-context-${i}`),
}));
}
}
diff --git a/packages/web/src/prompts.ts b/packages/web/src/prompts.ts
new file mode 100644
index 0000000000..ef0b5bf78b
--- /dev/null
+++ b/packages/web/src/prompts.ts
@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export const fixTestInstructions = `
+# Instructions
+
+- Following Playwright test failed.
+- Explain why, be concise, respect Playwright best practices.
+- Provide a snippet of code with the fix, if possible.
+`.trimStart();
diff --git a/tests/playwright-test/playwright.connect.spec.ts b/tests/playwright-test/playwright.connect.spec.ts
index e9d29dfe0e..e1644bb8ea 100644
--- a/tests/playwright-test/playwright.connect.spec.ts
+++ b/tests/playwright-test/playwright.connect.spec.ts
@@ -167,7 +167,7 @@ test('should print debug log when failed to connect', async ({ runInlineTest })
expect(result.exitCode).toBe(1);
expect(result.failed).toBe(1);
expect(result.output).toContain('b-debug-log-string');
- expect(result.results[0].attachments).toEqual([expect.objectContaining({ name: '_prompt-0' })]);
+ expect(result.results[0].attachments).toEqual([expect.objectContaining({ name: '_error-context-0' })]);
});
test('should record trace', async ({ runInlineTest }) => {
@@ -223,7 +223,7 @@ test('should record trace', async ({ runInlineTest }) => {
'After Hooks',
'fixture: page',
'fixture: context',
- '_attach "_prompt-0"',
+ '_attach "_error-context-0"',
'Worker Cleanup',
'fixture: browser',
]);
diff --git a/tests/playwright-test/playwright.spec.ts b/tests/playwright-test/playwright.spec.ts
index f75aba94a8..875133e7d4 100644
--- a/tests/playwright-test/playwright.spec.ts
+++ b/tests/playwright-test/playwright.spec.ts
@@ -510,13 +510,13 @@ test('should work with video: on-first-retry', async ({ runInlineTest }) => {
expect(fs.existsSync(dirPass)).toBeFalsy();
const dirFail = test.info().outputPath('test-results', 'a-fail-chromium');
- expect(fs.readdirSync(dirFail)).toEqual(['prompt.md']);
+ expect(fs.readdirSync(dirFail)).toEqual(['error-context.md']);
const dirRetry = test.info().outputPath('test-results', 'a-fail-chromium-retry1');
const videoFailRetry = fs.readdirSync(dirRetry).find(file => file.endsWith('webm'));
expect(videoFailRetry).toBeTruthy();
- const errorPrompt = expect.objectContaining({ name: '_prompt-0' });
+ const errorPrompt = expect.objectContaining({ name: '_error-context-0' });
expect(result.report.suites[0].specs[1].tests[0].results[0].attachments).toEqual([errorPrompt]);
expect(result.report.suites[0].specs[1].tests[0].results[1].attachments).toEqual([{
name: 'video',
diff --git a/tests/playwright-test/reporter-json.spec.ts b/tests/playwright-test/reporter-json.spec.ts
index 377e20aa68..b2cdbf44d8 100644
--- a/tests/playwright-test/reporter-json.spec.ts
+++ b/tests/playwright-test/reporter-json.spec.ts
@@ -359,7 +359,7 @@ test('should report parallelIndex', async ({ runInlineTest }, testInfo) => {
test('attaches error context', async ({ runInlineTest }) => {
const result = await runInlineTest({
'playwright.config.ts': `
- export default { use: { _optionAttachErrorContext: true } };
+ export default { use: { _optionErrorContext: { format: 'json' } } };
`,
'a.test.js': `
const { test, expect } = require('@playwright/test');
diff --git a/tests/playwright-test/reporter-line.spec.ts b/tests/playwright-test/reporter-line.spec.ts
index 777e771210..b3a84ac1dc 100644
--- a/tests/playwright-test/reporter-line.spec.ts
+++ b/tests/playwright-test/reporter-line.spec.ts
@@ -190,7 +190,7 @@ for (const useIntermediateMergeReport of [false, true] as const) {
expect(result.exitCode).toBe(1);
});
- test('should show error prompt with relative path', async ({ runInlineTest, useIntermediateMergeReport }) => {
+ test('should show error context with relative path', async ({ runInlineTest, useIntermediateMergeReport }) => {
const result = await runInlineTest({
'a.test.js': `
const { test, expect } = require('@playwright/test');
@@ -201,9 +201,9 @@ for (const useIntermediateMergeReport of [false, true] as const) {
}, { reporter: 'line' });
const text = result.output;
if (useIntermediateMergeReport)
- expect(text).toContain(`Error Prompt: ${path.join('blob-report', 'resources')}`);
+ expect(text).toContain(`Error Context: ${path.join('blob-report', 'resources')}`);
else
- expect(text).toContain(`Error Prompt: ${path.join('test-results', 'a-one', 'prompt.md')}`);
+ expect(text).toContain(`Error Context: ${path.join('test-results', 'a-one', 'error-context.md')}`);
expect(result.exitCode).toBe(1);
});
});