diff --git a/docs/src/test-reporter-api/class-testcase.md b/docs/src/test-reporter-api/class-testcase.md index 82ab6ca521..1aa5379ac4 100644 --- a/docs/src/test-reporter-api/class-testcase.md +++ b/docs/src/test-reporter-api/class-testcase.md @@ -12,8 +12,7 @@ The list of annotations applicable to the current test. Includes: * annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`]; -* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`]; -* annotations appended to [`property: TestInfo.annotations`] during the test execution. +* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] prior to test execution. Annotations are available during test execution through [`property: TestInfo.annotations`]. diff --git a/docs/src/test-reporter-api/class-testresult.md b/docs/src/test-reporter-api/class-testresult.md index 7674ac333d..0e60558632 100644 --- a/docs/src/test-reporter-api/class-testresult.md +++ b/docs/src/test-reporter-api/class-testresult.md @@ -14,6 +14,20 @@ A result of a single [TestCase] run. The list of files or buffers attached during the test execution through [`property: TestInfo.attachments`]. +## property: TestResult.annotations +* since: v1.52 +- type: <[Array]<[Object]>> + - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`. + - `description` ?<[string]> Optional description. + +The list of annotations appended during test execution. Includes: +* annotations implicitly added by methods [`method: Test.skip`], [`method: Test.fixme`] and [`method: Test.fail`] during test execution; +* annotations appended to [`property: TestInfo.annotations`]. + +Annotations are available during test execution through [`property: TestInfo.annotations`]. + +Learn more about [test annotations](../test-annotations.md). + ## property: TestResult.duration * since: v1.10 - type: <[float]> diff --git a/packages/html-reporter/src/tabbedPane.tsx b/packages/html-reporter/src/tabbedPane.tsx index 02d0c6f3b1..af15b6b4fa 100644 --- a/packages/html-reporter/src/tabbedPane.tsx +++ b/packages/html-reporter/src/tabbedPane.tsx @@ -30,14 +30,18 @@ export const TabbedPane: React.FunctionComponent<{ selectedTab: string, setSelectedTab: (tab: string) => void }> = ({ tabs, selectedTab, setSelectedTab }) => { + const idPrefix = React.useId(); return
-
{ +
{ tabs.map(tab => (
setSelectedTab(tab.id)} - key={tab.id}> + id={`${idPrefix}-${tab.id}`} + key={tab.id} + role='tab' + aria-selected={selectedTab === tab.id}>
{tab.title}
)) @@ -46,7 +50,7 @@ export const TabbedPane: React.FunctionComponent<{ { tabs.map(tab => { if (selectedTab === tab.id) - return
{tab.render()}
; + return
{tab.render()}
; }) }
diff --git a/packages/html-reporter/src/testCaseView.spec.tsx b/packages/html-reporter/src/testCaseView.spec.tsx index 8aa3fafc8f..3f6c7f2d05 100644 --- a/packages/html-reporter/src/testCaseView.spec.tsx +++ b/packages/html-reporter/src/testCaseView.spec.tsx @@ -42,6 +42,7 @@ const result: TestResult = { }], attachments: [], }], + annotations: [], attachments: [], status: 'passed', }; @@ -151,6 +152,7 @@ const resultWithAttachment: TestResult = { name: 'attachment with inline link https://github.com/microsoft/playwright/issues/31284', contentType: 'text/plain' }], + annotations: [], status: 'passed', }; @@ -238,13 +240,15 @@ test('total duration is selected run duration', async ({ mount, page }) => { const component = await mount(); await expect(component).toMatchAriaSnapshot(` - text: "My test test.spec.ts:42 200ms" - - text: "Run 50ms Retry #1 150ms" + - tablist: + - tab "Run 50ms" + - 'tab "Retry #1 150ms"' `); - await page.locator('.tabbed-pane-tab-label', { hasText: 'Run50ms' }).click(); + await page.getByRole('tab', { name: 'Run' }).click(); await expect(component).toMatchAriaSnapshot(` - text: "My test test.spec.ts:42 200ms" `); - await page.locator('.tabbed-pane-tab-label', { hasText: 'Retry #1150ms' }).click(); + await page.getByRole('tab', { name: 'Retry' }).click(); await expect(component).toMatchAriaSnapshot(` - text: "My test test.spec.ts:42 200ms" `); diff --git a/packages/html-reporter/src/testCaseView.tsx b/packages/html-reporter/src/testCaseView.tsx index ca5ff6d2e2..56ad193c8c 100644 --- a/packages/html-reporter/src/testCaseView.tsx +++ b/packages/html-reporter/src/testCaseView.tsx @@ -14,7 +14,7 @@ limitations under the License. */ -import type { TestCase, TestCaseAnnotation, TestCaseSummary } from './types'; +import type { TestCase, TestAnnotation, TestCaseSummary } from './types'; import * as React from 'react'; import { TabbedPane } from './tabbedPane'; import { AutoChip } from './chip'; @@ -46,8 +46,13 @@ export const TestCaseView: React.FC<{ }, [test]); const visibleAnnotations = React.useMemo(() => { - return test?.annotations?.filter(annotation => !annotation.type.startsWith('_')) || []; - }, [test?.annotations]); + if (!test) + return []; + const annotations = [...test.annotations]; + if (test.results[selectedResultIndex]) + annotations.push(...test.results[selectedResultIndex].annotations); + return annotations.filter(annotation => !annotation.type.startsWith('_')); + }, [test, selectedResultIndex]); return
{test &&
@@ -71,7 +76,7 @@ export const TestCaseView: React.FC<{ {test && !!test.projectName && } {labels && }
} - {!!visibleAnnotations.length && + {!!visibleAnnotations.length && {visibleAnnotations.map((annotation, index) => )} } {test && ; }; -function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestCaseAnnotation }) { +function TestCaseAnnotationView({ annotation: { type, description } }: { annotation: TestAnnotation }) { return (
{type} diff --git a/packages/html-reporter/src/types.d.ts b/packages/html-reporter/src/types.d.ts index 5db8199d34..17c5a3b373 100644 --- a/packages/html-reporter/src/types.d.ts +++ b/packages/html-reporter/src/types.d.ts @@ -59,7 +59,7 @@ export type TestFileSummary = { stats: Stats; }; -export type TestCaseAnnotation = { type: string, description?: string }; +export type TestAnnotation = { type: string, description?: string }; export type TestCaseSummary = { testId: string, @@ -67,7 +67,7 @@ export type TestCaseSummary = { path: string[]; projectName: string; location: Location; - annotations: TestCaseAnnotation[]; + annotations: TestAnnotation[]; tags: string[]; outcome: 'skipped' | 'expected' | 'unexpected' | 'flaky'; duration: number; @@ -98,6 +98,7 @@ export type TestResult = { errors: string[]; attachments: TestAttachment[]; status: 'passed' | 'failed' | 'timedOut' | 'skipped' | 'interrupted'; + annotations: TestAnnotation[]; }; export type TestStep = { diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.ts index ab676548b5..20c46e3157 100644 --- a/packages/playwright/src/common/suiteUtils.ts +++ b/packages/playwright/src/common/suiteUtils.ts @@ -64,10 +64,9 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit // Inherit properties from parent suites. let inheritedRetries: number | undefined; let inheritedTimeout: number | undefined; - test.annotations = []; for (let parentSuite: Suite | undefined = suite; parentSuite; parentSuite = parentSuite.parent) { if (parentSuite._staticAnnotations.length) - test.annotations = [...parentSuite._staticAnnotations, ...test.annotations]; + test.annotations.unshift(...parentSuite._staticAnnotations); if (inheritedRetries === undefined && parentSuite._retries !== undefined) inheritedRetries = parentSuite._retries; if (inheritedTimeout === undefined && parentSuite._timeout !== undefined) @@ -75,7 +74,6 @@ export function bindFileSuiteToProject(project: FullProjectInternal, suite: Suit } test.retries = inheritedRetries ?? project.project.retries; test.timeout = inheritedTimeout ?? project.project.timeout; - test.annotations.push(...test._staticAnnotations); // Skip annotations imply skipped expectedStatus. if (test.annotations.some(a => a.type === 'skip' || a.type === 'fixme')) diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index 7b54f0c291..2b6434a2ef 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -262,8 +262,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { _poolDigest = ''; _workerHash = ''; _projectId = ''; - // Annotations known statically before running the test, e.g. `test.skip()` or `test(title, { annotation }, body)`. - _staticAnnotations: Annotation[] = []; // Explicitly declared tags that are not a part of the title. _tags: string[] = []; @@ -306,7 +304,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { requireFile: this._requireFile, poolDigest: this._poolDigest, workerHash: this._workerHash, - staticAnnotations: this._staticAnnotations.slice(), annotations: this.annotations.slice(), tags: this._tags.slice(), projectId: this._projectId, @@ -323,7 +320,6 @@ export class TestCase extends Base implements reporterTypes.TestCase { test._requireFile = data.requireFile; test._poolDigest = data.poolDigest; test._workerHash = data.workerHash; - test._staticAnnotations = data.staticAnnotations; test.annotations = data.annotations; test._tags = data.tags; test._projectId = data.projectId; @@ -351,6 +347,7 @@ export class TestCase extends Base implements reporterTypes.TestCase { status: 'skipped', steps: [], errors: [], + annotations: [], }; this.results.push(result); return result; diff --git a/packages/playwright/src/common/testType.ts b/packages/playwright/src/common/testType.ts index 565ed58321..1fa0f4ce6c 100644 --- a/packages/playwright/src/common/testType.ts +++ b/packages/playwright/src/common/testType.ts @@ -107,16 +107,16 @@ export class TestTypeImpl { const validatedDetails = validateTestDetails(details); const test = new TestCase(title, body, this, location); test._requireFile = suite._requireFile; - test._staticAnnotations.push(...validatedDetails.annotations); + test.annotations.push(...validatedDetails.annotations); test._tags.push(...validatedDetails.tags); suite._addTest(test); if (type === 'only' || type === 'fail.only') test._only = true; if (type === 'skip' || type === 'fixme' || type === 'fail') - test._staticAnnotations.push({ type }); + test.annotations.push({ type }); else if (type === 'fail.only') - test._staticAnnotations.push({ type: 'fail' }); + test.annotations.push({ type: 'fail' }); } private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) { diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index 7143fcf822..4504c160fd 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -68,14 +68,14 @@ export type JsonTestCase = { retries: number; tags?: string[]; repeatEachIndex: number; - annotations?: { type: string, description?: string }[]; + annotations?: Annotation[]; }; export type JsonTestEnd = { testId: string; expectedStatus: reporterTypes.TestStatus; timeout: number; - annotations: { type: string, description?: string }[]; + annotations: Annotation[]; }; export type JsonTestResultStart = { @@ -94,6 +94,7 @@ export type JsonTestResultEnd = { status: reporterTypes.TestStatus; errors: reporterTypes.TestError[]; attachments: JsonAttachment[]; + annotations?: Annotation[]; }; export type JsonTestStepStart = { @@ -241,6 +242,8 @@ export class TeleReporterReceiver { result.errors = payload.errors; result.error = result.errors?.[0]; result.attachments = this._parseAttachments(payload.attachments); + if (payload.annotations) + result.annotations = payload.annotations; this._reporter.onTestEnd?.(test, result); // Free up the memory as won't see these step ids. result._stepMap = new Map(); @@ -562,6 +565,7 @@ export class TeleTestResult implements reporterTypes.TestResult { stdout: reporterTypes.TestResult['stdout'] = []; stderr: reporterTypes.TestResult['stderr'] = []; attachments: reporterTypes.TestResult['attachments'] = []; + annotations: reporterTypes.TestResult['annotations'] = []; status: reporterTypes.TestStatus = 'skipped'; steps: TeleTestStep[] = []; errors: reporterTypes.TestResult['errors'] = []; diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 30534d7e8f..c8326ece31 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -291,10 +291,13 @@ export class TerminalReporter implements ReporterV2 { } private _printWarnings() { - const warningTests = this.suite.allTests().filter(test => test.annotations.some(a => a.type === 'warning')); + const warningTests = this.suite.allTests().filter(test => { + const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)]; + return annotations.some(a => a.type === 'warning'); + }); const encounteredWarnings = new Map>(); for (const test of warningTests) { - for (const annotation of test.annotations) { + for (const annotation of [...test.annotations, ...test.results.flatMap(r => r.annotations)]) { if (annotation.type !== 'warning' || annotation.description === undefined) continue; let tests = encounteredWarnings.get(annotation.description); diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index ec83a58046..dfe51127cc 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -31,7 +31,7 @@ import { resolveReporterOutputPath, stripAnsiEscapes } from '../util'; import type { ReporterV2 } from './reporterV2'; import type { Metadata } from '../../types/test'; import type * as api from '../../types/testReporter'; -import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep } from '@html-reporter/types'; +import type { HTMLReport, Stats, TestAttachment, TestCase, TestCaseSummary, TestFile, TestFileSummary, TestResult, TestStep, TestAnnotation } from '@html-reporter/types'; import type { ZipFile } from 'playwright-core/lib/zipBundle'; import type { TransformCallback } from 'stream'; @@ -404,8 +404,7 @@ class HtmlBuilder { projectName, location, duration, - // Annotations can be pushed directly, with a wrong type. - annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })), + annotations: this._serializeAnnotations(test.annotations), tags: test.tags, outcome: test.outcome(), path, @@ -418,8 +417,7 @@ class HtmlBuilder { projectName, location, duration, - // Annotations can be pushed directly, with a wrong type. - annotations: test.annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })), + annotations: this._serializeAnnotations([...test.annotations, ...results.flatMap(r => r.annotations)]), tags: test.tags, outcome: test.outcome(), path, @@ -503,6 +501,11 @@ class HtmlBuilder { }).filter(Boolean) as TestAttachment[]; } + private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] { + // Annotations can be pushed directly, with a wrong type. + return annotations.map(a => ({ type: a.type, description: a.description ? String(a.description) : a.description })); + } + private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult { return { duration: result.duration, @@ -511,6 +514,7 @@ class HtmlBuilder { steps: dedupeSteps(result.steps).map(s => this._createTestStep(s, result)), errors: formatResultFailure(internalScreen, test, result, '').map(error => error.message), status: result.status, + annotations: this._serializeAnnotations(result.annotations), attachments: this._serializeAttachments([ ...result.attachments, ...result.stdout.map(m => stdioAttachment(m, 'stdout')), diff --git a/packages/playwright/src/reporters/json.ts b/packages/playwright/src/reporters/json.ts index ac33d25d9a..6cdd994531 100644 --- a/packages/playwright/src/reporters/json.ts +++ b/packages/playwright/src/reporters/json.ts @@ -213,6 +213,7 @@ class JSONReporter implements ReporterV2 { retry: result.retry, steps: steps.length ? steps.map(s => this._serializeTestStep(s)) : undefined, startTime: result.startTime.toISOString(), + annotations: result.annotations, attachments: result.attachments.map(a => ({ name: a.name, contentType: a.contentType, diff --git a/packages/playwright/src/reporters/junit.ts b/packages/playwright/src/reporters/junit.ts index a89ebd7df7..1ae8d4f133 100644 --- a/packages/playwright/src/reporters/junit.ts +++ b/packages/playwright/src/reporters/junit.ts @@ -166,7 +166,8 @@ class JUnitReporter implements ReporterV2 { children: [] as XMLEntry[] }; - for (const annotation of test.annotations) { + const annotations = [...test.annotations, ...test.results.flatMap(r => r.annotations)]; + for (const annotation of annotations) { const property: XMLEntry = { name: 'property', attributes: { diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 7b1571f12b..45532683cf 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -237,6 +237,7 @@ export class TeleReporterEmitter implements ReporterV2 { status: result.status, errors: result.errors, attachments: this._serializeAttachments(result.attachments), + annotations: result.annotations?.length ? result.annotations : undefined, }; } diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts index 462f7551c4..ec654de091 100644 --- a/packages/playwright/src/runner/dispatcher.ts +++ b/packages/playwright/src/runner/dispatcher.ts @@ -325,8 +325,8 @@ class JobDispatcher { result.errors = params.errors; result.error = result.errors[0]; result.status = params.status; + result.annotations = params.annotations; test.expectedStatus = params.expectedStatus; - test.annotations = params.annotations; test.timeout = params.timeout; const isFailure = result.status !== 'skipped' && result.status !== test.expectedStatus; if (isFailure) diff --git a/packages/playwright/src/worker/workerMain.ts b/packages/playwright/src/worker/workerMain.ts index 9b61c4b0b0..54f5bf6968 100644 --- a/packages/playwright/src/worker/workerMain.ts +++ b/packages/playwright/src/worker/workerMain.ts @@ -292,6 +292,8 @@ export class WorkerMain extends ProcessRunner { for (const annotation of test.annotations) processAnnotation(annotation); + const staticAnnotations = new Set(testInfo.annotations); + // Process existing annotations dynamically set for parent suites. for (const suite of suites) { const extraAnnotations = this._activeSuites.get(suite) || []; @@ -310,7 +312,7 @@ export class WorkerMain extends ProcessRunner { if (isSkipped && nextTest && !hasAfterAllToRunBeforeNextTest) { // Fast path - this test is skipped, and there are more tests that will handle cleanup. testInfo.status = 'skipped'; - this.dispatchEvent('testEnd', buildTestEndPayload(testInfo)); + this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations)); return; } @@ -492,7 +494,7 @@ export class WorkerMain extends ProcessRunner { this._currentTest = null; setCurrentTestInfo(null); - this.dispatchEvent('testEnd', buildTestEndPayload(testInfo)); + this.dispatchEvent('testEnd', buildTestEndPayload(testInfo, staticAnnotations)); const preserveOutput = this._config.config.preserveOutput === 'always' || (this._config.config.preserveOutput === 'failures-only' && testInfo._isFailure()); @@ -612,7 +614,7 @@ function buildTestBeginPayload(testInfo: TestInfoImpl): TestBeginPayload { }; } -function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { +function buildTestEndPayload(testInfo: TestInfoImpl, staticAnnotations: Set): TestEndPayload { return { testId: testInfo.testId, duration: testInfo.duration, @@ -620,7 +622,7 @@ function buildTestEndPayload(testInfo: TestInfoImpl): TestEndPayload { errors: testInfo.errors, hasNonRetriableError: testInfo._hasNonRetriableError, expectedStatus: testInfo.expectedStatus, - annotations: testInfo.annotations, + annotations: testInfo.annotations.filter(a => !staticAnnotations.has(a)), timeout: testInfo.timeout, }; } diff --git a/packages/playwright/types/testReporter.d.ts b/packages/playwright/types/testReporter.d.ts index 2abfa9592e..0c27925be2 100644 --- a/packages/playwright/types/testReporter.d.ts +++ b/packages/playwright/types/testReporter.d.ts @@ -307,6 +307,7 @@ export interface JSONReportTestResult { body?: string; contentType: string; }[]; + annotations: { type: string, description?: string }[]; errorLocation?: Location; } @@ -445,10 +446,8 @@ export interface TestCase { * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip), * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) * and - * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail); - * - annotations appended to - * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations) during the test - * execution. + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail) + * prior to test execution. * * Annotations are available during test execution through * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). @@ -592,6 +591,34 @@ export interface TestError { * A result of a single [TestCase](https://playwright.dev/docs/api/class-testcase) run. */ export interface TestResult { + /** + * The list of annotations appended during test execution. Includes: + * - annotations implicitly added by methods + * [test.skip([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-skip), + * [test.fixme([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fixme) + * and + * [test.fail([title, details, body, condition, callback, description])](https://playwright.dev/docs/api/class-test#test-fail) + * during test execution; + * - annotations appended to + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). + * + * Annotations are available during test execution through + * [testInfo.annotations](https://playwright.dev/docs/api/class-testinfo#test-info-annotations). + * + * Learn more about [test annotations](https://playwright.dev/docs/test-annotations). + */ + annotations: Array<{ + /** + * Annotation type, for example `'skip'` or `'fail'`. + */ + type: string; + + /** + * Optional description. + */ + description?: string; + }>; + /** * The list of files or buffers attached during the test execution through * [testInfo.attachments](https://playwright.dev/docs/api/class-testinfo#test-info-attachments). diff --git a/packages/trace-viewer/src/ui/uiModeTraceView.tsx b/packages/trace-viewer/src/ui/uiModeTraceView.tsx index cf35d89007..84e2e92cb3 100644 --- a/packages/trace-viewer/src/ui/uiModeTraceView.tsx +++ b/packages/trace-viewer/src/ui/uiModeTraceView.tsx @@ -86,6 +86,8 @@ export const TraceView: React.FC<{ }; }, [outputDir, item, setModel, counter, setCounter, pathSeparator]); + const annotations = item.testCase ? [...item.testCase.annotations, ...(item.testCase.results[0]?.annotations ?? [])] : []; + return ; diff --git a/tests/playwright-test/access-data.spec.ts b/tests/playwright-test/access-data.spec.ts index 67958003be..90a12942b1 100644 --- a/tests/playwright-test/access-data.spec.ts +++ b/tests/playwright-test/access-data.spec.ts @@ -60,7 +60,7 @@ test('should access annotations in fixture', async ({ runInlineTest }) => { }); expect(exitCode).toBe(0); const test = report.suites[0].specs[0].tests[0]; - expect(test.annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]); + expect(test.results[0].annotations).toEqual([{ type: 'slow', description: 'just slow' }, { type: 'myname', description: 'hello' }]); expect(test.results[0].stdout).toEqual([{ text: 'console.log\n' }]); expect(test.results[0].stderr).toEqual([{ text: 'console.error\n' }]); }); diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index c06019a3b2..168d072120 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -434,7 +434,7 @@ export function expectTestHelper(result: RunResult) { for (const test of tests) { expect(test.expectedStatus, `title: ${title}`).toBe(expectedStatus); expect(test.status, `title: ${title}`).toBe(status); - expect(test.annotations.map(a => a.type), `title: ${title}`).toEqual(annotations); + expect([...test.annotations, ...test.results.flatMap(r => r.annotations)].map(a => a.type), `title: ${title}`).toEqual(annotations); } }; } diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts index 727cbe4077..b4e1c5fa5e 100644 --- a/tests/playwright-test/reporter-html.spec.ts +++ b/tests/playwright-test/reporter-html.spec.ts @@ -829,6 +829,33 @@ for (const useIntermediateMergeReport of [true, false] as const) { await expect(page.locator('.test-case-annotation')).toHaveText('issue: I am not interested in this test'); }); + test('should render dynamic annotations at test result level', async ({ runInlineTest, page, showReport }) => { + const result = await runInlineTest({ + 'playwright.config.js': ` + module.exports = { timeout: 1500, retries: 3 }; + `, + 'a.test.js': ` + import { test, expect } from '@playwright/test'; + test('annotated test', async ({}) => { + test.info().annotations.push({ type: 'foo', description: 'retry #' + test.info().retry }); + test.info().annotations.push({ type: 'bar', description: 'static value' }); + throw new Error('fail'); + }); + `, + }, { reporter: 'dot,html' }, { PLAYWRIGHT_HTML_OPEN: 'never' }); + expect(result.failed).toBe(1); + + await showReport(); + await page.getByRole('link', { name: 'annotated test' }).click(); + await page.getByRole('tab', { name: 'Retry #1' }).click(); + await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #1'); + + await page.getByRole('tab', { name: 'Retry #3' }).click(); + await expect(page.getByTestId('test-case-annotations')).toContainText('foo: retry #3'); + + await expect(page.getByTestId('test-case-annotations').getByText('static value')).toHaveCount(1); + }); + test('should render annotations as link if needed', async ({ runInlineTest, page, showReport, server }) => { const result = await runInlineTest({ 'playwright.config.js': ` @@ -2431,12 +2458,13 @@ for (const useIntermediateMergeReport of [true, false] as const) { 'a.test.js': ` const { test, expect } = require('@playwright/test'); test('annotated test',{ annotation :[{type:'key',description:'value'}]}, async ({}) => {expect(1).toBe(1);}); + test('slow test', () => { test.slow(); }); test('non-annotated test', async ({}) => {expect(1).toBe(2);}); `, }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' }); expect(result.exitCode).toBe(1); - expect(result.passed).toBe(1); + expect(result.passed).toBe(2); expect(result.failed).toBe(1); await showReport(); @@ -2456,6 +2484,11 @@ for (const useIntermediateMergeReport of [true, false] as const) { await expect(page.getByText('non-annotated test')).not.toBeVisible(); await expect(page.getByText('annotated test')).toBeVisible(); }); + + await test.step('filter by result annotation', async () => { + await searchInput.fill('annot:slow'); + await expect(page.getByText('slow test')).toBeVisible(); + }); }); test('tests should filter by fileName:line/column', async ({ runInlineTest, showReport, page }) => { diff --git a/tests/playwright-test/retry.spec.ts b/tests/playwright-test/retry.spec.ts index b5dfc7a347..80e4d23123 100644 --- a/tests/playwright-test/retry.spec.ts +++ b/tests/playwright-test/retry.spec.ts @@ -260,5 +260,5 @@ test('failed and skipped on retry should be marked as flaky', async ({ runInline expect(result.failed).toBe(0); expect(result.flaky).toBe(1); expect(result.output).toContain('Failed on first run'); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]); + expect(result.report.suites[0].specs[0].tests[0].results[1].annotations).toEqual([{ type: 'skip', description: 'Skipped on first retry' }]); }); diff --git a/tests/playwright-test/test-modifiers.spec.ts b/tests/playwright-test/test-modifiers.spec.ts index f7fb9a6ae0..86b9c2233c 100644 --- a/tests/playwright-test/test-modifiers.spec.ts +++ b/tests/playwright-test/test-modifiers.spec.ts @@ -106,7 +106,7 @@ test('test modifiers should work', async ({ runInlineTest }) => { const test = spec.tests[0]; expect(test.expectedStatus).toBe(expectedStatus); expect(test.results[0].status).toBe(status); - expect(test.annotations).toEqual(annotations); + expect([...test.annotations, ...test.results.flatMap(r => r.annotations)]).toEqual(annotations); }; expectTest('passed1', 'passed', 'passed', []); expectTest('passed2', 'passed', 'passed', []); @@ -306,23 +306,6 @@ test.describe('test modifier annotations', () => { expectTest('focused fail by suite', 'failed', 'expected', ['fail']); }); - test('should not multiple on retry', async ({ runInlineTest }) => { - const result = await runInlineTest({ - 'a.test.ts': ` - import { test, expect } from '@playwright/test'; - test('retry', () => { - test.info().annotations.push({ type: 'example' }); - expect(1).toBe(2); - }); - `, - }, { retries: 3 }); - const expectTest = expectTestHelper(result); - - expect(result.exitCode).toBe(1); - expect(result.passed).toBe(0); - expectTest('retry', 'passed', 'unexpected', ['example']); - }); - test('should not multiply on repeat-each', async ({ runInlineTest }) => { const result = await runInlineTest({ 'a.test.ts': ` @@ -424,7 +407,7 @@ test('should skip inside fixture', async ({ runInlineTest }) => { }); expect(result.exitCode).toBe(0); expect(result.skipped).toBe(1); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); }); test('modifier with a function should throw in the test', async ({ runInlineTest }) => { @@ -477,8 +460,8 @@ test('test.skip with worker fixtures only should skip before hooks and tests', a expect(result.passed).toBe(1); expect(result.skipped).toBe(2); expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([]); - expect(result.report.suites[0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); - expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].suites![0].suites![0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); expect(result.outputLines).toEqual([ 'beforeEach', 'passed', @@ -615,8 +598,8 @@ test('should skip all tests from beforeAll', async ({ runInlineTest }) => { 'beforeAll', 'afterAll', ]); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); - expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); + expect(result.report.suites[0].specs[1].tests[0].results[0].annotations).toEqual([{ type: 'skip', description: 'reason' }]); }); test('should report skipped tests in-order with correct properties', async ({ runInlineTest }) => { @@ -712,7 +695,7 @@ test('static modifiers should be added in serial mode', async ({ runInlineTest } expect(result.passed).toBe(0); expect(result.skipped).toBe(2); expect(result.didNotRun).toBe(1); - expect(result.report.suites[0].specs[0].tests[0].annotations).toEqual([{ type: 'slow' }]); + expect(result.report.suites[0].specs[0].tests[0].results[0].annotations).toEqual([{ type: 'slow' }]); expect(result.report.suites[0].specs[1].tests[0].annotations).toEqual([{ type: 'fixme' }]); expect(result.report.suites[0].specs[2].tests[0].annotations).toEqual([{ type: 'skip' }]); expect(result.report.suites[0].specs[3].tests[0].annotations).toEqual([]); diff --git a/utils/generate_types/overrides-testReporter.d.ts b/utils/generate_types/overrides-testReporter.d.ts index 70365a911e..b949dd3d0c 100644 --- a/utils/generate_types/overrides-testReporter.d.ts +++ b/utils/generate_types/overrides-testReporter.d.ts @@ -125,6 +125,7 @@ export interface JSONReportTestResult { body?: string; contentType: string; }[]; + annotations: { type: string, description?: string }[]; errorLocation?: Location; }