From 396d9201458a48ee37cf1e4e51acb51a23df53b1 Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Mon, 28 Feb 2022 13:25:59 -0700 Subject: [PATCH] feat(test-runner): implement expect(pageOrLocator).toHaveScreenshot (#12242) Fixes #9938 --- docs/src/api/class-locatorassertions.md | 29 + docs/src/api/class-page.md | 16 +- docs/src/api/class-pageassertions.md | 33 +- docs/src/api/params.md | 28 + docs/src/test-assertions-js.md | 36 +- packages/playwright-core/src/client/page.ts | 37 + .../src/dispatchers/pageDispatcher.ts | 28 + .../playwright-core/src/protocol/channels.ts | 55 ++ .../playwright-core/src/protocol/protocol.yml | 39 ++ .../playwright-core/src/protocol/validator.ts | 24 + packages/playwright-core/src/server/dom.ts | 2 +- packages/playwright-core/src/server/frames.ts | 23 + packages/playwright-core/src/server/page.ts | 75 +- .../playwright-core/src/utils/comparators.ts | 7 +- packages/playwright-core/types/types.d.ts | 90 +-- packages/playwright-test/src/expect.ts | 3 +- .../src/matchers/toMatchSnapshot.ts | 120 +++- .../playwright-test/types/testExpect.d.ts | 41 +- tests/playwright-test/golden.spec.ts | 8 +- .../playwright-test-fixtures.ts | 4 +- .../to-have-screenshot.spec.ts | 645 ++++++++++++++++++ utils/generate_types/exported.json | 3 +- 22 files changed, 1259 insertions(+), 87 deletions(-) create mode 100644 tests/playwright-test/to-have-screenshot.spec.ts diff --git a/docs/src/api/class-locatorassertions.md b/docs/src/api/class-locatorassertions.md index 6bfd06a841..1d4b4b2890 100644 --- a/docs/src/api/class-locatorassertions.md +++ b/docs/src/api/class-locatorassertions.md @@ -968,6 +968,35 @@ Property value. ### option: LocatorAssertions.toHaveJSProperty.timeout = %%-assertions-timeout-%% +## async method: LocatorAssertions.toHaveScreenshot +* langs: js + +Ensures that [Locator] resolves to a given screenshot. This function will re-take +screenshots until it matches with the saved expectation. + +If there's no expectation yet, it will wait until two consecutive screenshots +yield the same result, and save the last one as an expectation. + +```js +const locator = page.locator('button'); +await expect(locator).toHaveScreenshot(); +``` + +### option: LocatorAssertions.toHaveScreenshot.timeout = %%-assertions-timeout-%% + +### option: LocatorAssertions.toHaveScreenshot.disableAnimations = %%-screenshot-option-disable-animations-%% + +### option: LocatorAssertions.toHaveScreenshot.omitBackground = %%-screenshot-option-omit-background-%% + +### option: LocatorAssertions.toHaveScreenshot.mask = %%-screenshot-option-mask-%% + +### option: LocatorAssertions.toHaveScreenshot.pixelCount = %%-assertions-pixel-count-%% + +### option: LocatorAssertions.toHaveScreenshot.pixelRatio = %%-assertions-pixel-ratio-%% + +### option: LocatorAssertions.toHaveScreenshot.threshold = %%-assertions-threshold-%% + + ## async method: LocatorAssertions.toHaveText * langs: - alias-java: hasText diff --git a/docs/src/api/class-page.md b/docs/src/api/class-page.md index 37307cf5d3..5979b8d552 100644 --- a/docs/src/api/class-page.md +++ b/docs/src/api/class-page.md @@ -2650,21 +2650,9 @@ Returns the buffer with the captured screenshot. ### option: Page.screenshot.-inline- = %%-screenshot-options-common-list-%% -### option: Page.screenshot.fullPage -- `fullPage` <[boolean]> +### option: Page.screenshot.fullPage = %%-screenshot-option-full-page-%% -When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to -`false`. - - -### option: Page.screenshot.clip -- `clip` <[Object]> - - `x` <[float]> x-coordinate of top-left corner of clip area - - `y` <[float]> y-coordinate of top-left corner of clip area - - `width` <[float]> width of clipping area - - `height` <[float]> height of clipping area - -An object which specifies clipping of the resulting image. Should have the following fields: +### option: Page.screenshot.clip = %%-screenshot-option-clip-%% ## async method: Page.selectOption - returns: <[Array]<[string]>> diff --git a/docs/src/api/class-pageassertions.md b/docs/src/api/class-pageassertions.md index e8805b3850..49019c01bb 100644 --- a/docs/src/api/class-pageassertions.md +++ b/docs/src/api/class-pageassertions.md @@ -114,6 +114,37 @@ Expected substring or RegExp. ### option: PageAssertions.NotToHaveURL.timeout = %%-assertions-timeout-%% +## async method: PageAssertions.toHaveScreenshot +* langs: js + +Ensures that the page resolves to a given screenshot. This function will re-take +screenshots until it matches with the saved expectation. + +If there's no expectation yet, it will wait until two consecutive screenshots +yield the same result, and save the last one as an expectation. + +```js +await expect(page).toHaveScreenshot(); +``` + +### option: PageAssertions.toHaveScreenshot.timeout = %%-assertions-timeout-%% + +### option: PageAssertions.toHaveScreenshot.disableAnimations = %%-screenshot-option-disable-animations-%% + +### option: PageAssertions.toHaveScreenshot.omitBackground = %%-screenshot-option-omit-background-%% + +### option: PageAssertions.toHaveScreenshot.fullPage = %%-screenshot-option-full-page-%% + +### option: PageAssertions.toHaveScreenshot.clip = %%-screenshot-option-clip-%% + +### option: PageAssertions.toHaveScreenshot.mask = %%-screenshot-option-mask-%% + +### option: PageAssertions.toHaveScreenshot.pixelCount = %%-assertions-pixel-count-%% + +### option: PageAssertions.toHaveScreenshot.pixelRatio = %%-assertions-pixel-ratio-%% + +### option: PageAssertions.toHaveScreenshot.threshold = %%-assertions-threshold-%% + ## async method: PageAssertions.toHaveTitle * langs: - alias-java: hasTitle @@ -194,4 +225,4 @@ await Expect(page).ToHaveURL(new Regex(".*checkout")); Expected substring or RegExp. -### option: PageAssertions.toHaveURL.timeout = %%-assertions-timeout-%% \ No newline at end of file +### option: PageAssertions.toHaveURL.timeout = %%-assertions-timeout-%% diff --git a/docs/src/api/params.md b/docs/src/api/params.md index 60ae684d17..8da0274650 100644 --- a/docs/src/api/params.md +++ b/docs/src/api/params.md @@ -709,6 +709,19 @@ using the [`method: AndroidDevice.setDefaultTimeout`] method. Time to retry the assertion for. Defaults to `timeout` in [`property: TestConfig.expect`]. +## assertions-pixel-count +* langs: js +- `pixelCount` <[int]> an acceptable amount of pixels that could be different, unset by default. + +## assertions-pixel-ratio +* langs: js +- `pixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`, unset by default. + +## assertions-threshold +* langs: js +- `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax), default is configurable with [`property: TestConfig.expect`]. Defaults to `0.2`. + + ## assertions-timeout * langs: java, python, csharp - `timeout` <[float]> @@ -919,6 +932,21 @@ Specify screenshot type, defaults to `png`. Specify locators that should be masked when the screenshot is taken. Masked elements will be overlayed with a pink box `#FF00FF` that completely covers its bounding box. +## screenshot-option-full-page +- `fullPage` <[boolean]> + +When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to +`false`. + +## screenshot-option-clip +- `clip` <[Object]> + - `x` <[float]> x-coordinate of top-left corner of clip area + - `y` <[float]> y-coordinate of top-left corner of clip area + - `width` <[float]> width of clipping area + - `height` <[float]> height of clipping area + +An object which specifies clipping of the resulting image. Should have the following fields: + ## screenshot-options-common-list - %%-screenshot-option-disable-animations-%% - %%-screenshot-option-omit-background-%% diff --git a/docs/src/test-assertions-js.md b/docs/src/test-assertions-js.md index 9bc9e5e61e..530d890594 100644 --- a/docs/src/test-assertions-js.md +++ b/docs/src/test-assertions-js.md @@ -349,7 +349,7 @@ await expect(page).toHaveURL(/.*checkout/); - `options` - `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax), default is configurable with [`property: TestConfig.expect`]. Defaults to `0.2`. - `pixelCount` <[int]> an acceptable amount of pixels that could be different, unset by default. - - `pixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1` , unset by default. + - `pixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`, unset by default. Ensures that passed value, either a [string] or a [Buffer], matches the expected snapshot stored in the test snapshots directory. @@ -366,3 +366,37 @@ expect(await page.screenshot()).toMatchSnapshot(['landing', 'step3.png']); ``` Learn more about [visual comparisons](./test-snapshots.md). + +## expect(pageOrLocator).toHaveScreenshot([options]) +- `options` + - `name` <[string] | [Array]<[string]>> Optional snapshot name. + - `disableAnimations` <[boolean]> When true, stops CSS animations, CSS transitions and Web Animations. Animations get different treatment depending on their duration: + - finite animations are fast-forwarded to completion, so they'll fire `transitionend` event. + - infinite animations are canceled to initial state, and then played over after the screenshot. + - `omitBackground` <[boolean]> Hides default white background and allows capturing screenshots with transparency. Defaults to `false`. + - `fullPage` <[boolean]> When true, takes a screenshot of the full scrollable page, instead of the currently visible viewport. Defaults to `false`. + - `mask` <[Array]<[Locator]>> Specify locators that should be masked when the screenshot is taken. Masked elements will be overlayed with +a pink box `#FF00FF` that completely covers its bounding box. + - `clip` <[Object]> An object which specifies clipping of the resulting image. + - `x` <[float]> x-coordinate of top-left corner of clip area + - `y` <[float]> y-coordinate of top-left corner of clip area + - `width` <[float]> width of clipping area + - `height` <[float]> height of clipping area + - `threshold` <[float]> an acceptable percieved color difference in the [YIQ color space](https://en.wikipedia.org/wiki/YIQ) between pixels in compared images, between zero (strict) and one (lax), default is configurable with [`property: TestConfig.expect`]. Defaults to `0.2`. + - `pixelCount` <[int]> an acceptable amount of pixels that could be different, unset by default. + - `pixelRatio` <[float]> an acceptable ratio of pixels that are different to the total amount of pixels, between `0` and `1`, unset by default. + - `timeout` <[number]> Time to retry assertion for, defaults to `timeout` in [`property: TestConfig.expect`]. + +Ensures that passed value, either a [string] or a [Buffer], matches the expected snapshot stored in the test snapshots directory. + +```js +// Basic usage. +await expect(page).toHaveScreenshot({ name: 'landing-page.png' }); +await expect(page.locator('text=Submit')).toHaveScreenshot(); + +// Take a full page screenshot and auto-generate screenshot name +await expect(page).toHaveScreenshot({ fullPage: true }); + +// Configure image matching properties. +await expect(page.locator('text=Submit').toHaveScreenshot({ pixelRatio: 0.01 }); +``` diff --git a/packages/playwright-core/src/client/page.ts b/packages/playwright-core/src/client/page.ts index 6bf2c68c71..22ad916239 100644 --- a/packages/playwright-core/src/client/page.ts +++ b/packages/playwright-core/src/client/page.ts @@ -62,6 +62,13 @@ type PDFOptions = Omit & }; type Listener = (...args: any[]) => void; +type ExpectScreenshotOptions = Omit & { + expected?: Buffer, + locator?: Locator, + isNot: boolean, + screenshotOptions: Omit & { mask?: Locator[] } +}; + export class Page extends ChannelOwner implements api.Page { private _browserContext: BrowserContext; _ownedContext: BrowserContext | undefined; @@ -476,6 +483,36 @@ export class Page extends ChannelOwner implements api.Page return buffer; } + async _expectScreenshot(options: ExpectScreenshotOptions): Promise<{ actual?: Buffer, previous?: Buffer, diff?: Buffer, errorMessage?: string, log?: string[]}> { + const mask = options.screenshotOptions?.mask ? options.screenshotOptions?.mask.map(locator => ({ + frame: locator._frame._channel, + selector: locator._selector, + })) : undefined; + const locator = options.locator ? { + frame: options.locator._frame._channel, + selector: options.locator._selector, + } : undefined; + const expected = options.expected ? options.expected.toString('base64') : undefined; + + const result = await this._channel.expectScreenshot({ + ...options, + isNot: !!options.isNot, + expected, + locator, + screenshotOptions: { + ...options.screenshotOptions, + mask, + } + }); + return { + log: result.log, + actual: result.actual ? Buffer.from(result.actual, 'base64') : undefined, + previous: result.previous ? Buffer.from(result.previous, 'base64') : undefined, + diff: result.diff ? Buffer.from(result.diff, 'base64') : undefined, + errorMessage: result.errorMessage, + }; + } + async title(): Promise { return this._mainFrame.title(); } diff --git a/packages/playwright-core/src/dispatchers/pageDispatcher.ts b/packages/playwright-core/src/dispatchers/pageDispatcher.ts index 68000ea5de..65ae5c2ca1 100644 --- a/packages/playwright-core/src/dispatchers/pageDispatcher.ts +++ b/packages/playwright-core/src/dispatchers/pageDispatcher.ts @@ -150,6 +150,34 @@ export class PageDispatcher extends Dispatcher imple }); } + async expectScreenshot(params: channels.PageExpectScreenshotParams, metadata: CallMetadata): Promise { + const mask: { frame: Frame, selector: string }[] = (params.screenshotOptions?.mask || []).map(({ frame, selector }) => ({ + frame: (frame as FrameDispatcher)._object, + selector, + })); + const locator: { frame: Frame, selector: string } | undefined = params.locator ? { + frame: (params.locator.frame as FrameDispatcher)._object, + selector: params.locator.selector, + } : undefined; + const expected = params.expected ? Buffer.from(params.expected, 'base64') : undefined; + const result = await this._page.expectScreenshot(metadata, { + ...params, + expected, + locator, + screenshotOptions: { + ...params.screenshotOptions, + mask, + }, + }); + return { + diff: result.diff?.toString('base64'), + errorMessage: result.errorMessage, + actual: result.actual?.toString('base64'), + previous: result.previous?.toString('base64'), + log: result.log, + }; + } + async screenshot(params: channels.PageScreenshotParams, metadata: CallMetadata): Promise { const mask: { frame: Frame, selector: string }[] = (params.mask || []).map(({ frame, selector }) => ({ frame: (frame as FrameDispatcher)._object, diff --git a/packages/playwright-core/src/protocol/channels.ts b/packages/playwright-core/src/protocol/channels.ts index 996f25857c..8eef17b58e 100644 --- a/packages/playwright-core/src/protocol/channels.ts +++ b/packages/playwright-core/src/protocol/channels.ts @@ -1330,6 +1330,7 @@ export interface PageChannel extends PageEventTarget, EventTargetChannel { goBack(params: PageGoBackParams, metadata?: Metadata): Promise; goForward(params: PageGoForwardParams, metadata?: Metadata): Promise; reload(params: PageReloadParams, metadata?: Metadata): Promise; + expectScreenshot(params: PageExpectScreenshotParams, metadata?: Metadata): Promise; screenshot(params: PageScreenshotParams, metadata?: Metadata): Promise; setExtraHTTPHeaders(params: PageSetExtraHTTPHeadersParams, metadata?: Metadata): Promise; setNetworkInterceptionEnabled(params: PageSetNetworkInterceptionEnabledParams, metadata?: Metadata): Promise; @@ -1486,6 +1487,60 @@ export type PageReloadOptions = { export type PageReloadResult = { response?: ResponseChannel, }; +export type PageExpectScreenshotParams = { + expected?: Binary, + timeout?: number, + isNot: boolean, + locator?: { + frame: FrameChannel, + selector: string, + }, + comparatorOptions?: { + pixelCount?: number, + pixelRatio?: number, + threshold?: number, + }, + screenshotOptions?: { + omitBackground?: boolean, + fullPage?: boolean, + disableAnimations?: boolean, + clip?: Rect, + mask?: { + frame: FrameChannel, + selector: string, + }[], + }, +}; +export type PageExpectScreenshotOptions = { + expected?: Binary, + timeout?: number, + locator?: { + frame: FrameChannel, + selector: string, + }, + comparatorOptions?: { + pixelCount?: number, + pixelRatio?: number, + threshold?: number, + }, + screenshotOptions?: { + omitBackground?: boolean, + fullPage?: boolean, + disableAnimations?: boolean, + clip?: Rect, + mask?: { + frame: FrameChannel, + selector: string, + }[], + }, +}; +export type PageExpectScreenshotResult = { + diff?: Binary, + errorMessage?: string, + actual?: Binary, + previous?: Binary, + log?: string[], +}; export type PageScreenshotParams = { timeout?: number, type?: 'png' | 'jpeg', diff --git a/packages/playwright-core/src/protocol/protocol.yml b/packages/playwright-core/src/protocol/protocol.yml index 910538a208..8e350bc7a5 100644 --- a/packages/playwright-core/src/protocol/protocol.yml +++ b/packages/playwright-core/src/protocol/protocol.yml @@ -990,6 +990,45 @@ Page: tracing: snapshot: true + expectScreenshot: + parameters: + expected: binary? + timeout: number? + isNot: boolean + locator: + type: object? + properties: + frame: Frame + selector: string + comparatorOptions: + type: object? + properties: + pixelCount: number? + pixelRatio: number? + threshold: number? + screenshotOptions: + type: object? + properties: + omitBackground: boolean? + fullPage: boolean? + disableAnimations: boolean? + clip: Rect? + mask: + type: array? + items: + type: object + properties: + frame: Frame + selector: string + returns: + diff: binary? + errorMessage: string? + actual: binary? + previous: binary? + log: + type: array? + items: string + screenshot: parameters: timeout: number? diff --git a/packages/playwright-core/src/protocol/validator.ts b/packages/playwright-core/src/protocol/validator.ts index 88006c0a1f..da3ace8ae4 100644 --- a/packages/playwright-core/src/protocol/validator.ts +++ b/packages/playwright-core/src/protocol/validator.ts @@ -540,6 +540,30 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme { timeout: tOptional(tNumber), waitUntil: tOptional(tType('LifecycleEvent')), }); + scheme.PageExpectScreenshotParams = tObject({ + expected: tOptional(tBinary), + timeout: tOptional(tNumber), + isNot: tBoolean, + locator: tOptional(tObject({ + frame: tChannel('Frame'), + selector: tString, + })), + comparatorOptions: tOptional(tObject({ + pixelCount: tOptional(tNumber), + pixelRatio: tOptional(tNumber), + threshold: tOptional(tNumber), + })), + screenshotOptions: tOptional(tObject({ + omitBackground: tOptional(tBoolean), + fullPage: tOptional(tBoolean), + disableAnimations: tOptional(tBoolean), + clip: tOptional(tType('Rect')), + mask: tOptional(tArray(tObject({ + frame: tChannel('Frame'), + selector: tString, + }))), + })), + }); scheme.PageScreenshotParams = tObject({ timeout: tOptional(tNumber), type: tOptional(tEnum(['png', 'jpeg'])), diff --git a/packages/playwright-core/src/server/dom.ts b/packages/playwright-core/src/server/dom.ts index 1c7090c745..aab15dbf27 100644 --- a/packages/playwright-core/src/server/dom.ts +++ b/packages/playwright-core/src/server/dom.ts @@ -119,7 +119,7 @@ export class ElementHandle extends js.JSHandle { declare readonly _context: FrameExecutionContext; readonly _page: Page; declare readonly _objectId: string; - private _frame: frames.Frame; + readonly _frame: frames.Frame; constructor(context: FrameExecutionContext, objectId: string) { super(context, 'node', undefined, objectId); diff --git a/packages/playwright-core/src/server/frames.ts b/packages/playwright-core/src/server/frames.ts index f99fe23834..fc2e258b00 100644 --- a/packages/playwright-core/src/server/frames.ts +++ b/packages/playwright-core/src/server/frames.ts @@ -36,6 +36,7 @@ import type { ElementStateWithoutStable, FrameExpectParams, InjectedScriptPoll, import { isSessionClosedError } from './protocolError'; import { isInvalidSelectorError, splitSelectorByFrame, stringifySelector, ParsedSelector } from './common/selectorParser'; import { SelectorInfo } from './selectors'; +import { ScreenshotOptions } from './screenshotter'; type ContextData = { contextPromise: ManualPromise; @@ -1057,6 +1058,13 @@ export class Frame extends SdkObject { }); } + async rafrafTimeoutScreenshotElementWithProgress(progress: Progress, selector: string, timeout: number, options: ScreenshotOptions): Promise { + return await this._retryWithProgressIfNotConnected(progress, selector, true /* strict */, async handle => { + await handle._frame.rafrafTimeout(timeout); + return await this._page._screenshotter.screenshotElement(progress, handle, options); + }); + } + async click(metadata: CallMetadata, selector: string, options: types.MouseClickOptions & types.PointerActionWaitOptions & types.NavigatingActionWaitOptions) { const controller = new ProgressController(metadata, this); return controller.run(async progress => { @@ -1371,6 +1379,21 @@ export class Frame extends SdkObject { return context.evaluate(() => document.title); } + async rafrafTimeout(timeout: number): Promise { + if (timeout === 0) + return; + const context = await this._utilityContext(); + await Promise.all([ + // wait for double raf + context.evaluate(() => new Promise(x => { + requestAnimationFrame(() => { + requestAnimationFrame(x); + }); + })), + new Promise(fulfill => setTimeout(fulfill, timeout)), + ]); + } + _onDetached() { this._stopNetworkIdleTimer(); this._detached = true; diff --git a/packages/playwright-core/src/server/page.ts b/packages/playwright-core/src/server/page.ts index 2373934b72..39542ac8d9 100644 --- a/packages/playwright-core/src/server/page.ts +++ b/packages/playwright-core/src/server/page.ts @@ -31,11 +31,12 @@ import { Progress, ProgressController } from './progress'; import { assert, isError } from '../utils/utils'; import { ManualPromise } from '../utils/async'; import { debugLogger } from '../utils/debugLogger'; +import { mimeTypeToComparator, ImageComparatorOptions, ComparatorResult } from '../utils/comparators'; import { SelectorInfo, Selectors } from './selectors'; import { CallMetadata, SdkObject } from './instrumentation'; import { Artifact } from './artifact'; -import { ParsedSelector } from './common/selectorParser'; import { TimeoutOptions } from '../common/types'; +import { isInvalidSelectorError, ParsedSelector } from './common/selectorParser'; export interface PageDelegate { readonly rawMouse: input.RawMouse; @@ -94,6 +95,18 @@ type PageState = { extraHTTPHeaders: types.HeadersArray | null; }; +type ExpectScreenshotOptions = { + timeout?: number, + expected?: Buffer, + isNot?: boolean, + locator?: { + frame: frames.Frame, + selector: string, + }, + comparatorOptions?: ImageComparatorOptions, + screenshotOptions?: ScreenshotOptions, +}; + export class Page extends SdkObject { static Events = { Close: 'close', @@ -425,7 +438,65 @@ export class Page extends SdkObject { route.continue(); } - async screenshot(metadata: CallMetadata, options: ScreenshotOptions & TimeoutOptions): Promise { + async expectScreenshot(metadata: CallMetadata, options: ExpectScreenshotOptions = {}): Promise<{ actual?: Buffer, previous?: Buffer, diff?: Buffer, errorMessage?: string, log?: string[] }> { + const locator = options.locator; + const rafrafScreenshot = locator ? async (progress: Progress, timeout: number) => { + return await locator.frame.rafrafTimeoutScreenshotElementWithProgress(progress, locator.selector, timeout, options.screenshotOptions || {}); + } : async (progress: Progress, timeout: number) => { + await this.mainFrame().rafrafTimeout(timeout); + return await this._screenshotter.screenshotPage(progress, options.screenshotOptions || {}); + }; + + const comparator = mimeTypeToComparator['image/png']; + const controller = new ProgressController(metadata, this); + const isGeneratingNewScreenshot = !options.expected; + if (isGeneratingNewScreenshot && options.isNot) + return { errorMessage: '"not" matcher requires expected result' }; + let intermediateResult: { + actual?: Buffer, + previous?: Buffer, + errorMessage?: string, + diff?: Buffer, + } | undefined = undefined; + return controller.run(async progress => { + let actual: Buffer | undefined; + let previous: Buffer | undefined; + let screenshotTimeout = 0; + while (true) { + progress.throwIfAborted(); + if (this.isClosed()) + throw new Error('The page has closed'); + let comparatorResult: ComparatorResult | undefined; + if (isGeneratingNewScreenshot) { + previous = actual; + actual = await rafrafScreenshot(progress, screenshotTimeout); + comparatorResult = actual && previous ? comparator(actual, previous, options.comparatorOptions) : undefined; + } else { + actual = await rafrafScreenshot(progress, screenshotTimeout); + comparatorResult = actual ? comparator(actual, options.expected!, options.comparatorOptions) : undefined; + } + screenshotTimeout = 150; + if (comparatorResult !== undefined && !!comparatorResult === !!options.isNot) + break; + if (comparatorResult) + intermediateResult = { errorMessage: comparatorResult.errorMessage, diff: comparatorResult.diff, actual, previous }; + } + + return isGeneratingNewScreenshot ? { actual } : {}; + }, this._timeoutSettings.timeout(options)).catch(e => { + // Q: Why not throw upon isSessionClosedError(e) as in other places? + // A: We want user to receive a friendly diff between actual and expected/previous. + if (js.isJavaScriptErrorInEvaluate(e) || isInvalidSelectorError(e)) + throw e; + return { + log: metadata.log, + ...intermediateResult, + errorMessage: intermediateResult?.errorMessage ?? e.message, + }; + }); + } + + async screenshot(metadata: CallMetadata, options: ScreenshotOptions & TimeoutOptions = {}): Promise { const controller = new ProgressController(metadata, this); return controller.run( progress => this._screenshotter.screenshotPage(progress, options), diff --git a/packages/playwright-core/src/utils/comparators.ts b/packages/playwright-core/src/utils/comparators.ts index 144d83a953..3b16b670e7 100644 --- a/packages/playwright-core/src/utils/comparators.ts +++ b/packages/playwright-core/src/utils/comparators.ts @@ -51,12 +51,13 @@ function compareImages(mimeType: string, actualBuffer: Buffer | string, expected const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpeg.decode(expectedBuffer); if (expected.width !== actual.width || expected.height !== actual.height) { return { - errorMessage: `Sizes differ; expected image ${expected.width}px X ${expected.height}px, but got ${actual.width}px X ${actual.height}px. ` + errorMessage: `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. ` }; } const diff = new PNG({ width: expected.width, height: expected.height }); - const thresholdOptions = { threshold: 0.2, ...options }; - const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, thresholdOptions); + const count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, { + threshold: options.threshold ?? 0.2, + }); const pixelCount1 = options.pixelCount; const pixelCount2 = options.pixelRatio !== undefined ? expected.width * expected.height * options.pixelRatio : undefined; diff --git a/packages/playwright-core/types/types.d.ts b/packages/playwright-core/types/types.d.ts index 87964aeced..c26b4236a1 100644 --- a/packages/playwright-core/types/types.d.ts +++ b/packages/playwright-core/types/types.d.ts @@ -9373,50 +9373,7 @@ export interface Locator { * screenshot. If the element is detached from DOM, the method throws an error. * @param options */ - screenshot(options?: { - /** - * When true, stops CSS animations, CSS transitions and Web Animations. Animations get different treatment depending on - * their duration: - */ - disableAnimations?: boolean; - - /** - * Specify locators that should be masked when the screenshot is taken. Masked elements will be overlayed with a pink box - * `#FF00FF` that completely covers its bounding box. - */ - mask?: Array; - - /** - * Hides default white background and allows capturing screenshots with transparency. Not applicable to `jpeg` images. - * Defaults to `false`. - */ - omitBackground?: boolean; - - /** - * The file path to save the image to. The screenshot type will be inferred from file extension. If `path` is a relative - * path, then it is resolved relative to the current working directory. If no path is provided, the image won't be saved to - * the disk. - */ - path?: string; - - /** - * The quality of the image, between 0-100. Not applicable to `png` images. - */ - quality?: number; - - /** - * Maximum time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The default value can be changed by - * using the - * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) - * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. - */ - timeout?: number; - - /** - * Specify screenshot type, defaults to `png`. - */ - type?: "png"|"jpeg"; - }): Promise; + screenshot(options?: LocatorScreenshotOptions): Promise; /** * This method waits for [actionability](https://playwright.dev/docs/actionability) checks, then tries to scroll element into view, unless it is @@ -15607,6 +15564,51 @@ export interface ConnectOptions { timeout?: number; } +export interface LocatorScreenshotOptions { + /** + * When true, stops CSS animations, CSS transitions and Web Animations. Animations get different treatment depending on + * their duration: + */ + disableAnimations?: boolean; + + /** + * Specify locators that should be masked when the screenshot is taken. Masked elements will be overlayed with a pink box + * `#FF00FF` that completely covers its bounding box. + */ + mask?: Array; + + /** + * Hides default white background and allows capturing screenshots with transparency. Not applicable to `jpeg` images. + * Defaults to `false`. + */ + omitBackground?: boolean; + + /** + * The file path to save the image to. The screenshot type will be inferred from file extension. If `path` is a relative + * path, then it is resolved relative to the current working directory. If no path is provided, the image won't be saved to + * the disk. + */ + path?: string; + + /** + * The quality of the image, between 0-100. Not applicable to `png` images. + */ + quality?: number; + + /** + * Maximum time in milliseconds, defaults to 30 seconds, pass `0` to disable timeout. The default value can be changed by + * using the + * [browserContext.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-browsercontext#browser-context-set-default-timeout) + * or [page.setDefaultTimeout(timeout)](https://playwright.dev/docs/api/class-page#page-set-default-timeout) methods. + */ + timeout?: number; + + /** + * Specify screenshot type, defaults to `png`. + */ + type?: "png"|"jpeg"; +} + interface ElementHandleWaitForSelectorOptions { /** * Defaults to `'visible'`. Can be either: diff --git a/packages/playwright-test/src/expect.ts b/packages/playwright-test/src/expect.ts index 96c7d9f564..16c8f45d7a 100644 --- a/packages/playwright-test/src/expect.ts +++ b/packages/playwright-test/src/expect.ts @@ -43,7 +43,7 @@ import { toHaveURL, toHaveValue } from './matchers/matchers'; -import { toMatchSnapshot } from './matchers/toMatchSnapshot'; +import { toMatchSnapshot, toHaveScreenshot } from './matchers/toMatchSnapshot'; import type { Expect, TestError } from './types'; import matchers from 'expect/build/matchers'; import { currentTestInfo } from './globals'; @@ -132,6 +132,7 @@ const customMatchers = { toHaveURL, toHaveValue, toMatchSnapshot, + toHaveScreenshot, }; type ExpectMetaInfo = { diff --git a/packages/playwright-test/src/matchers/toMatchSnapshot.ts b/packages/playwright-test/src/matchers/toMatchSnapshot.ts index 1c2019b1f7..777d5baa8b 100644 --- a/packages/playwright-test/src/matchers/toMatchSnapshot.ts +++ b/packages/playwright-test/src/matchers/toMatchSnapshot.ts @@ -14,10 +14,14 @@ * limitations under the License. */ +import { Locator, Page } from 'playwright-core'; +import type { Page as PageEx } from 'playwright-core/lib/client/page'; +import type { Locator as LocatorEx } from 'playwright-core/lib/client/locator'; import type { Expect } from '../types'; import { currentTestInfo } from '../globals'; import { mimeTypeToComparator, ImageComparatorOptions, Comparator } from 'playwright-core/lib/utils/comparators'; -import { addSuffixToFilePath, serializeError, sanitizeForFilePath, trimLongString, callLogText } from '../util'; +import type { PageScreenshotOptions } from 'playwright-core/types/types'; +import { addSuffixToFilePath, serializeError, sanitizeForFilePath, trimLongString, callLogText, currentExpectTimeout } from '../util'; import { UpdateSnapshots } from '../types'; import colors from 'colors/safe'; import fs from 'fs'; @@ -41,8 +45,10 @@ class SnapshotHelper { readonly actualPath: string; readonly diffPath: string; readonly mimeType: string; + readonly kind: 'Screenshot'|'Snapshot'; readonly updateSnapshots: UpdateSnapshots; - readonly comparatorOptions: T; + readonly comparatorOptions: ImageComparatorOptions; + readonly allOptions: T; constructor( testInfo: TestInfoImpl, @@ -103,7 +109,13 @@ class SnapshotHelper { this.diffPath = diffPath; this.snapshotPath = snapshotPath; this.updateSnapshots = updateSnapshots; - this.comparatorOptions = options; + this.allOptions = options; + this.comparatorOptions = { + pixelCount: options.pixelCount, + pixelRatio: options.pixelRatio, + threshold: options.threshold, + }; + this.kind = this.mimeType.startsWith('image/') ? 'Screenshot' : 'Snapshot'; } handleMissingNegated() { @@ -123,7 +135,7 @@ class SnapshotHelper { handleMatchingNegated() { const message = [ - colors.red('Snapshot comparison failed:'), + colors.red(`${this.kind} comparison failed:`), '', indent('Expected result should be different from the actual one.', ' '), ].join('\n'); @@ -156,7 +168,7 @@ class SnapshotHelper { diff: Buffer | string | undefined, diffError: string | undefined, log: string[] | undefined, - title = `Snapshot comparison failed:`) { + title = `${this.kind} comparison failed:`) { const output = [ colors.red(title), '', @@ -232,6 +244,104 @@ export function toMatchSnapshot( return helper.handleDifferent(received, expected, result.diff, result.errorMessage, undefined); } +type HaveScreenshotOptions = ImageComparatorOptions & Omit; + +export async function toHaveScreenshot( + this: ReturnType, + pageOrLocator: Page | Locator, + nameOrOptions: NameOrSegments | { name?: NameOrSegments } & HaveScreenshotOptions = {}, + optOptions: HaveScreenshotOptions = {} +): Promise { + const testInfo = currentTestInfo(); + if (!testInfo) + throw new Error(`toHaveScreenshot() must be called during the test`); + const helper = new SnapshotHelper(testInfo, 'png', nameOrOptions, optOptions); + const [page, locator] = pageOrLocator.constructor.name === 'Page' ? [(pageOrLocator as PageEx), undefined] : [(pageOrLocator as Locator).page() as PageEx, pageOrLocator as LocatorEx]; + const screenshotOptions = { + ...helper.allOptions, + mask: (helper.allOptions.mask || []) as LocatorEx[], + name: undefined, + threshold: undefined, + pixelCount: undefined, + pixelRatio: undefined, + }; + + const hasSnapshot = fs.existsSync(helper.snapshotPath); + if (this.isNot) { + if (!hasSnapshot) + return helper.handleMissingNegated(); + + // Having `errorMessage` means we timed out while waiting + // for screenshots not to match, so screenshots + // are actually the same in the end. + const isDifferent = !(await page._expectScreenshot({ + expected: await fs.promises.readFile(helper.snapshotPath), + isNot: true, + locator, + comparatorOptions: helper.comparatorOptions, + screenshotOptions, + timeout: currentExpectTimeout(helper.allOptions), + })).errorMessage; + return isDifferent ? helper.handleDifferentNegated() : helper.handleMatchingNegated(); + } + + // Fast path: there's no screenshot and we don't intend to update it. + if (helper.updateSnapshots === 'none' && !hasSnapshot) + return { pass: false, message: () => `${helper.snapshotPath} is missing in snapshots.` }; + + if (helper.updateSnapshots === 'all' || !hasSnapshot) { + // Regenerate a new screenshot by waiting until two screenshots are the same. + const timeout = currentExpectTimeout(helper.allOptions); + const { actual, previous, diff, errorMessage, log } = await page._expectScreenshot({ + expected: undefined, + isNot: false, + locator, + comparatorOptions: helper.comparatorOptions, + screenshotOptions, + timeout, + }); + // We tried re-generating new snapshot but failed. + // This can be due to e.g. spinning animation, so we want to show it as a diff. + if (errorMessage) { + // TODO(aslushnikov): rename attachments to "actual" and "previous". They still should be somehow shown in HTML reporter. + const title = actual && previous ? + `Timeout ${timeout}ms exceeded while generating screenshot because ${locator ? 'element' : 'page'} kept changing:` : + `Timeout ${timeout}ms exceeded while generating screenshot:`; + return helper.handleDifferent(actual, previous, diff, undefined, log, title); + } + + // We successfully (re-)generated new screenshot. + if (!hasSnapshot) + return helper.handleMissing(actual!); + + writeFileSync(helper.snapshotPath, actual!); + /* eslint-disable no-console */ + console.log(helper.snapshotPath + ' is re-generated, writing actual.'); + return { + pass: true, + message: () => helper.snapshotPath + ' running with --update-snapshots, writing actual.' + }; + } + + // General case: + // - snapshot exists + // - regular matcher (i.e. not a `.not`) + // - no flags to update screenshots + const expected = await fs.promises.readFile(helper.snapshotPath); + const { actual, diff, errorMessage, log } = await page._expectScreenshot({ + expected, + isNot: false, + locator, + comparatorOptions: helper.comparatorOptions, + screenshotOptions, + timeout: currentExpectTimeout(helper.allOptions), + }); + + return errorMessage ? + helper.handleDifferent(actual, expected, diff, errorMessage, log) : + helper.handleMatching(); +} + function writeFileSync(aPath: string, content: Buffer | string) { fs.mkdirSync(path.dirname(aPath), { recursive: true }); fs.writeFileSync(aPath, content); diff --git a/packages/playwright-test/types/testExpect.d.ts b/packages/playwright-test/types/testExpect.d.ts index 0c638176ec..7976d69b09 100644 --- a/packages/playwright-test/types/testExpect.d.ts +++ b/packages/playwright-test/types/testExpect.d.ts @@ -15,7 +15,7 @@ */ import type * as expect from 'expect'; -import type { Page, Locator, APIResponse } from 'playwright-core'; +import type { Page, Locator, APIResponse, PageScreenshotOptions, LocatorScreenshotOptions } from 'playwright-core'; export declare type AsymmetricMatcher = Record; @@ -46,6 +46,12 @@ export declare type Expect = { stringMatching(expected: string | RegExp): AsymmetricMatcher; }; +type ImageComparatorOptions = { + threshold?: number, + pixelCount?: number, + pixelRatio?: number, +}; + type Awaited = T extends PromiseLike ? U : T; type OverriddenExpectProperties = @@ -77,18 +83,13 @@ declare global { /** * Match snapshot */ - toMatchSnapshot(options?: { + toMatchSnapshot(options?: ImageComparatorOptions & { name?: string | string[], - threshold?: number, - pixelCount?: number, - pixelRatio?: number, }): R; /** * Match snapshot */ - toMatchSnapshot(name: string | string[], options?: { - threshold?: number - }): R; + toMatchSnapshot(name: string | string[], options?: ImageComparatorOptions): R; } } } @@ -178,6 +179,18 @@ interface LocatorMatchers { * Asserts given DOM node visible on the screen. */ toBeVisible(options?: { timeout?: number }): Promise; + + /** + * Asserts element's screenshot is matching to the snapshot. + */ + toHaveScreenshot(options?: Omit & ImageComparatorOptions & { + name?: string | string[], + }): Promise; + + /** + * Asserts element's screenshot is matching to the snapshot. + */ + toHaveScreenshot(name: string | string[], options?: Omit & ImageComparatorOptions): Promise; } interface PageMatchers { /** @@ -189,6 +202,18 @@ interface PageMatchers { * Asserts page's URL. */ toHaveURL(expected: string | RegExp, options?: { timeout?: number }): Promise; + + /** + * Asserts page screenshot is matching to the snapshot. + */ + toHaveScreenshot(options?: Omit & ImageComparatorOptions & { + name?: string | string[], + }): Promise; + + /** + * Asserts page screenshot is matching to the snapshot. + */ + toHaveScreenshot(name: string | string[], options?: Omit & ImageComparatorOptions): Promise; } interface APIResponseMatchers { diff --git a/tests/playwright-test/golden.spec.ts b/tests/playwright-test/golden.spec.ts index ddebe109d3..fcda868af4 100644 --- a/tests/playwright-test/golden.spec.ts +++ b/tests/playwright-test/golden.spec.ts @@ -74,8 +74,8 @@ test('should generate default name', async ({ runInlineTest }, testInfo) => { test('should compile with different option combinations', async ({ runTSC }) => { const result = await runTSC({ - 'a.spec.js': ` - const { test, expect } = pwt; + 'a.spec.ts': ` + const { test } = pwt; test('is a test', async ({ page }) => { expect('foo').toMatchSnapshot(); expect('foo').toMatchSnapshot({ threshold: 0.2 }); @@ -610,7 +610,7 @@ test('should compare different PNG images', async ({ runInlineTest }, testInfo) const outputText = stripAnsi(result.output); expect(result.exitCode).toBe(1); - expect(outputText).toContain('Snapshot comparison failed:'); + expect(outputText).toContain('Screenshot comparison failed:'); const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.png'); const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.png'); const diffSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-diff.png'); @@ -915,7 +915,7 @@ test('should attach expected/actual and no diff', async ({ runInlineTest }, test }); const outputText = stripAnsi(result.output); - expect(outputText).toContain('Sizes differ; expected image 2px X 2px, but got 1px X 1px.'); + expect(outputText).toContain('Expected an image 2px by 2px, received 1px by 1px.'); const attachments = outputText.split('\n').filter(l => l.startsWith('## ')).map(l => l.substring(3)).map(l => JSON.parse(l))[0]; for (const attachment of attachments) attachment.path = attachment.path.replace(/\\/g, '/').replace(/.*test-results\//, ''); diff --git a/tests/playwright-test/playwright-test-fixtures.ts b/tests/playwright-test/playwright-test-fixtures.ts index 8f0fc97537..e0c6dbad76 100644 --- a/tests/playwright-test/playwright-test-fixtures.ts +++ b/tests/playwright-test/playwright-test-fixtures.ts @@ -267,14 +267,14 @@ export function countTimes(s: string, sub: string): number { return result; } -export function createImage(width: number, height: number, r: number = 0, g: number = 0, b: number = 0): Buffer { +export function createImage(width: number, height: number, r: number = 0, g: number = 0, b: number = 0, a: number = 255): Buffer { const image = new PNG({ width, height }); // Make both images red. for (let i = 0; i < width * height; ++i) { image.data[i * 4 + 0] = r; image.data[i * 4 + 1] = g; image.data[i * 4 + 2] = b; - image.data[i * 4 + 3] = 255; + image.data[i * 4 + 3] = a; } return PNG.sync.write(image); } diff --git a/tests/playwright-test/to-have-screenshot.spec.ts b/tests/playwright-test/to-have-screenshot.spec.ts new file mode 100644 index 0000000000..a8ac19eea5 --- /dev/null +++ b/tests/playwright-test/to-have-screenshot.spec.ts @@ -0,0 +1,645 @@ +/** + * Copyright Microsoft Corporation. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { mimeTypeToComparator } from 'playwright-core/lib/utils/comparators'; +import * as fs from 'fs'; +import { PNG } from 'pngjs'; +import * as path from 'path'; +import { pathToFileURL } from 'url'; +import { test, expect, stripAnsi, createImage, paintBlackPixels } from './playwright-test-fixtures'; + +const pngComparator = mimeTypeToComparator['image/png']; + +test.describe.configure({ mode: 'parallel' }); + +const IMG_WIDTH = 1280; +const IMG_HEIGHT = 720; +const whiteImage = createImage(IMG_WIDTH, IMG_HEIGHT, 255, 255, 255); +const redImage = createImage(IMG_WIDTH, IMG_HEIGHT, 255, 0, 0); +const greenImage = createImage(IMG_WIDTH, IMG_HEIGHT, 0, 255, 0); +const blueImage = createImage(IMG_WIDTH, IMG_HEIGHT, 0, 0, 255); + +const files = { + 'helper.ts': ` + export const test = pwt.test.extend({ + auto: [ async ({}, run, testInfo) => { + testInfo.snapshotSuffix = ''; + await run(); + }, { auto: true } ] + }); + ` +}; + +test('should fail to screenshot a page with infinite animation', async ({ runInlineTest }, testInfo) => { + const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await page.goto('${infiniteAnimationURL}'); + await expect(page).toHaveScreenshot({ timeout: 2000 }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain(`Timeout 2000ms exceeded while generating screenshot because page kept changing`); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-actual.png'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-expected.png'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-diff.png'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(false); +}); + +test('should not fail when racing with navigation', async ({ runInlineTest }, testInfo) => { + const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': createImage(10, 10, 255, 0, 0), + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await Promise.all([ + page.goto('${infiniteAnimationURL}'), + expect(page).toHaveScreenshot({ + name: 'snapshot.png', + disableAnimations: true, + clip: { x: 0, y: 0, width: 10, height: 10 }, + }), + ]); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should successfully screenshot a page with infinite animation with disableAnimation: true', async ({ runInlineTest }, testInfo) => { + const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await page.goto('${infiniteAnimationURL}'); + await expect(page).toHaveScreenshot({ + disableAnimations: true, + }); + }); + ` + }, { 'update-snapshots': true }); + expect(result.exitCode).toBe(0); + expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(true); +}); + +test('should support clip option for page', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': createImage(50, 50, 255, 255, 255), + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot({ + name: 'snapshot.png', + clip: { x: 0, y: 0, width: 50, height: 50, }, + }); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should support omitBackground option for locator', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await page.evaluate(() => { + document.body.style.setProperty('width', '100px'); + document.body.style.setProperty('height', '100px'); + }); + await expect(page.locator('body')).toHaveScreenshot({ + name: 'snapshot.png', + omitBackground: true, + }); + }); + ` + }, { 'update-snapshots': true }); + expect(result.exitCode).toBe(0); + const snapshotPath = testInfo.outputPath('a.spec.js-snapshots', 'snapshot.png'); + expect(fs.existsSync(snapshotPath)).toBe(true); + const png = PNG.sync.read(fs.readFileSync(snapshotPath)); + expect.soft(png.width, 'image width must be 100').toBe(100); + expect.soft(png.height, 'image height must be 100').toBe(100); + expect.soft(png.data[0], 'image R must be 0').toBe(0); + expect.soft(png.data[1], 'image G must be 0').toBe(0); + expect.soft(png.data[2], 'image B must be 0').toBe(0); + expect.soft(png.data[3], 'image A must be 0').toBe(0); +}); + +test('should fail to screenshot an element with infinite animation', async ({ runInlineTest }, testInfo) => { + const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await page.goto('${infiniteAnimationURL}'); + await expect(page.locator('body')).toHaveScreenshot({ timeout: 2000 }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain(`Timeout 2000ms exceeded while generating screenshot because element kept changing`); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-actual.png'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-expected.png'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-diff.png'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(false); +}); + +test('should fail to screenshot an element that keeps moving', async ({ runInlineTest }, testInfo) => { + const infiniteAnimationURL = pathToFileURL(path.join(__dirname, '../assets/rotate-z.html')); + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await page.goto('${infiniteAnimationURL}'); + await expect(page.locator('div')).toHaveScreenshot({ timeout: 2000 }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(stripAnsi(result.output)).toContain(`Timeout 2000ms exceeded`); + expect(stripAnsi(result.output)).toContain(`element is not stable - waiting`); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-actual.png'))).toBe(false); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-expected.png'))).toBe(false); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-diff.png'))).toBe(false); + expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(false); +}); + +test('should generate default name', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot(); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(fs.existsSync(testInfo.outputPath('test-results', 'a-is-a-test', 'is-a-test-1-actual.png'))).toBe(true); + expect(fs.existsSync(testInfo.outputPath('a.spec.js-snapshots', 'is-a-test-1.png'))).toBe(true); +}); + +test('should compile with different option combinations', async ({ runTSC }) => { + const result = await runTSC({ + 'a.spec.ts': ` + const { test } = pwt; + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot(); + await expect(page.locator('body')).toHaveScreenshot({ threshold: 0.2 }); + await expect(page).toHaveScreenshot({ pixelRatio: 0.2 }); + await expect(page).toHaveScreenshot({ + threshold: 0.2, + pixelCount: 10, + pixelRatio: 0.2, + disableAnimations: true, + omitBackground: true, + timeout: 1000, + }); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should fail when screenshot is different size', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': createImage(22, 33), + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Expected an image 22px by 33px, received 1280px by 720px.'); +}); + +test('should fail when screenshot is different pixels', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': blueImage, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }); + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Timeout 2000ms exceeded'); + expect(result.output).toContain('Screenshot comparison failed'); + expect(result.output).toContain('Expected:'); + expect(result.output).toContain('Received:'); +}); + +test('doesn\'t create comparison artifacts in an output folder for passed negated snapshot matcher', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': blueImage, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).not.toHaveScreenshot('snapshot.png'); + }); + ` + }); + + expect(result.exitCode).toBe(0); + const outputText = stripAnsi(result.output); + const expectedSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-expected.png'); + const actualSnapshotArtifactPath = testInfo.outputPath('test-results', 'a-is-a-test', 'snapshot-actual.png'); + expect(outputText).not.toContain(`Expected: ${expectedSnapshotArtifactPath}`); + expect(outputText).not.toContain(`Received: ${actualSnapshotArtifactPath}`); + expect(fs.existsSync(expectedSnapshotArtifactPath)).toBe(false); + expect(fs.existsSync(actualSnapshotArtifactPath)).toBe(false); +}); + +test('should fail on same snapshots with negate matcher', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': whiteImage, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).not.toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.output).toContain('Screenshot comparison failed:'); + expect(result.output).toContain('Expected result should be different from the actual one.'); +}); + +test('should write missing expectations locally twice and continue', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + await expect(page).toHaveScreenshot('snapshot2.png'); + console.log('Here we are!'); + }); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + + const snapshot1OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(result.output).toContain(`Error: ${snapshot1OutputPath} is missing in snapshots, writing actual`); + expect(pngComparator(fs.readFileSync(snapshot1OutputPath), whiteImage)).toBe(null); + + const snapshot2OutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot2.png'); + expect(result.output).toContain(`Error: ${snapshot2OutputPath} is missing in snapshots, writing actual`); + expect(pngComparator(fs.readFileSync(snapshot2OutputPath), whiteImage)).toBe(null); + + expect(result.output).toContain('Here we are!'); + + const stackLines = stripAnsi(result.output).split('\n').filter(line => line.includes(' at ')).filter(line => !line.includes(testInfo.outputPath())); + expect(result.output).toContain('a.spec.js:8'); + expect(stackLines.length).toBe(0); +}); + +test('shouldn\'t write missing expectations locally for negated matcher', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).not.toHaveScreenshot('snapshot.png'); + }); + ` + }); + + expect(result.exitCode).toBe(1); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`); + expect(fs.existsSync(snapshotOutputPath)).toBe(false); +}); + +test('should update snapshot with the update-snapshots flag', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': blueImage, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(0); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(result.output).toContain(`${snapshotOutputPath} is re-generated, writing actual.`); + expect(pngComparator(fs.readFileSync(snapshotOutputPath), whiteImage)).toBe(null); +}); + +test('shouldn\'t update snapshot with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => { + const EXPECTED_SNAPSHOT = blueImage; + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).not.toHaveScreenshot('snapshot.png'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(0); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(fs.readFileSync(snapshotOutputPath).equals(EXPECTED_SNAPSHOT)).toBe(true); +}); + +test('should silently write missing expectations locally with the update-snapshots flag', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(0); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`); + const data = fs.readFileSync(snapshotOutputPath); + expect(pngComparator(data, whiteImage)).toBe(null); +}); + +test('should not write missing expectations locally with the update-snapshots flag for negated matcher', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).not.toHaveScreenshot('snapshot.png'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(1); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, matchers using ".not" won\'t write them automatically.`); + expect(fs.existsSync(snapshotOutputPath)).toBe(false); +}); + +test('should match multiple snapshots', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/red.png': redImage, + 'a.spec.js-snapshots/green.png': greenImage, + 'a.spec.js-snapshots/blue.png': blueImage, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await Promise.all([ + page.evaluate(() => document.documentElement.style.setProperty('background', '#f00')), + expect(page).toHaveScreenshot('red.png'), + ]); + await Promise.all([ + page.evaluate(() => document.documentElement.style.setProperty('background', '#0f0')), + expect(page).toHaveScreenshot('green.png'), + ]); + await Promise.all([ + page.evaluate(() => document.documentElement.style.setProperty('background', '#00f')), + expect(page).toHaveScreenshot('blue.png'), + ]); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should use provided name', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/provided.png': whiteImage, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('provided.png'); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should use provided name via options', async ({ runInlineTest }) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/provided.png': whiteImage, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot({ name: 'provided.png' }); + }); + ` + }); + expect(result.exitCode).toBe(0); +}); + +test('should respect pixelCount option', async ({ runInlineTest }) => { + const BAD_PIXELS = 120; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + })).exitCode, 'make sure default comparison fails').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { + pixelCount: ${BAD_PIXELS} + }); + }); + ` + })).exitCode, 'make sure pixelCount option is respected').toBe(0); + + expect((await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { projects: [ + { expect: { toMatchSnapshot: { pixelCount: ${BAD_PIXELS} } } }, + ]}; + `, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure pixelCount option in project config is respected').toBe(0); +}); + +test('should respect pixelRatio option', async ({ runInlineTest }) => { + const BAD_RATIO = 0.25; + const BAD_PIXELS = IMG_WIDTH * IMG_HEIGHT * BAD_RATIO; + const EXPECTED_SNAPSHOT = paintBlackPixels(whiteImage, BAD_PIXELS); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + })).exitCode, 'make sure default comparison fails').toBe(1); + + expect((await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { + pixelRatio: ${BAD_RATIO} + }); + }); + ` + })).exitCode, 'make sure pixelRatio option is respected').toBe(0); + + expect((await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { projects: [ + { expect: { toMatchSnapshot: { pixelRatio: ${BAD_RATIO} } } }, + ]}; + `, + 'a.spec.js-snapshots/snapshot.png': EXPECTED_SNAPSHOT, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + }); + ` + })).exitCode, 'make sure pixelCount option in project config is respected').toBe(0); +}); + +test('should attach expected/actual and no diff when sizes are different', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'a.spec.js-snapshots/snapshot.png': createImage(2, 2), + 'a.spec.js': ` + const { test } = require('./helper'); + test.afterEach(async ({}, testInfo) => { + console.log('## ' + JSON.stringify(testInfo.attachments)); + }); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + ` + }); + + expect(result.exitCode).toBe(1); + const outputText = stripAnsi(result.output); + expect(outputText).toContain('Expected an image 2px by 2px, received 1280px by 720px.'); + const attachments = outputText.split('\n').filter(l => l.startsWith('## ')).map(l => l.substring(3)).map(l => JSON.parse(l))[0]; + for (const attachment of attachments) + attachment.path = attachment.path.replace(/\\/g, '/').replace(/.*test-results\//, ''); + expect(attachments).toEqual([ + { + name: 'expected', + contentType: 'image/png', + path: 'a-is-a-test/snapshot-expected.png' + }, + { + name: 'actual', + contentType: 'image/png', + path: 'a-is-a-test/snapshot-actual.png' + }, + ]); +}); + +test('should fail with missing expectations and retries', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { retries: 1 }; + `, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + }); + ` + }); + + expect(result.exitCode).toBe(1); + expect(result.failed).toBe(1); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`); + const data = fs.readFileSync(snapshotOutputPath); + expect(pngComparator(data, whiteImage)).toBe(null); +}); + +test('should update expectations with retries', async ({ runInlineTest }, testInfo) => { + const result = await runInlineTest({ + ...files, + 'playwright.config.ts': ` + module.exports = { retries: 1 }; + `, + 'a.spec.js': ` + const { test } = require('./helper'); + test('is a test', async ({ page }) => { + await expect(page).toHaveScreenshot('snapshot.png'); + }); + ` + }, { 'update-snapshots': true }); + + expect(result.exitCode).toBe(0); + expect(result.passed).toBe(1); + const snapshotOutputPath = testInfo.outputPath('a.spec.js-snapshots/snapshot.png'); + expect(result.output).toContain(`${snapshotOutputPath} is missing in snapshots, writing actual`); + const data = fs.readFileSync(snapshotOutputPath); + expect(pngComparator(data, whiteImage)).toBe(null); +}); + diff --git a/utils/generate_types/exported.json b/utils/generate_types/exported.json index 8bc1d72515..7cd191c644 100644 --- a/utils/generate_types/exported.json +++ b/utils/generate_types/exported.json @@ -7,5 +7,6 @@ "BrowserNewContextOptionsViewport": "ViewportSize", "BrowserNewContextOptionsGeolocation": "Geolocation", "BrowserNewContextOptionsHttpCredentials": "HTTPCredentials", - "PageScreenshotOptions": "PageScreenshotOptions" + "PageScreenshotOptions": "PageScreenshotOptions", + "LocatorScreenshotOptions": "LocatorScreenshotOptions" }