125 lines
5.3 KiB
TypeScript
Raw Normal View History

/**
* Copyright 2017 Google Inc. All rights reserved.
* Modifications copyright (c) Microsoft Corporation.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
2022-04-18 19:20:49 -08:00
import { colors, jpegjs } from '../utilsBundle';
chore: roll stable-test-runner to 1.28.1 (#19310) This required `allowJs: false` in our `tsconfig.json` due to the following error: ``` Error: Cannot find module './utilsBundleImpl' Require stack: - <playwright>/packages/playwright-core/src/utilsBundle.ts - <playwright>/packages/playwright-test/lib/reporters/html.js - <playwright>/tests/playwright-test/reporter-html.spec.ts - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/loader.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/runner.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/playwright-core/lib/cli/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/playwright-core/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli.js at ../../packages/playwright-core/src/utilsBundle.ts:20 18 | import path from 'path'; 19 | > 20 | export const colors: typeof import('../bundles/utils/node_modules/colors/safe') = require('./utilsBundleImpl').colors; | ^ 21 | export const debug: typeof import('../bundles/utils/node_modules/@types/debug') = require('./utilsBundleImpl').debug; 22 | export const getProxyForUrl: typeof import('../bundles/utils/node_modules/@types/proxy-from-env').getProxyForUrl = require('./utilsBundleImpl').getProxyForUrl; 23 | export const HttpsProxyAgent: typeof import('../bundles/utils/node_modules/https-proxy-agent').HttpsProxyAgent = require('./utilsBundleImpl').HttpsProxyAgent; at Object.<anonymous> (<playwright>/packages/playwright-core/src/utilsBundle.ts:20:83) at Object.<anonymous> (<playwright>/packages/playwright-test/src/reporters/html.ts:17:1) ```
2022-12-06 15:46:19 -08:00
const pixelmatch = require('../third_party/pixelmatch');
feat: implement a new image comparison function (#19166) This patch implements a new image comparison function, codenamed "ssim-cie94". The goal of the new comparison function is to cancel out browser non-determenistic rendering. To use the new comparison function: ```ts await expect(page).toHaveScreenshot({ comparator: 'ssim-cie94', }); ``` As of Nov 30, 2022, we identified the following sources of non-determenistic rendering for Chromium: - Anti-aliasing for certain shapes might be different due to the way skia rasterizes certain shapes. - Color blending might be different on `x86` and `aarch64` architectures. The new function employs a few heuristics to fight these differences. Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`: 1. If the [CIE94] metric is less then 1.0, then we consider these pixels **EQUAL**. (The value `1.0` is the [just-noticeable difference] for [CIE94].). Otherwise, proceed to next step. 1. If all the 8 neighbors of the first pixel match its color, or if the 8 neighbors of the second pixel match its color, then these pixels are **DIFFERENT**. (In case of anti-aliasing, some of the direct neighbors have to be blended up or down.) Otherwise, proceed to next step. 1. If SSIM in some locality around the different pixels is more than 0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be very close to 1.0). [CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94 [just-noticeable difference]: https://en.wikipedia.org/wiki/Just-noticeable_difference
2022-12-02 15:22:05 -08:00
import { compare } from '../image_tools/compare';
chore: roll stable-test-runner to 1.28.1 (#19310) This required `allowJs: false` in our `tsconfig.json` due to the following error: ``` Error: Cannot find module './utilsBundleImpl' Require stack: - <playwright>/packages/playwright-core/src/utilsBundle.ts - <playwright>/packages/playwright-test/lib/reporters/html.js - <playwright>/tests/playwright-test/reporter-html.spec.ts - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/loader.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/runner.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/playwright-core/lib/cli/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/playwright-core/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli.js at ../../packages/playwright-core/src/utilsBundle.ts:20 18 | import path from 'path'; 19 | > 20 | export const colors: typeof import('../bundles/utils/node_modules/colors/safe') = require('./utilsBundleImpl').colors; | ^ 21 | export const debug: typeof import('../bundles/utils/node_modules/@types/debug') = require('./utilsBundleImpl').debug; 22 | export const getProxyForUrl: typeof import('../bundles/utils/node_modules/@types/proxy-from-env').getProxyForUrl = require('./utilsBundleImpl').getProxyForUrl; 23 | export const HttpsProxyAgent: typeof import('../bundles/utils/node_modules/https-proxy-agent').HttpsProxyAgent = require('./utilsBundleImpl').HttpsProxyAgent; at Object.<anonymous> (<playwright>/packages/playwright-core/src/utilsBundle.ts:20:83) at Object.<anonymous> (<playwright>/packages/playwright-test/src/reporters/html.ts:17:1) ```
2022-12-06 15:46:19 -08:00
const { diff_match_patch, DIFF_INSERT, DIFF_DELETE, DIFF_EQUAL } = require('../third_party/diff_match_patch');
2022-04-18 19:20:49 -08:00
import { PNG } from '../utilsBundle';
export type ImageComparatorOptions = { threshold?: number, maxDiffPixels?: number, maxDiffPixelRatio?: number, _comparator?: string };
feat(toHaveScreenshot): align screenshot generation & comparison (#12812) This patch aligns the strategies that are used to generate new screnshot expectations and to compare screenshot expectations against baseline. With this patch, `toHaveScreenshot` will: - when generating a new expectation: will wait for 2 consecutive screenshots to match and accept the last one as expectation. - when given an expectation: * will compare first screenshot against expectation. If matches, resolve successfully * if first screenshot doesn't match, then wait for 2 consecutive screenshots to match and then compare last screenshot with the expectation. An example of a new detailed call log: ``` 1) a.spec.ts:3:1 › should work =================================================================== Error: Screenshot comparison failed: 20000 pixels (ratio 0.03 of all image pixels) are different Call log: - expect.toHaveScreenshot with timeout 5000ms - verifying given screenshot expectation - fast-path: checking first screenshot to match expectation - taking page screenshot - disabled all CSS animations - waiting for fonts to load... - fonts in all frames are loaded - fast-path failed: first screenshot did not match expectation - 20000 pixels (ratio 0.03 of all image pixels) are different - waiting for 2 consecutive screenshots to match - waiting 100ms before taking screenshot - taking page screenshot - disabled all CSS animations - waiting for fonts to load... - fonts in all frames are loaded - 2 consecutive screenshots matched - final screenshot did not match expectation - 20000 pixels (ratio 0.03 of all image pixels) are different - 20000 pixels (ratio 0.03 of all image pixels) are different Expected: /Users/andreylushnikov/tmp/test-results/a-should-work/should-work-1-expected.png Received: /Users/andreylushnikov/tmp/test-results/a-should-work/should-work-1-actual.png Diff: /Users/andreylushnikov/tmp/test-results/a-should-work/should-work-1-diff.png 3 | test('should work', async ({ page }) => { 4 | await page.goto('file:///Users/andreylushnikov/prog/playwright/tests/assets/rotate-z.html'); > 5 | await expect(page).toHaveScreenshot(); | ^ 6 | }); 7 | ```
2022-03-21 16:10:33 -06:00
export type ComparatorResult = { diff?: Buffer; errorMessage: string; } | null;
export type Comparator = (actualBuffer: Buffer | string, expectedBuffer: Buffer, options?: any) => ComparatorResult;
export function getComparator(mimeType: string): Comparator {
if (mimeType === 'image/png')
feat: implement a new image comparison function (#19166) This patch implements a new image comparison function, codenamed "ssim-cie94". The goal of the new comparison function is to cancel out browser non-determenistic rendering. To use the new comparison function: ```ts await expect(page).toHaveScreenshot({ comparator: 'ssim-cie94', }); ``` As of Nov 30, 2022, we identified the following sources of non-determenistic rendering for Chromium: - Anti-aliasing for certain shapes might be different due to the way skia rasterizes certain shapes. - Color blending might be different on `x86` and `aarch64` architectures. The new function employs a few heuristics to fight these differences. Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`: 1. If the [CIE94] metric is less then 1.0, then we consider these pixels **EQUAL**. (The value `1.0` is the [just-noticeable difference] for [CIE94].). Otherwise, proceed to next step. 1. If all the 8 neighbors of the first pixel match its color, or if the 8 neighbors of the second pixel match its color, then these pixels are **DIFFERENT**. (In case of anti-aliasing, some of the direct neighbors have to be blended up or down.) Otherwise, proceed to next step. 1. If SSIM in some locality around the different pixels is more than 0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be very close to 1.0). [CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94 [just-noticeable difference]: https://en.wikipedia.org/wiki/Just-noticeable_difference
2022-12-02 15:22:05 -08:00
return compareImages.bind(null, 'image/png');
if (mimeType === 'image/jpeg')
return compareImages.bind(null, 'image/jpeg');
if (mimeType === 'text/plain')
return compareText;
return compareBuffersOrStrings;
}
const JPEG_JS_MAX_BUFFER_SIZE_IN_MB = 5 * 1024; // ~5 GB
function compareBuffersOrStrings(actualBuffer: Buffer | string, expectedBuffer: Buffer): ComparatorResult {
if (typeof actualBuffer === 'string')
return compareText(actualBuffer, expectedBuffer);
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be a Buffer or a string.' };
if (Buffer.compare(actualBuffer, expectedBuffer))
return { errorMessage: 'Buffers differ' };
return null;
}
function compareImages(mimeType: string, actualBuffer: Buffer | string, expectedBuffer: Buffer, options: ImageComparatorOptions = {}): ComparatorResult {
if (!actualBuffer || !(actualBuffer instanceof Buffer))
return { errorMessage: 'Actual result should be a Buffer.' };
const actual = mimeType === 'image/png' ? PNG.sync.read(actualBuffer) : jpegjs.decode(actualBuffer, { maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB });
const expected = mimeType === 'image/png' ? PNG.sync.read(expectedBuffer) : jpegjs.decode(expectedBuffer, { maxMemoryUsageInMB: JPEG_JS_MAX_BUFFER_SIZE_IN_MB });
if (expected.width !== actual.width || expected.height !== actual.height) {
return {
errorMessage: `Expected an image ${expected.width}px by ${expected.height}px, received ${actual.width}px by ${actual.height}px. `
};
}
const diff = new PNG({ width: expected.width, height: expected.height });
feat: implement a new image comparison function (#19166) This patch implements a new image comparison function, codenamed "ssim-cie94". The goal of the new comparison function is to cancel out browser non-determenistic rendering. To use the new comparison function: ```ts await expect(page).toHaveScreenshot({ comparator: 'ssim-cie94', }); ``` As of Nov 30, 2022, we identified the following sources of non-determenistic rendering for Chromium: - Anti-aliasing for certain shapes might be different due to the way skia rasterizes certain shapes. - Color blending might be different on `x86` and `aarch64` architectures. The new function employs a few heuristics to fight these differences. Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`: 1. If the [CIE94] metric is less then 1.0, then we consider these pixels **EQUAL**. (The value `1.0` is the [just-noticeable difference] for [CIE94].). Otherwise, proceed to next step. 1. If all the 8 neighbors of the first pixel match its color, or if the 8 neighbors of the second pixel match its color, then these pixels are **DIFFERENT**. (In case of anti-aliasing, some of the direct neighbors have to be blended up or down.) Otherwise, proceed to next step. 1. If SSIM in some locality around the different pixels is more than 0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be very close to 1.0). [CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94 [just-noticeable difference]: https://en.wikipedia.org/wiki/Just-noticeable_difference
2022-12-02 15:22:05 -08:00
let count;
if (options._comparator === 'ssim-cie94') {
feat: implement a new image comparison function (#19166) This patch implements a new image comparison function, codenamed "ssim-cie94". The goal of the new comparison function is to cancel out browser non-determenistic rendering. To use the new comparison function: ```ts await expect(page).toHaveScreenshot({ comparator: 'ssim-cie94', }); ``` As of Nov 30, 2022, we identified the following sources of non-determenistic rendering for Chromium: - Anti-aliasing for certain shapes might be different due to the way skia rasterizes certain shapes. - Color blending might be different on `x86` and `aarch64` architectures. The new function employs a few heuristics to fight these differences. Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`: 1. If the [CIE94] metric is less then 1.0, then we consider these pixels **EQUAL**. (The value `1.0` is the [just-noticeable difference] for [CIE94].). Otherwise, proceed to next step. 1. If all the 8 neighbors of the first pixel match its color, or if the 8 neighbors of the second pixel match its color, then these pixels are **DIFFERENT**. (In case of anti-aliasing, some of the direct neighbors have to be blended up or down.) Otherwise, proceed to next step. 1. If SSIM in some locality around the different pixels is more than 0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be very close to 1.0). [CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94 [just-noticeable difference]: https://en.wikipedia.org/wiki/Just-noticeable_difference
2022-12-02 15:22:05 -08:00
count = compare(expected.data, actual.data, diff.data, expected.width, expected.height, {
// All ΔE* formulae are originally designed to have the difference of 1.0 stand for a "just noticeable difference" (JND).
// See https://en.wikipedia.org/wiki/Color_difference#CIELAB_%CE%94E*
maxColorDeltaE94: 1.0,
feat: implement a new image comparison function (#19166) This patch implements a new image comparison function, codenamed "ssim-cie94". The goal of the new comparison function is to cancel out browser non-determenistic rendering. To use the new comparison function: ```ts await expect(page).toHaveScreenshot({ comparator: 'ssim-cie94', }); ``` As of Nov 30, 2022, we identified the following sources of non-determenistic rendering for Chromium: - Anti-aliasing for certain shapes might be different due to the way skia rasterizes certain shapes. - Color blending might be different on `x86` and `aarch64` architectures. The new function employs a few heuristics to fight these differences. Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`: 1. If the [CIE94] metric is less then 1.0, then we consider these pixels **EQUAL**. (The value `1.0` is the [just-noticeable difference] for [CIE94].). Otherwise, proceed to next step. 1. If all the 8 neighbors of the first pixel match its color, or if the 8 neighbors of the second pixel match its color, then these pixels are **DIFFERENT**. (In case of anti-aliasing, some of the direct neighbors have to be blended up or down.) Otherwise, proceed to next step. 1. If SSIM in some locality around the different pixels is more than 0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be very close to 1.0). [CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94 [just-noticeable difference]: https://en.wikipedia.org/wiki/Just-noticeable_difference
2022-12-02 15:22:05 -08:00
});
} else if ((options._comparator ?? 'pixelmatch') === 'pixelmatch') {
feat: implement a new image comparison function (#19166) This patch implements a new image comparison function, codenamed "ssim-cie94". The goal of the new comparison function is to cancel out browser non-determenistic rendering. To use the new comparison function: ```ts await expect(page).toHaveScreenshot({ comparator: 'ssim-cie94', }); ``` As of Nov 30, 2022, we identified the following sources of non-determenistic rendering for Chromium: - Anti-aliasing for certain shapes might be different due to the way skia rasterizes certain shapes. - Color blending might be different on `x86` and `aarch64` architectures. The new function employs a few heuristics to fight these differences. Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`: 1. If the [CIE94] metric is less then 1.0, then we consider these pixels **EQUAL**. (The value `1.0` is the [just-noticeable difference] for [CIE94].). Otherwise, proceed to next step. 1. If all the 8 neighbors of the first pixel match its color, or if the 8 neighbors of the second pixel match its color, then these pixels are **DIFFERENT**. (In case of anti-aliasing, some of the direct neighbors have to be blended up or down.) Otherwise, proceed to next step. 1. If SSIM in some locality around the different pixels is more than 0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be very close to 1.0). [CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94 [just-noticeable difference]: https://en.wikipedia.org/wiki/Just-noticeable_difference
2022-12-02 15:22:05 -08:00
count = pixelmatch(expected.data, actual.data, diff.data, expected.width, expected.height, {
threshold: options.threshold ?? 0.2,
});
} else {
throw new Error(`Configuration specifies unknown comparator "${options._comparator}"`);
feat: implement a new image comparison function (#19166) This patch implements a new image comparison function, codenamed "ssim-cie94". The goal of the new comparison function is to cancel out browser non-determenistic rendering. To use the new comparison function: ```ts await expect(page).toHaveScreenshot({ comparator: 'ssim-cie94', }); ``` As of Nov 30, 2022, we identified the following sources of non-determenistic rendering for Chromium: - Anti-aliasing for certain shapes might be different due to the way skia rasterizes certain shapes. - Color blending might be different on `x86` and `aarch64` architectures. The new function employs a few heuristics to fight these differences. Consider two non-equal image pixels `(r1, g1, b1)` and `(r2, g2, b2)`: 1. If the [CIE94] metric is less then 1.0, then we consider these pixels **EQUAL**. (The value `1.0` is the [just-noticeable difference] for [CIE94].). Otherwise, proceed to next step. 1. If all the 8 neighbors of the first pixel match its color, or if the 8 neighbors of the second pixel match its color, then these pixels are **DIFFERENT**. (In case of anti-aliasing, some of the direct neighbors have to be blended up or down.) Otherwise, proceed to next step. 1. If SSIM in some locality around the different pixels is more than 0.99, then consider this pixels to be **EQUAL**. Otherwise, mark them as **DIFFERENT**. (Local SSIM for anti-aliased pixels turns out to be very close to 1.0). [CIE94]: https://en.wikipedia.org/wiki/Color_difference#CIE94 [just-noticeable difference]: https://en.wikipedia.org/wiki/Just-noticeable_difference
2022-12-02 15:22:05 -08:00
}
const maxDiffPixels1 = options.maxDiffPixels;
const maxDiffPixels2 = options.maxDiffPixelRatio !== undefined ? expected.width * expected.height * options.maxDiffPixelRatio : undefined;
let maxDiffPixels;
if (maxDiffPixels1 !== undefined && maxDiffPixels2 !== undefined)
maxDiffPixels = Math.min(maxDiffPixels1, maxDiffPixels2);
else
maxDiffPixels = maxDiffPixels1 ?? maxDiffPixels2 ?? 0;
const ratio = Math.ceil(count / (expected.width * expected.height) * 100) / 100;
return count > maxDiffPixels ? {
errorMessage: `${count} pixels (ratio ${ratio.toFixed(2)} of all image pixels) are different`,
diff: PNG.sync.write(diff),
} : null;
}
function compareText(actual: Buffer | string, expectedBuffer: Buffer): ComparatorResult {
if (typeof actual !== 'string')
return { errorMessage: 'Actual result should be a string' };
const expected = expectedBuffer.toString('utf-8');
if (expected === actual)
return null;
const dmp = new diff_match_patch();
const d = dmp.diff_main(expected, actual);
dmp.diff_cleanupSemantic(d);
return {
errorMessage: diff_prettyTerminal(d)
};
}
chore: roll stable-test-runner to 1.28.1 (#19310) This required `allowJs: false` in our `tsconfig.json` due to the following error: ``` Error: Cannot find module './utilsBundleImpl' Require stack: - <playwright>/packages/playwright-core/src/utilsBundle.ts - <playwright>/packages/playwright-test/lib/reporters/html.js - <playwright>/tests/playwright-test/reporter-html.spec.ts - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/loader.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/runner.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/lib/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/playwright-core/lib/cli/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/playwright-core/cli.js - <playwright>/tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli.js at ../../packages/playwright-core/src/utilsBundle.ts:20 18 | import path from 'path'; 19 | > 20 | export const colors: typeof import('../bundles/utils/node_modules/colors/safe') = require('./utilsBundleImpl').colors; | ^ 21 | export const debug: typeof import('../bundles/utils/node_modules/@types/debug') = require('./utilsBundleImpl').debug; 22 | export const getProxyForUrl: typeof import('../bundles/utils/node_modules/@types/proxy-from-env').getProxyForUrl = require('./utilsBundleImpl').getProxyForUrl; 23 | export const HttpsProxyAgent: typeof import('../bundles/utils/node_modules/https-proxy-agent').HttpsProxyAgent = require('./utilsBundleImpl').HttpsProxyAgent; at Object.<anonymous> (<playwright>/packages/playwright-core/src/utilsBundle.ts:20:83) at Object.<anonymous> (<playwright>/packages/playwright-test/src/reporters/html.ts:17:1) ```
2022-12-06 15:46:19 -08:00
function diff_prettyTerminal(diffs: [number, string][]) {
const html = [];
for (let x = 0; x < diffs.length; x++) {
const op = diffs[x][0]; // Operation (insert, delete, equal)
const data = diffs[x][1]; // Text of change.
const text = data;
switch (op) {
case DIFF_INSERT:
html[x] = colors.green(text);
break;
case DIFF_DELETE:
html[x] = colors.reset(colors.strikethrough(colors.red(text)));
break;
case DIFF_EQUAL:
html[x] = text;
break;
}
}
return html.join('');
}