From 2a675026de60d6c05dba1ee448d60bbfd490c2b2 Mon Sep 17 00:00:00 2001 From: Pavel Feldman Date: Fri, 28 Apr 2023 14:18:46 -0700 Subject: [PATCH] chore: add image diff test to ui mode (#22637) --- tests/playwright-test/ui-mode-fixtures.ts | 3 +- .../ui-mode-test-filters.spec.ts | 20 ++++----- .../ui-mode-test-output.spec.ts | 2 +- .../ui-mode-test-progress.spec.ts | 12 +++--- .../playwright-test/ui-mode-test-run.spec.ts | 42 +++++++++---------- .../ui-mode-test-setup.spec.ts | 8 ++-- .../ui-mode-test-source.spec.ts | 2 +- .../playwright-test/ui-mode-test-tree.spec.ts | 30 ++++++------- .../ui-mode-test-update.spec.ts | 22 +++++----- .../ui-mode-test-watch.spec.ts | 38 ++++++++--------- tests/playwright-test/ui-mode-trace.spec.ts | 36 +++++++++++++--- 11 files changed, 121 insertions(+), 94 deletions(-) diff --git a/tests/playwright-test/ui-mode-fixtures.ts b/tests/playwright-test/ui-mode-fixtures.ts index 397661fca1..4b39d89655 100644 --- a/tests/playwright-test/ui-mode-fixtures.ts +++ b/tests/playwright-test/ui-mode-fixtures.ts @@ -124,7 +124,8 @@ export const test = base }, }); -export { expect } from './stable-test-runner'; +import { expect as baseExpect } from './stable-test-runner'; +export const expect = baseExpect.configure({ timeout: 15000 }); async function waitForLatch(latchFile: string) { const fs = require('fs'); diff --git a/tests/playwright-test/ui-mode-test-filters.spec.ts b/tests/playwright-test/ui-mode-test-filters.spec.ts index 206ebf626d..24476eadf7 100644 --- a/tests/playwright-test/ui-mode-test-filters.spec.ts +++ b/tests/playwright-test/ui-mode-test-filters.spec.ts @@ -38,7 +38,7 @@ const basicTestTree = { test('should filter by title', async ({ runUITest }) => { const { page } = await runUITest(basicTestTree); await page.getByPlaceholder('Filter').fill('inner'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ▼ ◯ suite ◯ inner passes @@ -51,7 +51,7 @@ test('should filter by status', async ({ runUITest }) => { await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ✅ passes ❌ fails <= @@ -67,7 +67,7 @@ test('should filter by status', async ({ runUITest }) => { await page.getByLabel('failed').setChecked(true); await expect(page.getByText('Status: failed')).toBeVisible(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ❌ fails <= ► ❌ suite @@ -104,7 +104,7 @@ test('should filter by project', async ({ runUITest }) => { ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -121,7 +121,7 @@ test('should filter by project', async ({ runUITest }) => { await expect(page.getByLabel('bar')).not.toBeChecked(); await page.getByLabel('bar').setChecked(true); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ► ◯ passes ► ◯ fails @@ -134,7 +134,7 @@ test('should filter by project', async ({ runUITest }) => { await page.getByText('passes').first().click(); await page.keyboard.press('ArrowRight'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ▼ ◯ passes <= ◯ foo @@ -163,7 +163,7 @@ test('should not hide filtered while running', async ({ runUITest, createLatch } }); await page.getByTitle('Run all').click(); latch.open(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ✅ passes ❌ fails <= @@ -173,7 +173,7 @@ test('should not hide filtered while running', async ({ runUITest, createLatch } await page.getByText('Status:').click(); await page.getByLabel('failed').setChecked(true); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ↻ a.test.ts ↻ fails <= `); @@ -190,7 +190,7 @@ test('should filter skipped', async ({ runUITest, createLatch }) => { `, }); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ✅ a.test.ts ✅ passes ⊘ fails @@ -198,7 +198,7 @@ test('should filter skipped', async ({ runUITest, createLatch }) => { await page.getByText('Status:').click(); await page.getByLabel('skipped').setChecked(true); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ⊘ a.test.ts ⊘ fails `); diff --git a/tests/playwright-test/ui-mode-test-output.spec.ts b/tests/playwright-test/ui-mode-test-output.spec.ts index 870a41fa28..31d0fe3b3c 100644 --- a/tests/playwright-test/ui-mode-test-output.spec.ts +++ b/tests/playwright-test/ui-mode-test-output.spec.ts @@ -72,5 +72,5 @@ test('should print buffers', async ({ runUITest }) => { }); await page.getByTitle('Toggle output').click(); await page.getByTitle('Run all').click(); - await expect(page.getByTestId('output')).toContainText('HELLO', { timeout: 15000 }); + await expect(page.getByTestId('output')).toContainText('HELLO'); }); diff --git a/tests/playwright-test/ui-mode-test-progress.spec.ts b/tests/playwright-test/ui-mode-test-progress.spec.ts index eb2fce07c9..caf9ada3ef 100644 --- a/tests/playwright-test/ui-mode-test-progress.spec.ts +++ b/tests/playwright-test/ui-mode-test-progress.spec.ts @@ -52,7 +52,7 @@ test('should update trace live', async ({ runUITest, server }) => { ).toHaveText([ /browserContext.newPage[\d.]+m?s/, /page.gotohttp:\/\/localhost:\d+\/one.html/ - ], { timeout: 15000 }); + ]); await expect( listItem.locator(':scope.selected'), @@ -74,7 +74,7 @@ test('should update trace live', async ({ runUITest, server }) => { await expect( page.frameLocator('iframe.snapshot-visible[name=snapshot]').locator('body'), 'verify snapshot' - ).toHaveText('One', { timeout: 15000 }); + ).toHaveText('One'); await expect(listItem).toHaveText([ /browserContext.newPage[\d.]+m?s/, /page.gotohttp:\/\/localhost:\d+\/one.html[\d.]+m?s/, @@ -137,7 +137,7 @@ test('should preserve action list selection upon live trace update', async ({ ru /browserContext.newPage[\d.]+m?s/, /page.gotoabout:blank[\d.]+m?s/, /page.setContent[\d.]+m?s/, - ], { timeout: 15000 }); + ]); // Manually select page.goto. await page.getByTestId('action-list').getByText('page.goto').click(); @@ -198,7 +198,7 @@ test('should update tracing network live', async ({ runUITest, server }) => { /browserContext.newPage[\d.]+m?s/, /page.gotohttp:\/\/localhost:\d+\/one.html[\d.]+m?s/, /page.setContent[\d.]+m?s/, - ], { timeout: 15000 }); + ]); // Once page.setContent is visible, we can be sure that page.goto has all required // resources in the trace. Switch to it and check that everything renders. @@ -207,7 +207,7 @@ test('should update tracing network live', async ({ runUITest, server }) => { await expect( page.frameLocator('iframe.snapshot-visible[name=snapshot]').locator('body'), 'verify background' - ).toHaveCSS('background-color', 'rgb(255, 0, 0)', { timeout: 15000 }); + ).toHaveCSS('background-color', 'rgb(255, 0, 0)'); }); test('should show trace w/ multiple contexts', async ({ runUITest, server, createLatch }) => { @@ -238,7 +238,7 @@ test('should show trace w/ multiple contexts', async ({ runUITest, server, creat /apiRequestContext.get[\d.]+m?s/, /browserContext.newPage[\d.]+m?s/, /page.gotoabout:blank[\d.]+m?s/, - ], { timeout: 15000 }); + ]); latch.open(); }); diff --git a/tests/playwright-test/ui-mode-test-run.spec.ts b/tests/playwright-test/ui-mode-test-run.spec.ts index a86d8c6d01..286fada894 100644 --- a/tests/playwright-test/ui-mode-test-run.spec.ts +++ b/tests/playwright-test/ui-mode-test-run.spec.ts @@ -41,13 +41,13 @@ const basicTestTree = { test('should run visible', async ({ runUITest }) => { const { page } = await runUITest(basicTestTree); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts `); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ✅ passes ❌ fails <= @@ -75,9 +75,9 @@ test('should show running progress', async ({ runUITest }) => { }); await page.getByTitle('Run all').click(); - await expect(page.getByTestId('status-line')).toHaveText('Running 1/4 passed (25%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('Running 1/4 passed (25%)'); await page.getByTitle('Stop').click(); - await expect(page.getByTestId('status-line')).toHaveText('1/4 passed (25%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('1/4 passed (25%)'); await page.getByTitle('Reload').click(); await expect(page.getByTestId('status-line')).toBeHidden(); }); @@ -94,7 +94,7 @@ test('should run on hover', async ({ runUITest }) => { await page.getByText('passes').hover(); await page.getByRole('listitem').filter({ hasText: 'passes' }).getByTitle('Run').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ✅ passes <= ◯ fails @@ -112,7 +112,7 @@ test('should run on double click', async ({ runUITest }) => { await page.getByText('passes').dblclick(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ✅ passes <= ◯ fails @@ -131,7 +131,7 @@ test('should run on Enter', async ({ runUITest }) => { await page.getByText('fails').click(); await page.keyboard.press('Enter'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ◯ passes ❌ fails <= @@ -154,7 +154,7 @@ test('should run by project', async ({ runUITest }) => { await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ✅ passes ❌ fails <= @@ -170,7 +170,7 @@ test('should run by project', async ({ runUITest }) => { await page.getByText('Status:').click(); await page.getByLabel('bar').setChecked(true); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ► ◯ passes ► ❌ fails <= @@ -188,7 +188,7 @@ test('should run by project', async ({ runUITest }) => { await page.getByTestId('test-tree').getByText('passes').first().click(); await page.keyboard.press('ArrowRight'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ❌ a.test.ts ▼ ◯ passes <= ✅ foo @@ -199,7 +199,7 @@ test('should run by project', async ({ runUITest }) => { await expect(page.getByText('Projects: foo bar')).toBeVisible(); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ▼ ✅ passes ✅ foo @@ -228,12 +228,12 @@ test('should stop', async ({ runUITest }) => { `, }); - await expect(page.getByTitle('Run all')).toBeEnabled({ timeout: 15000 }); + await expect(page.getByTitle('Run all')).toBeEnabled(); await expect(page.getByTitle('Stop')).toBeDisabled(); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ↻ a.test.ts ⊘ test 0 ✅ test 1 @@ -246,7 +246,7 @@ test('should stop', async ({ runUITest }) => { await page.getByTitle('Stop').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ⊘ test 0 ✅ test 1 @@ -274,7 +274,7 @@ test('should run folder', async ({ runUITest }) => { await page.getByText('folder-b').hover(); await page.getByRole('listitem').filter({ hasText: 'folder-b' }).getByTitle('Run').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ✅ folder-b <= ► ✅ folder-c ► ✅ in-b.test.ts @@ -285,13 +285,13 @@ test('should run folder', async ({ runUITest }) => { test('should show time', async ({ runUITest }) => { const { page } = await runUITest(basicTestTree); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts `); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page, { time: true }), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page, { time: true })).toBe(` ▼ ❌ a.test.ts ✅ passes XXms ❌ fails XXms <= @@ -317,13 +317,13 @@ test('should show test.fail as passing', async ({ runUITest }) => { }); `, }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts `); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page, { time: true }), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page, { time: true })).toBe(` ▼ ✅ a.test.ts ✅ should fail XXms `); @@ -346,13 +346,13 @@ test('should ignore repeatEach', async ({ runUITest }) => { }); `, }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts `); await page.getByTitle('Run all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ✅ a.test.ts ✅ should pass `); diff --git a/tests/playwright-test/ui-mode-test-setup.spec.ts b/tests/playwright-test/ui-mode-test-setup.spec.ts index 52840b79b3..f5025a9488 100644 --- a/tests/playwright-test/ui-mode-test-setup.spec.ts +++ b/tests/playwright-test/ui-mode-test-setup.spec.ts @@ -39,9 +39,9 @@ test('should run global setup and teardown', async ({ runUITest }) => { ` }); await page.getByTitle('Run all').click(); - await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)'); await page.close(); - await expect.poll(() => testProcess.outputLines(), { timeout: 15000 }).toEqual([ + await expect.poll(() => testProcess.outputLines()).toEqual([ 'from-global-setup', 'from-global-teardown', ]); @@ -69,9 +69,9 @@ test('should teardown on sigint', async ({ runUITest }) => { ` }); await page.getByTitle('Run all').click(); - await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)'); testProcess.process.kill('SIGINT'); - await expect.poll(() => testProcess.outputLines(), { timeout: 15000 }).toEqual([ + await expect.poll(() => testProcess.outputLines()).toEqual([ 'from-global-setup', 'from-global-teardown', ]); diff --git a/tests/playwright-test/ui-mode-test-source.spec.ts b/tests/playwright-test/ui-mode-test-source.spec.ts index 4469db9356..f8b9d33f58 100644 --- a/tests/playwright-test/ui-mode-test-source.spec.ts +++ b/tests/playwright-test/ui-mode-test-source.spec.ts @@ -32,7 +32,7 @@ const basicTestTree = { test('should show selected test in sources', async ({ runUITest }) => { const { page } = await runUITest(basicTestTree); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ first ◯ second diff --git a/tests/playwright-test/ui-mode-test-tree.spec.ts b/tests/playwright-test/ui-mode-test-tree.spec.ts index f9e727e393..56256bad4a 100644 --- a/tests/playwright-test/ui-mode-test-tree.spec.ts +++ b/tests/playwright-test/ui-mode-test-tree.spec.ts @@ -37,7 +37,7 @@ const basicTestTree = { test('should list tests', async ({ runUITest }) => { const { page } = await runUITest(basicTestTree); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -51,7 +51,7 @@ test('should list tests', async ({ runUITest }) => { test('should traverse up/down', async ({ runUITest }) => { const { page } = await runUITest(basicTestTree); await page.getByText('a.test.ts').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts <= ◯ passes ◯ fails @@ -59,14 +59,14 @@ test('should traverse up/down', async ({ runUITest }) => { `); await page.keyboard.press('ArrowDown'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes <= ◯ fails ► ◯ suite `); await page.keyboard.press('ArrowDown'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes ◯ fails <= @@ -74,7 +74,7 @@ test('should traverse up/down', async ({ runUITest }) => { `); await page.keyboard.press('ArrowUp'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes <= ◯ fails @@ -87,7 +87,7 @@ test('should expand / collapse groups', async ({ runUITest }) => { await page.getByTestId('test-tree').getByText('suite').click(); await page.keyboard.press('ArrowRight'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -97,7 +97,7 @@ test('should expand / collapse groups', async ({ runUITest }) => { `); await page.keyboard.press('ArrowLeft'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -106,14 +106,14 @@ test('should expand / collapse groups', async ({ runUITest }) => { await page.getByTestId('test-tree').getByText('passes').first().click(); await page.keyboard.press('ArrowLeft'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts <= ◯ passes ◯ fails `); await page.keyboard.press('ArrowLeft'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ► ◯ a.test.ts <= `); }); @@ -134,7 +134,7 @@ test('should merge folder trees', async ({ runUITest }) => { `, }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ b ► ◯ c ► ◯ in-b.test.ts @@ -162,7 +162,7 @@ test('should list parametrized tests', async ({ runUITest }) => { await page.getByText('').click(); await page.keyboard.press('ArrowRight'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ▼ ◯ cookies ▼ ◯ <= @@ -191,7 +191,7 @@ test('should update parametrized tests', async ({ runUITest, writeFiles }) => { await page.getByText('').click(); await page.keyboard.press('ArrowRight'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ▼ ◯ cookies ▼ ◯ <= @@ -214,7 +214,7 @@ test('should update parametrized tests', async ({ runUITest, writeFiles }) => { }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ▼ ◯ cookies ▼ ◯ <= @@ -228,7 +228,7 @@ test('should collapse all', async ({ runUITest }) => { await page.getByTestId('test-tree').getByText('suite').click(); await page.keyboard.press('ArrowRight'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -238,7 +238,7 @@ test('should collapse all', async ({ runUITest }) => { `); await page.getByTitle('Collapse all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ► ◯ a.test.ts `); }); diff --git a/tests/playwright-test/ui-mode-test-update.spec.ts b/tests/playwright-test/ui-mode-test-update.spec.ts index 9b06d93a05..77f520aef8 100644 --- a/tests/playwright-test/ui-mode-test-update.spec.ts +++ b/tests/playwright-test/ui-mode-test-update.spec.ts @@ -37,7 +37,7 @@ const basicTestTree = { test('should pick new / deleted files', async ({ runUITest, writeFiles, deleteFile }) => { const { page } = await runUITest(basicTestTree); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -55,7 +55,7 @@ test('should pick new / deleted files', async ({ runUITest, writeFiles, deleteFi ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -70,7 +70,7 @@ test('should pick new / deleted files', async ({ runUITest, writeFiles, deleteFi await deleteFile('a.test.ts'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ b.test.ts ◯ passes ◯ fails @@ -82,7 +82,7 @@ test('should pick new / deleted files', async ({ runUITest, writeFiles, deleteFi test('should pick new / deleted tests', async ({ runUITest, writeFiles, deleteFile }) => { const { page } = await runUITest(basicTestTree); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -101,7 +101,7 @@ test('should pick new / deleted tests', async ({ runUITest, writeFiles, deleteFi ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ◯ new @@ -120,7 +120,7 @@ test('should pick new / deleted tests', async ({ runUITest, writeFiles, deleteFi ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ new ▼ ◯ b.test.ts @@ -131,7 +131,7 @@ test('should pick new / deleted tests', async ({ runUITest, writeFiles, deleteFi test('should pick new / deleted nested tests', async ({ runUITest, writeFiles, deleteFile }) => { const { page } = await runUITest(basicTestTree); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -140,7 +140,7 @@ test('should pick new / deleted nested tests', async ({ runUITest, writeFiles, d await page.getByText('suite').click(); await page.keyboard.press('ArrowRight'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes ◯ fails @@ -160,7 +160,7 @@ test('should pick new / deleted nested tests', async ({ runUITest, writeFiles, d ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes ▼ ◯ suite <= @@ -177,7 +177,7 @@ test('should update test locations', async ({ runUITest, writeFiles, deleteFile `, }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ passes `); @@ -205,7 +205,7 @@ test('should update test locations', async ({ runUITest, writeFiles, deleteFile ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toContain(` + await expect.poll(dumpTestTree(page)).toContain(` ▼ ◯ a.test.ts ◯ new-test ◯ passes <= diff --git a/tests/playwright-test/ui-mode-test-watch.spec.ts b/tests/playwright-test/ui-mode-test-watch.spec.ts index 1c7ff3076d..cca01d0b6d 100644 --- a/tests/playwright-test/ui-mode-test-watch.spec.ts +++ b/tests/playwright-test/ui-mode-test-watch.spec.ts @@ -29,7 +29,7 @@ test('should watch files', async ({ runUITest, writeFiles }) => { await page.getByText('fails').click(); await page.getByRole('listitem').filter({ hasText: 'fails' }).getByTitle('Watch').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ◯ fails 👁 <= @@ -37,7 +37,7 @@ test('should watch files', async ({ runUITest, writeFiles }) => { await page.getByRole('listitem').filter({ hasText: 'fails' }).getByTitle('Run').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ❌ a.test.ts ◯ passes ❌ fails 👁 <= @@ -51,7 +51,7 @@ test('should watch files', async ({ runUITest, writeFiles }) => { ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ passes ✅ fails 👁 <= @@ -76,7 +76,7 @@ test('should watch e2e deps', async ({ runUITest, writeFiles }) => { await page.getByText('answer').click(); await page.getByRole('listitem').filter({ hasText: 'answer' }).getByTitle('Watch').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ answer 👁 <= `); @@ -87,7 +87,7 @@ test('should watch e2e deps', async ({ runUITest, writeFiles }) => { ` }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ✅ a.test.ts ✅ answer 👁 <= `); @@ -110,7 +110,7 @@ test('should batch watch updates', async ({ runUITest, writeFiles }) => { await page.getByText('d.test.ts').click(); await page.getByRole('listitem').filter({ hasText: 'd.test.ts' }).getByTitle('Watch').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts 👁 ◯ test ▼ ◯ b.test.ts 👁 @@ -128,9 +128,9 @@ test('should batch watch updates', async ({ runUITest, writeFiles }) => { 'd.test.ts': `import { test } from '@playwright/test'; test('test', () => {});`, }); - await expect(page.getByTestId('status-line')).toHaveText('4/4 passed (100%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('4/4 passed (100%)'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ✅ a.test.ts 👁 ✅ test ▼ ✅ b.test.ts 👁 @@ -150,7 +150,7 @@ test('should watch all', async ({ runUITest, writeFiles }) => { 'd.test.ts': `import { test } from '@playwright/test'; test('test', () => {});`, }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ test ▼ ◯ b.test.ts @@ -167,9 +167,9 @@ test('should watch all', async ({ runUITest, writeFiles }) => { 'd.test.ts': `import { test } from '@playwright/test'; test('test', () => {});`, }); - await expect(page.getByTestId('status-line')).toHaveText('2/2 passed (100%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('2/2 passed (100%)'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ✅ a.test.ts ✅ test ▼ ◯ b.test.ts @@ -188,7 +188,7 @@ test('should watch new file', async ({ runUITest, writeFiles }) => { await page.getByTitle('Watch all').click(); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ test `); @@ -198,7 +198,7 @@ test('should watch new file', async ({ runUITest, writeFiles }) => { 'b.test.ts': ` import { test } from '@playwright/test'; test('test', () => {});`, }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ test ▼ ◯ b.test.ts @@ -210,9 +210,9 @@ test('should watch new file', async ({ runUITest, writeFiles }) => { 'b.test.ts': ` import { test } from '@playwright/test'; test('test', () => {});`, }); - await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('1/1 passed (100%)'); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ test ▼ ✅ b.test.ts @@ -231,7 +231,7 @@ test('should queue watches', async ({ runUITest, writeFiles, createLatch }) => { 'd.test.ts': `import { test } from '@playwright/test'; test('test', () => {});`, }); - await expect.poll(dumpTestTree(page), { timeout: 15000 }).toBe(` + await expect.poll(dumpTestTree(page)).toBe(` ▼ ◯ a.test.ts ◯ test ▼ ◯ b.test.ts @@ -245,7 +245,7 @@ test('should queue watches', async ({ runUITest, writeFiles, createLatch }) => { await page.getByTitle('Watch all').click(); await page.getByTitle('Run all').click(); - await expect(page.getByTestId('status-line')).toHaveText('Running 1/4 passed (25%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('Running 1/4 passed (25%)'); await writeFiles({ 'a.test.ts': `import { test } from '@playwright/test'; test('test', () => {});`, @@ -255,10 +255,10 @@ test('should queue watches', async ({ runUITest, writeFiles, createLatch }) => { // Now watches should not kick in. await new Promise(f => setTimeout(f, 1000)); - await expect(page.getByTestId('status-line')).toHaveText('Running 1/4 passed (25%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('Running 1/4 passed (25%)'); // Allow test to finish and new watch to kick in. latch.open(); - await expect(page.getByTestId('status-line')).toHaveText('3/3 passed (100%)', { timeout: 15000 }); + await expect(page.getByTestId('status-line')).toHaveText('3/3 passed (100%)'); }); diff --git a/tests/playwright-test/ui-mode-trace.spec.ts b/tests/playwright-test/ui-mode-trace.spec.ts index e4f5a1766b..91ba32b003 100644 --- a/tests/playwright-test/ui-mode-trace.spec.ts +++ b/tests/playwright-test/ui-mode-trace.spec.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import { createImage } from './playwright-test-fixtures'; import { test, expect } from './ui-mode-fixtures'; test.describe.configure({ mode: 'parallel' }); @@ -42,7 +43,7 @@ test('should merge trace events', async ({ runUITest, server }) => { /expect\.toBe[\d.]+m?s/, /locator\.clickgetByRole\('button'\)[\d.]+m?s/, /expect\.toBe[\d.]+m?s/, - ], { timeout: 15000 }); + ]); }); test('should merge web assertion events', async ({ runUITest }, testInfo) => { @@ -66,7 +67,7 @@ test('should merge web assertion events', async ({ runUITest }, testInfo) => { /browserContext\.newPage[\d.]+m?s/, /page\.setContent[\d.]+m?s/, /expect\.toBeVisiblelocator\('button'\)[\d.]+m?s/, - ], { timeout: 15000 }); + ]); }); test('should merge screenshot assertions', async ({ runUITest }, testInfo) => { @@ -90,7 +91,7 @@ test('should merge screenshot assertions', async ({ runUITest }, testInfo) => { /browserContext\.newPage[\d.]+m?s/, /page\.setContent[\d.]+m?s/, /expect\.toHaveScreenshot[\d.]+m?s/, - ], { timeout: 15000 }); + ]); }); test('should locate sync assertions in source', async ({ runUITest, server }) => { @@ -108,7 +109,7 @@ test('should locate sync assertions in source', async ({ runUITest, server }) => await expect( page.locator('.CodeMirror .source-line-running'), 'check source tab', - ).toHaveText('4 expect(1).toBe(1);', { timeout: 15000 }); + ).toHaveText('4 expect(1).toBe(1);'); }); test('should show snapshots for sync assertions', async ({ runUITest, server }) => { @@ -134,10 +135,35 @@ test('should show snapshots for sync assertions', async ({ runUITest, server }) /page\.setContent[\d.]+m?s/, /locator\.clickgetByRole\('button'\)[\d.]+m?s/, /expect\.toBe[\d.]+m?s/, - ], { timeout: 15000 }); + ]); await expect( page.frameLocator('iframe.snapshot-visible[name=snapshot]').locator('button'), 'verify snapshot' ).toHaveText('Submit'); }); + +test('should show image diff', async ({ runUITest, server }) => { + const { page } = await runUITest({ + 'playwright.config.js': ` + module.exports = { + snapshotPathTemplate: '{arg}{ext}' + }; + `, + 'snapshot.png': createImage(100, 100, 255, 0, 0), + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('vrt test', async ({ page }) => { + await page.setViewportSize({ width: 100, height: 100 }); + await expect(page).toHaveScreenshot('snapshot.png', { timeout: 2000 }); + }); + `, + }); + + await page.getByText('vrt test').dblclick(); + await page.getByText(/Log/).click(); + await expect(page.getByText('Diff', { exact: true })).toBeVisible(); + await expect(page.getByText('Actual', { exact: true })).toBeVisible(); + await expect(page.getByText('Expected', { exact: true })).toBeVisible(); + await expect(page.locator('.image-diff-view .image-wrapper img')).toBeVisible(); +});