mirror of
https://github.com/microsoft/playwright.git
synced 2025-06-26 21:40:17 +00:00
120aaa777e
Unhandled error/rejection interrupts current test and produces a TimeoutError for it that should be ignored.
524 lines
14 KiB
TypeScript
524 lines
14 KiB
TypeScript
/**
|
||
* Copyright (c) Microsoft Corporation.
|
||
*
|
||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||
* you may not use this file except in compliance with the License.
|
||
* You may obtain a copy of the License at
|
||
*
|
||
* http://www.apache.org/licenses/LICENSE-2.0
|
||
*
|
||
* Unless required by applicable law or agreed to in writing, software
|
||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
* See the License for the specific language governing permissions and
|
||
* limitations under the License.
|
||
*/
|
||
|
||
import { test, expect } from './playwright-test-fixtures';
|
||
|
||
test('should fail', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'one-failure.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('fails', () => {
|
||
expect(1 + 1).toBe(7);
|
||
});
|
||
`
|
||
});
|
||
expect(result.exitCode).toBe(1);
|
||
expect(result.passed).toBe(0);
|
||
expect(result.failed).toBe(1);
|
||
expect(result.output).toContain('1) one-failure.spec.ts:3');
|
||
});
|
||
|
||
test('should timeout', async ({ runInlineTest }) => {
|
||
const { exitCode, passed, failed, output } = await runInlineTest({
|
||
'one-timeout.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('timeout', async () => {
|
||
await new Promise(f => setTimeout(f, 10000));
|
||
});
|
||
`
|
||
}, { timeout: 100 });
|
||
expect(exitCode).toBe(1);
|
||
expect(passed).toBe(0);
|
||
expect(failed).toBe(1);
|
||
expect(output).toContain('Test timeout of 100ms exceeded.');
|
||
});
|
||
|
||
test('should succeed', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'one-success.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('succeeds', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
`
|
||
});
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.passed).toBe(1);
|
||
expect(result.failed).toBe(0);
|
||
});
|
||
|
||
test('should report suite errors', async ({ runInlineTest }) => {
|
||
const { exitCode, failed, output } = await runInlineTest({
|
||
'suite-error.spec.ts': `
|
||
if (new Error().stack.includes('workerMain'))
|
||
throw new Error('Suite error');
|
||
|
||
import { test, expect } from '@playwright/test';
|
||
test('passes',() => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
`
|
||
});
|
||
expect(exitCode).toBe(1);
|
||
expect(failed).toBe(1);
|
||
expect(output).toContain('Suite error');
|
||
});
|
||
|
||
test('should respect nested skip', async ({ runInlineTest }) => {
|
||
const { exitCode, passed, failed, skipped } = await runInlineTest({
|
||
'nested-skip.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test.describe('skipped', () => {
|
||
test.skip();
|
||
test('succeeds',() => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
});
|
||
`
|
||
});
|
||
expect(exitCode).toBe(0);
|
||
expect(passed).toBe(0);
|
||
expect(failed).toBe(0);
|
||
expect(skipped).toBe(1);
|
||
});
|
||
|
||
test('should respect excluded tests', async ({ runInlineTest }) => {
|
||
const { exitCode, passed } = await runInlineTest({
|
||
'excluded.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('included test', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
|
||
test('excluded test 1', () => {
|
||
test.skip();
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
|
||
test('excluded test 2', () => {
|
||
test.skip();
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
|
||
test.describe('included describe', () => {
|
||
test('included describe test', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
});
|
||
|
||
test.describe('excluded describe', () => {
|
||
test.skip();
|
||
test('excluded describe test', () => {
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
});
|
||
`,
|
||
});
|
||
expect(passed).toBe(2);
|
||
expect(exitCode).toBe(0);
|
||
});
|
||
|
||
test('should respect focused tests', async ({ runInlineTest }) => {
|
||
const { exitCode, passed } = await runInlineTest({
|
||
'focused.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('included test', () => {
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
|
||
test.only('focused test', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
|
||
test.only('focused only test', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
|
||
test.describe.only('focused describe', () => {
|
||
test('describe test', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
});
|
||
|
||
test.describe('non-focused describe', () => {
|
||
test('describe test', () => {
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
});
|
||
|
||
test.describe.only('focused describe', () => {
|
||
test('test1', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
test.only('test2', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
test('test3', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
test.only('test4', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
});
|
||
`
|
||
});
|
||
expect(passed).toBe(5);
|
||
expect(exitCode).toBe(0);
|
||
});
|
||
|
||
test('skip should take priority over fail', async ({ runInlineTest }) => {
|
||
const { passed, skipped, failed } = await runInlineTest({
|
||
'test.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test.describe('failing suite', () => {
|
||
test.fail();
|
||
|
||
test('skipped', () => {
|
||
test.skip();
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
|
||
test('passing', () => {
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
test('passing2', () => {
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
|
||
test('failing', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
});
|
||
`
|
||
});
|
||
expect(passed).toBe(2);
|
||
expect(skipped).toBe(1);
|
||
expect(failed).toBe(1);
|
||
});
|
||
|
||
test('should focus test from one project', async ({ runInlineTest }) => {
|
||
const { exitCode, passed, skipped, failed } = await runInlineTest({
|
||
'playwright.config.ts': `
|
||
import * as path from 'path';
|
||
module.exports = { projects: [
|
||
{ testDir: path.join(__dirname, 'a') },
|
||
{ testDir: path.join(__dirname, 'b') },
|
||
] };
|
||
`,
|
||
'a/afile.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('just a test', () => {
|
||
expect(1 + 1).toBe(3);
|
||
});
|
||
`,
|
||
'b/bfile.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test.only('focused test', () => {
|
||
expect(1 + 1).toBe(2);
|
||
});
|
||
`,
|
||
}, { reporter: 'list,json' });
|
||
expect(passed).toBe(1);
|
||
expect(failed).toBe(0);
|
||
expect(skipped).toBe(0);
|
||
expect(exitCode).toBe(0);
|
||
});
|
||
|
||
test('should work with default export', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'file.spec.ts': `
|
||
import t from '@playwright/test';
|
||
t('passed', () => {
|
||
t.expect(1 + 1).toBe(2);
|
||
});
|
||
`
|
||
});
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.passed).toBe(1);
|
||
expect(result.failed).toBe(0);
|
||
});
|
||
|
||
test('should work with test wrapper', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'helper.js': `
|
||
const { test, expect } = require('@playwright/test');
|
||
console.log('%%helper');
|
||
exports.wrap = (title, fn) => {
|
||
test(title, fn);
|
||
};
|
||
`,
|
||
'a.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
console.log('%%a.spec');
|
||
const { wrap } = require('./helper');
|
||
wrap('test1', () => {
|
||
console.log('%%test1');
|
||
});
|
||
test.describe('suite1', () => {
|
||
wrap('suite1.test1', () => {
|
||
console.log('%%suite1.test1');
|
||
});
|
||
});
|
||
`,
|
||
'b.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
console.log('%%b.spec');
|
||
const { wrap } = require('./helper');
|
||
wrap('test2', () => {
|
||
console.log('%%test2');
|
||
});
|
||
test.describe('suite2', () => {
|
||
wrap('suite2.test2', () => {
|
||
console.log('%%suite2.test2');
|
||
});
|
||
});
|
||
`,
|
||
}, { workers: 1, reporter: 'line' });
|
||
expect(result.passed).toBe(4);
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.outputLines).toEqual([
|
||
'a.spec',
|
||
'helper',
|
||
'b.spec',
|
||
'a.spec',
|
||
'helper',
|
||
'test1',
|
||
'suite1.test1',
|
||
'b.spec',
|
||
'test2',
|
||
'suite2.test2',
|
||
]);
|
||
});
|
||
|
||
test('should work with test helper', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'helper-a.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
console.log('%%helper-a');
|
||
test('test1', () => {
|
||
console.log('%%test1');
|
||
});
|
||
test.describe('suite1', () => {
|
||
test('suite1.test1', () => {
|
||
console.log('%%suite1.test1');
|
||
});
|
||
});
|
||
`,
|
||
'a.spec.ts': `
|
||
console.log('%%a.spec');
|
||
require('./helper-a');
|
||
`,
|
||
'helper-b.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
console.log('%%helper-b');
|
||
test('test1', () => {
|
||
console.log('%%test2');
|
||
});
|
||
test.describe('suite2', () => {
|
||
test('suite2.test2', () => {
|
||
console.log('%%suite2.test2');
|
||
});
|
||
});
|
||
`,
|
||
'b.spec.ts': `
|
||
console.log('%%b.spec');
|
||
require('./helper-b');
|
||
`,
|
||
}, { workers: 1, reporter: 'line' });
|
||
expect(result.passed).toBe(4);
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.outputLines).toEqual([
|
||
'a.spec',
|
||
'helper-a',
|
||
'b.spec',
|
||
'helper-b',
|
||
'a.spec',
|
||
'helper-a',
|
||
'test1',
|
||
'suite1.test1',
|
||
'b.spec',
|
||
'helper-b',
|
||
'test2',
|
||
'suite2.test2',
|
||
]);
|
||
});
|
||
|
||
test('should support describe() without a title', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'a.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test.describe('suite1', () => {
|
||
test.describe(() => {
|
||
test.describe('suite2', () => {
|
||
test('my test', () => {});
|
||
});
|
||
});
|
||
});
|
||
`,
|
||
}, { reporter: 'list' });
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.passed).toBe(1);
|
||
expect(result.output).toContain('a.spec.ts:6:17 › suite1 › suite2 › my test');
|
||
});
|
||
|
||
test('test.{skip,fixme} should define a skipped test', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'a.test.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
const logs = [];
|
||
test.skip('foo', () => {
|
||
console.log('%%dontseethis');
|
||
throw new Error('foo');
|
||
});
|
||
test.fixme('bar', () => {
|
||
console.log('%%dontseethis');
|
||
throw new Error('bar');
|
||
});
|
||
`,
|
||
});
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.skipped).toBe(2);
|
||
expect(result.output).not.toContain('%%dontseethis');
|
||
});
|
||
|
||
test('should report unhandled error during test and not report timeout', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'a.test.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('unhandled rejection', async () => {
|
||
setTimeout(() => {
|
||
throw new Error('Unhandled');
|
||
}, 0);
|
||
await new Promise(f => setTimeout(f, 100));
|
||
});
|
||
`,
|
||
});
|
||
expect(result.exitCode).toBe(1);
|
||
expect(result.failed).toBe(1);
|
||
expect(result.output).toContain('Error: Unhandled');
|
||
expect(result.output).not.toContain('Test timeout of 30000ms exceeded');
|
||
});
|
||
|
||
test('should report unhandled rejection during worker shutdown', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'a.test.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('unhandled rejection', async () => {
|
||
new Promise((f, r) => r(new Error('Unhandled')));
|
||
});
|
||
`,
|
||
});
|
||
expect(result.exitCode).toBe(1);
|
||
expect(result.passed).toBe(1);
|
||
expect(result.output).toContain('Error: Unhandled');
|
||
expect(result.output).toContain('a.test.ts:4:33');
|
||
});
|
||
|
||
test('should not reuse worker after unhandled rejection in test.fail', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'a.spec.ts': `
|
||
import { test as base, expect } from '@playwright/test';
|
||
const test = base.extend({
|
||
needsCleanup: async ({}, use) => {
|
||
await use();
|
||
await new Promise(f => setTimeout(f, 3000));
|
||
}
|
||
});
|
||
|
||
test('failing', async ({ needsCleanup }) => {
|
||
test.fail();
|
||
new Promise(() => { throw new Error('Oh my!') });
|
||
});
|
||
|
||
test('passing', async () => {
|
||
});
|
||
`
|
||
}, { workers: 1 });
|
||
expect(result.exitCode).toBe(1);
|
||
expect(result.failed).toBe(1);
|
||
expect(result.interrupted).toBe(1);
|
||
expect(result.output).toContain(`Error: Oh my!`);
|
||
expect(result.output).not.toContain(`Did not teardown test scope`);
|
||
});
|
||
|
||
test('should allow unhandled expects in test.fail', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'a.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test('failing1', async ({}) => {
|
||
test.fail();
|
||
Promise.resolve().then(() => expect(1).toBe(2));
|
||
await new Promise(f => setTimeout(f, 100));
|
||
});
|
||
`
|
||
});
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.passed).toBe(1);
|
||
expect(result.output).not.toContain(`Error: expect`);
|
||
});
|
||
|
||
test('should support describe.skip', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'nested-skip.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test.describe.skip('skipped', () => {
|
||
test.describe('nested', () => {
|
||
test('test1', () => {});
|
||
});
|
||
test('test2', () => {});
|
||
});
|
||
test.describe('not skipped', () => {
|
||
test.describe.skip('skipped', () => {
|
||
test('test4', () => {});
|
||
});
|
||
test('test4', () => {
|
||
console.log('heytest4');
|
||
});
|
||
});
|
||
`
|
||
});
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.passed).toBe(1);
|
||
expect(result.skipped).toBe(3);
|
||
expect(result.output).toContain('heytest4');
|
||
});
|
||
|
||
test('should support describe.fixme', async ({ runInlineTest }) => {
|
||
const result = await runInlineTest({
|
||
'nested-skip.spec.ts': `
|
||
import { test, expect } from '@playwright/test';
|
||
test.describe.fixme('skipped', () => {
|
||
test.describe('nested', () => {
|
||
test('test1', () => {});
|
||
});
|
||
test('test2', () => {});
|
||
});
|
||
test.describe('not skipped', () => {
|
||
test.describe.fixme('skipped', () => {
|
||
test('test4', () => {});
|
||
});
|
||
test('test4', () => {
|
||
console.log('heytest4');
|
||
});
|
||
});
|
||
`
|
||
});
|
||
expect(result.exitCode).toBe(0);
|
||
expect(result.passed).toBe(1);
|
||
expect(result.skipped).toBe(3);
|
||
expect(result.output).toContain('heytest4');
|
||
});
|