2021-09-13 17:49:58 -07:00
|
|
|
/**
|
|
|
|
* Copyright (c) Microsoft Corporation.
|
|
|
|
*
|
|
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
* you may not use this file except in compliance with the License.
|
|
|
|
* You may obtain a copy of the License at
|
|
|
|
*
|
|
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
*
|
|
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
* See the License for the specific language governing permissions and
|
|
|
|
* limitations under the License.
|
|
|
|
*/
|
|
|
|
|
2024-04-13 20:07:18 -07:00
|
|
|
import { test, expect, stripAnsi } from './playwright-test-fixtures';
|
2021-09-13 17:49:58 -07:00
|
|
|
|
2024-04-13 20:07:18 -07:00
|
|
|
const stepIndentReporter = `
|
2024-09-18 16:57:11 +02:00
|
|
|
import { FullConfig, Location, Reporter, Suite, TestStep } from '@playwright/test/reporter';
|
2024-04-13 20:07:18 -07:00
|
|
|
import * as path from 'path';
|
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
function formatPrefix(str: string) {
|
2024-04-13 20:07:18 -07:00
|
|
|
return str.padEnd(10, ' ') + '|';
|
2023-09-29 09:44:00 -07:00
|
|
|
}
|
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
function formatLocation(location?: Location) {
|
|
|
|
if (!location)
|
|
|
|
throw new Error('Location is missing');
|
2024-04-13 20:07:18 -07:00
|
|
|
return ' @ ' + path.basename(location.file) + ':' + location.line;
|
|
|
|
}
|
2021-09-13 17:49:58 -07:00
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
function formatStack(indent: string, rawStack: string) {
|
|
|
|
let stack = rawStack.split('\\n').filter(s => s.startsWith(' at '));
|
2024-04-13 20:07:18 -07:00
|
|
|
stack = stack.map(s => {
|
|
|
|
const match = /^( at.* )\\(?([^ )]+)\\)?/.exec(s);
|
2024-09-18 16:57:11 +02:00
|
|
|
let location = match![2];
|
2024-04-13 20:07:18 -07:00
|
|
|
location = location.substring(location.lastIndexOf(path.sep) + 1);
|
|
|
|
return ' at ' + location;
|
|
|
|
});
|
|
|
|
return indent + stack.join('\\n' + indent);
|
|
|
|
}
|
2023-12-22 12:00:17 -08:00
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
export default class MyReporter implements Reporter {
|
2024-04-13 20:07:18 -07:00
|
|
|
printErrorLocation: boolean;
|
2024-04-22 15:30:51 -07:00
|
|
|
skipErrorMessage: boolean;
|
2024-09-18 16:57:11 +02:00
|
|
|
suite!: Suite;
|
2024-04-22 15:30:51 -07:00
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
constructor(options: { printErrorLocation: boolean, skipErrorMessage: boolean }) {
|
2024-04-13 20:07:18 -07:00
|
|
|
this.printErrorLocation = options.printErrorLocation;
|
2024-04-22 15:30:51 -07:00
|
|
|
this.skipErrorMessage = options.skipErrorMessage;
|
|
|
|
}
|
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
trimError(message: string) {
|
2024-04-22 15:30:51 -07:00
|
|
|
if (this.skipErrorMessage)
|
|
|
|
return '<error message>';
|
|
|
|
const lines = message.split('\\n');
|
|
|
|
return lines[0];
|
2024-04-13 20:07:18 -07:00
|
|
|
}
|
2024-04-22 15:30:51 -07:00
|
|
|
|
2024-04-13 20:07:18 -07:00
|
|
|
onBegin(config: FullConfig, suite: Suite) {
|
|
|
|
this.suite = suite;
|
2023-04-19 17:31:07 -07:00
|
|
|
}
|
|
|
|
|
2024-04-16 12:51:20 -07:00
|
|
|
// For easier debugging.
|
2024-09-18 16:57:11 +02:00
|
|
|
onStdOut(data: string|Buffer) {
|
2024-04-16 12:51:20 -07:00
|
|
|
process.stdout.write(data.toString());
|
|
|
|
}
|
|
|
|
// For easier debugging.
|
2024-09-18 16:57:11 +02:00
|
|
|
onStdErr(data: string|Buffer) {
|
2024-04-16 12:51:20 -07:00
|
|
|
process.stderr.write(data.toString());
|
|
|
|
}
|
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
printStep(step: TestStep, indent: string) {
|
2024-04-13 20:07:18 -07:00
|
|
|
let location = '';
|
|
|
|
if (step.location)
|
|
|
|
location = formatLocation(step.location);
|
|
|
|
console.log(formatPrefix(step.category) + indent + step.title + location);
|
|
|
|
if (step.error) {
|
|
|
|
const errorLocation = this.printErrorLocation ? formatLocation(step.error.location) : '';
|
2024-09-18 16:57:11 +02:00
|
|
|
console.log(formatPrefix(step.category) + indent + '↪ error: ' + this.trimError(step.error.message!) + errorLocation);
|
2024-04-13 20:07:18 -07:00
|
|
|
if (this.printErrorLocation)
|
2024-09-18 16:57:11 +02:00
|
|
|
console.log(formatStack(formatPrefix(step.category) + indent, step.error.stack!));
|
2024-04-13 20:07:18 -07:00
|
|
|
}
|
|
|
|
indent += ' ';
|
|
|
|
for (const child of step.steps)
|
|
|
|
this.printStep(child, indent);
|
2023-04-19 17:31:07 -07:00
|
|
|
}
|
|
|
|
|
2021-09-16 15:51:27 -07:00
|
|
|
async onEnd() {
|
2024-04-13 20:07:18 -07:00
|
|
|
console.log(); // for nicer expectations
|
2021-09-16 15:51:27 -07:00
|
|
|
const processSuite = (suite: Suite) => {
|
|
|
|
for (const child of suite.suites)
|
|
|
|
processSuite(child);
|
|
|
|
for (const test of suite.tests) {
|
|
|
|
for (const result of test.results) {
|
2024-04-13 20:07:18 -07:00
|
|
|
for (const step of result.steps)
|
|
|
|
this.printStep(step, '');
|
2023-12-22 12:00:17 -08:00
|
|
|
for (const error of result.errors) {
|
2024-04-13 20:07:18 -07:00
|
|
|
const errorLocation = this.printErrorLocation ? formatLocation(error.location) : '';
|
2024-09-18 16:57:11 +02:00
|
|
|
console.log(formatPrefix('') + this.trimError(error.message!) + errorLocation);
|
2024-04-13 20:07:18 -07:00
|
|
|
if (this.printErrorLocation)
|
2024-09-18 16:57:11 +02:00
|
|
|
console.log(formatStack(formatPrefix(''), error.stack!));
|
2023-12-22 12:00:17 -08:00
|
|
|
}
|
2021-09-16 15:51:27 -07:00
|
|
|
}
|
2021-09-13 17:49:58 -07:00
|
|
|
}
|
2021-09-16 15:51:27 -07:00
|
|
|
};
|
|
|
|
processSuite(this.suite);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
`;
|
2021-09-13 17:49:58 -07:00
|
|
|
|
2021-09-16 15:51:27 -07:00
|
|
|
test('should report api step hierarchy', async ({ runInlineTest }) => {
|
2021-09-13 17:49:58 -07:00
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2021-09-13 17:49:58 -07:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
2023-02-14 19:20:56 -08:00
|
|
|
import { test, expect } from '@playwright/test';
|
2021-09-13 17:49:58 -07:00
|
|
|
test('pass', async ({ page }) => {
|
|
|
|
await test.step('outer step 1', async () => {
|
|
|
|
await test.step('inner step 1.1', async () => {});
|
|
|
|
await test.step('inner step 1.2', async () => {});
|
|
|
|
});
|
|
|
|
await test.step('outer step 2', async () => {
|
|
|
|
await test.step('inner step 2.1', async () => {});
|
|
|
|
await test.step('inner step 2.2', async () => {});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(result.output).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
test.step |outer step 1 @ a.test.ts:4
|
|
|
|
test.step | inner step 1.1 @ a.test.ts:5
|
|
|
|
test.step | inner step 1.2 @ a.test.ts:6
|
|
|
|
test.step |outer step 2 @ a.test.ts:8
|
|
|
|
test.step | inner step 2.1 @ a.test.ts:9
|
|
|
|
test.step | inner step 2.2 @ a.test.ts:10
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
2021-09-13 17:49:58 -07:00
|
|
|
});
|
|
|
|
|
2023-04-19 17:31:07 -07:00
|
|
|
test('should report before hooks step error', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-04-19 17:31:07 -07:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test.beforeEach(async ({}) => {
|
|
|
|
throw new Error('oh my');
|
|
|
|
});
|
|
|
|
test('pass', async ({}) => {
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(result.output).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
hook |↪ error: Error: oh my
|
|
|
|
hook | beforeEach hook @ a.test.ts:3
|
|
|
|
hook | ↪ error: Error: oh my
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: oh my
|
|
|
|
`);
|
2023-04-19 17:31:07 -07:00
|
|
|
});
|
|
|
|
|
2021-09-16 15:51:27 -07:00
|
|
|
test('should not report nested after hooks', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2021-09-16 15:51:27 -07:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
2023-02-14 19:20:56 -08:00
|
|
|
import { test, expect } from '@playwright/test';
|
2021-09-16 15:51:27 -07:00
|
|
|
test('timeout', async ({ page }) => {
|
|
|
|
await test.step('my step', async () => {
|
|
|
|
await new Promise(() => {});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1, timeout: 2000 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
test.step |my step @ a.test.ts:4
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
hook |Worker Cleanup
|
|
|
|
fixture | fixture: browser
|
|
|
|
|Test timeout of 2000ms exceeded.
|
|
|
|
`);
|
2021-09-16 15:51:27 -07:00
|
|
|
});
|
|
|
|
|
2021-09-13 17:49:58 -07:00
|
|
|
test('should report test.step from fixtures', async ({ runInlineTest }) => {
|
|
|
|
const expectReporterJS = `
|
|
|
|
class Reporter {
|
|
|
|
onStepBegin(test, result, step) {
|
|
|
|
console.log('%% begin ' + step.title);
|
|
|
|
}
|
|
|
|
onStepEnd(test, result, step) {
|
|
|
|
console.log('%% end ' + step.title);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
module.exports = Reporter;
|
|
|
|
`;
|
|
|
|
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': expectReporterJS,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
2023-02-14 19:20:56 -08:00
|
|
|
import { test as base, expect } from '@playwright/test';
|
|
|
|
const test = base.extend({
|
2021-09-13 17:49:58 -07:00
|
|
|
foo: async ({}, use) => {
|
2023-02-14 19:20:56 -08:00
|
|
|
await base.step('setup foo', () => {});
|
2021-09-13 17:49:58 -07:00
|
|
|
await use(async () => {
|
|
|
|
await test.step('inside foo', () => {});
|
|
|
|
});
|
|
|
|
await test.step('teardown foo', () => {});
|
|
|
|
},
|
|
|
|
});
|
|
|
|
test('pass', async ({ foo }) => {
|
|
|
|
await test.step('test step', async () => {
|
|
|
|
await foo();
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2023-02-07 15:11:44 -08:00
|
|
|
expect(result.outputLines).toEqual([
|
|
|
|
`begin Before Hooks`,
|
2023-05-06 10:25:32 -07:00
|
|
|
`begin fixture: foo`,
|
2023-02-07 15:11:44 -08:00
|
|
|
`begin setup foo`,
|
|
|
|
`end setup foo`,
|
2023-05-06 10:25:32 -07:00
|
|
|
`end fixture: foo`,
|
2023-02-07 15:11:44 -08:00
|
|
|
`end Before Hooks`,
|
|
|
|
`begin test step`,
|
|
|
|
`begin inside foo`,
|
|
|
|
`end inside foo`,
|
|
|
|
`end test step`,
|
|
|
|
`begin After Hooks`,
|
2023-05-06 10:25:32 -07:00
|
|
|
`begin fixture: foo`,
|
2023-02-07 15:11:44 -08:00
|
|
|
`begin teardown foo`,
|
|
|
|
`end teardown foo`,
|
2023-05-06 10:25:32 -07:00
|
|
|
`end fixture: foo`,
|
2023-02-07 15:11:44 -08:00
|
|
|
`end After Hooks`,
|
2021-09-13 17:49:58 -07:00
|
|
|
]);
|
|
|
|
});
|
2021-10-18 20:06:18 -08:00
|
|
|
|
|
|
|
test('should report expect step locations', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2021-10-18 20:06:18 -08:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
2023-02-14 19:20:56 -08:00
|
|
|
import { test, expect } from '@playwright/test';
|
2024-04-13 20:07:18 -07:00
|
|
|
test('pass', async ({}) => {
|
2021-10-18 20:06:18 -08:00
|
|
|
expect(true).toBeTruthy();
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(result.output).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
expect |expect.toBeTruthy @ a.test.ts:4
|
|
|
|
hook |After Hooks
|
|
|
|
`);
|
2021-10-18 20:06:18 -08:00
|
|
|
});
|
2022-04-05 16:47:35 -08:00
|
|
|
|
|
|
|
test('should report custom expect steps', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2022-04-05 16:47:35 -08:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
2024-04-13 20:07:18 -07:00
|
|
|
reporter: [['./reporter']],
|
2022-04-05 16:47:35 -08:00
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
2024-09-12 19:56:38 +02:00
|
|
|
import { test, expect as baseExpect } from '@playwright/test';
|
|
|
|
|
|
|
|
const expect = baseExpect.extend({
|
2022-04-05 16:47:35 -08:00
|
|
|
toBeWithinRange(received, floor, ceiling) {
|
|
|
|
const pass = received >= floor && received <= ceiling;
|
|
|
|
if (pass) {
|
|
|
|
return {
|
|
|
|
message: () =>
|
|
|
|
"expected " + received + " not to be within range " + floor + " - " + ceiling,
|
|
|
|
pass: true,
|
|
|
|
};
|
|
|
|
} else {
|
|
|
|
return {
|
|
|
|
message: () =>
|
|
|
|
"expected " + received + " to be within range " + floor + " - " + ceiling,
|
|
|
|
pass: false,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
},
|
2023-05-15 19:37:12 -07:00
|
|
|
|
|
|
|
async toBeFailingAsync(received) {
|
|
|
|
await new Promise(f => setTimeout(f, 0));
|
|
|
|
return {
|
|
|
|
message: () => "It fails!",
|
|
|
|
pass: false,
|
|
|
|
};
|
|
|
|
},
|
2022-04-05 16:47:35 -08:00
|
|
|
});
|
|
|
|
|
2023-05-15 19:37:12 -07:00
|
|
|
test('fail', async ({}) => {
|
2022-04-05 16:47:35 -08:00
|
|
|
expect(15).toBeWithinRange(10, 20);
|
2023-05-15 19:37:12 -07:00
|
|
|
await expect(1).toBeFailingAsync(22);
|
2022-04-05 16:47:35 -08:00
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
2023-05-15 19:37:12 -07:00
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(result.output).toBe(`
|
|
|
|
hook |Before Hooks
|
2024-09-12 19:56:38 +02:00
|
|
|
expect |expect.toBeWithinRange @ a.test.ts:32
|
|
|
|
expect |expect.toBeFailingAsync @ a.test.ts:33
|
2024-04-13 20:07:18 -07:00
|
|
|
expect |↪ error: Error: It fails!
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: It fails!
|
|
|
|
`);
|
2022-04-05 16:47:35 -08:00
|
|
|
});
|
2022-07-29 15:16:07 -07:00
|
|
|
|
2023-04-19 17:31:07 -07:00
|
|
|
test('should not pass arguments and return value from step', async ({ runInlineTest }) => {
|
2022-07-29 15:16:07 -07:00
|
|
|
const result = await runInlineTest({
|
|
|
|
'a.test.ts': `
|
2023-02-14 19:20:56 -08:00
|
|
|
import { test, expect } from '@playwright/test';
|
2022-07-29 15:16:07 -07:00
|
|
|
test('steps with return values', async ({ page }) => {
|
2023-04-19 17:31:07 -07:00
|
|
|
const v1 = await test.step('my step', (...args) => {
|
|
|
|
expect(args.length).toBe(0);
|
2022-07-29 15:16:07 -07:00
|
|
|
return 10;
|
|
|
|
});
|
|
|
|
console.log('v1 = ' + v1);
|
2023-04-19 17:31:07 -07:00
|
|
|
const v2 = await test.step('my step', async (...args) => {
|
|
|
|
expect(args.length).toBe(0);
|
2022-07-29 15:16:07 -07:00
|
|
|
return new Promise(f => setTimeout(() => f(v1 + 10), 100));
|
|
|
|
});
|
|
|
|
console.log('v2 = ' + v2);
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
|
|
expect(result.passed).toBe(1);
|
|
|
|
expect(result.output).toContain('v1 = 10');
|
|
|
|
expect(result.output).toContain('v2 = 20');
|
|
|
|
});
|
2023-01-09 16:17:06 -08:00
|
|
|
|
|
|
|
test('should mark step as failed when soft expect fails', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-01-09 16:17:06 -08:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
2023-02-14 19:20:56 -08:00
|
|
|
import { test, expect } from '@playwright/test';
|
2023-01-09 16:17:06 -08:00
|
|
|
test('pass', async ({}) => {
|
|
|
|
await test.step('outer', async () => {
|
|
|
|
await test.step('inner', async () => {
|
|
|
|
expect.soft(1).toBe(2);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
await test.step('passing', () => {});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |outer @ a.test.ts:4
|
|
|
|
test.step |↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
test.step | inner @ a.test.ts:5
|
|
|
|
test.step | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.soft.toBe @ a.test.ts:6
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
test.step |passing @ a.test.ts:9
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
`);
|
2023-01-09 16:17:06 -08:00
|
|
|
});
|
2023-03-30 21:05:07 -07:00
|
|
|
|
|
|
|
test('should nest steps based on zones', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-03-30 21:05:07 -07:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test.beforeAll(async () => {
|
|
|
|
await test.step('in beforeAll', () => {});
|
|
|
|
});
|
2023-04-19 17:31:07 -07:00
|
|
|
|
2023-03-30 21:05:07 -07:00
|
|
|
test.afterAll(async () => {
|
|
|
|
await test.step('in afterAll', () => {});
|
|
|
|
});
|
2023-04-19 17:31:07 -07:00
|
|
|
|
2023-03-30 21:05:07 -07:00
|
|
|
test.beforeEach(async () => {
|
|
|
|
await test.step('in beforeEach', () => {});
|
|
|
|
});
|
2023-04-19 17:31:07 -07:00
|
|
|
|
2023-03-30 21:05:07 -07:00
|
|
|
test.afterEach(async () => {
|
|
|
|
await test.step('in afterEach', () => {});
|
|
|
|
});
|
2023-04-19 17:31:07 -07:00
|
|
|
|
2023-03-30 21:05:07 -07:00
|
|
|
test.only('foo', async ({ page }) => {
|
|
|
|
await test.step('grand', async () => {
|
|
|
|
await Promise.all([
|
|
|
|
test.step('parent1', async () => {
|
|
|
|
await test.step('child1', async () => {
|
|
|
|
await page.click('body');
|
|
|
|
});
|
|
|
|
}),
|
|
|
|
test.step('parent2', async () => {
|
|
|
|
await test.step('child2', async () => {
|
|
|
|
await expect(page.locator('body')).toBeVisible();
|
|
|
|
});
|
|
|
|
}),
|
|
|
|
]);
|
|
|
|
});
|
2023-04-19 17:31:07 -07:00
|
|
|
});
|
2023-03-30 21:05:07 -07:00
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(result.output).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
hook | beforeAll hook @ a.test.ts:3
|
|
|
|
test.step | in beforeAll @ a.test.ts:4
|
|
|
|
hook | beforeEach hook @ a.test.ts:11
|
|
|
|
test.step | in beforeEach @ a.test.ts:12
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
test.step |grand @ a.test.ts:20
|
|
|
|
test.step | parent1 @ a.test.ts:22
|
|
|
|
test.step | child1 @ a.test.ts:23
|
|
|
|
pw:api | page.click(body) @ a.test.ts:24
|
|
|
|
test.step | parent2 @ a.test.ts:27
|
|
|
|
test.step | child2 @ a.test.ts:28
|
|
|
|
expect | expect.toBeVisible @ a.test.ts:29
|
|
|
|
hook |After Hooks
|
|
|
|
hook | afterEach hook @ a.test.ts:15
|
|
|
|
test.step | in afterEach @ a.test.ts:16
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
hook | afterAll hook @ a.test.ts:7
|
|
|
|
test.step | in afterAll @ a.test.ts:8
|
|
|
|
`);
|
2023-03-30 21:05:07 -07:00
|
|
|
});
|
2023-05-02 18:50:00 -07:00
|
|
|
|
|
|
|
test('should not mark page.close as failed when page.click fails', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-05-02 18:50:00 -07:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
let page: Page;
|
|
|
|
|
|
|
|
test.beforeAll(async ({ browser }) => {
|
|
|
|
page = await browser.newPage();
|
|
|
|
});
|
|
|
|
|
|
|
|
test.afterAll(async () => {
|
|
|
|
await page.close();
|
|
|
|
});
|
|
|
|
|
|
|
|
test('fails', async () => {
|
|
|
|
test.setTimeout(2000);
|
|
|
|
await page.setContent('hello');
|
|
|
|
await page.click('div');
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
hook | beforeAll hook @ a.test.ts:5
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
pw:api | browser.newPage @ a.test.ts:6
|
|
|
|
pw:api |page.setContent @ a.test.ts:15
|
|
|
|
pw:api |page.click(div) @ a.test.ts:16
|
|
|
|
pw:api |↪ error: Error: page.click: Target page, context or browser has been closed
|
|
|
|
hook |After Hooks
|
|
|
|
hook | afterAll hook @ a.test.ts:9
|
|
|
|
pw:api | page.close @ a.test.ts:10
|
|
|
|
hook |Worker Cleanup
|
|
|
|
fixture | fixture: browser
|
|
|
|
|Test timeout of 2000ms exceeded.
|
|
|
|
|Error: page.click: Target page, context or browser has been closed
|
|
|
|
`);
|
2023-05-02 18:50:00 -07:00
|
|
|
});
|
2023-05-03 16:04:20 -07:00
|
|
|
|
2023-06-01 13:22:08 -07:00
|
|
|
test('should not propagate errors from within toPass', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-06-01 13:22:08 -07:00
|
|
|
'playwright.config.ts': `module.exports = { reporter: './reporter', };`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async () => {
|
|
|
|
let i = 0;
|
|
|
|
await expect(() => {
|
|
|
|
expect(i++).toBe(2);
|
|
|
|
}).toPass();
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(result.output).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
expect |expect.toPass @ a.test.ts:7
|
|
|
|
expect | expect.toBe @ a.test.ts:6
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.toBe @ a.test.ts:6
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.toBe @ a.test.ts:6
|
|
|
|
hook |After Hooks
|
|
|
|
`);
|
2023-06-01 13:22:08 -07:00
|
|
|
});
|
|
|
|
|
|
|
|
test('should show final toPass error', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-06-01 13:22:08 -07:00
|
|
|
'playwright.config.ts': `module.exports = { reporter: './reporter', };`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('fail', async () => {
|
|
|
|
await expect(() => {
|
|
|
|
expect(true).toBe(false);
|
|
|
|
}).toPass({ timeout: 1 });
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
expect |expect.toPass @ a.test.ts:6
|
|
|
|
expect |↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.toBe @ a.test.ts:5
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
`);
|
2023-06-01 13:22:08 -07:00
|
|
|
});
|
2023-07-05 15:30:53 -07:00
|
|
|
|
|
|
|
test('should propagate nested soft errors', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-07-05 15:30:53 -07:00
|
|
|
'playwright.config.ts': `module.exports = { reporter: './reporter', };`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('fail', async () => {
|
|
|
|
await test.step('first outer', async () => {
|
|
|
|
await test.step('first inner', async () => {
|
|
|
|
expect.soft(1).toBe(2);
|
|
|
|
});
|
|
|
|
});
|
2024-02-15 09:45:48 -08:00
|
|
|
|
2023-07-05 15:30:53 -07:00
|
|
|
await test.step('second outer', async () => {
|
|
|
|
await test.step('second inner', async () => {
|
|
|
|
expect(1).toBe(2);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |first outer @ a.test.ts:4
|
|
|
|
test.step |↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
test.step | first inner @ a.test.ts:5
|
|
|
|
test.step | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.soft.toBe @ a.test.ts:6
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
test.step |second outer @ a.test.ts:10
|
|
|
|
test.step |↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
test.step | second inner @ a.test.ts:11
|
|
|
|
test.step | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.toBe @ a.test.ts:12
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
`);
|
2023-07-05 15:30:53 -07:00
|
|
|
});
|
|
|
|
|
|
|
|
test('should not propagate nested hard errors', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-07-05 15:30:53 -07:00
|
|
|
'playwright.config.ts': `module.exports = { reporter: './reporter', };`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('fail', async () => {
|
|
|
|
await test.step('first outer', async () => {
|
|
|
|
await test.step('first inner', async () => {
|
|
|
|
try {
|
|
|
|
expect(1).toBe(2);
|
|
|
|
} catch (e) {
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
2024-02-15 09:45:48 -08:00
|
|
|
|
2023-07-05 15:30:53 -07:00
|
|
|
await test.step('second outer', async () => {
|
|
|
|
await test.step('second inner', async () => {
|
|
|
|
expect(1).toBe(2);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |first outer @ a.test.ts:4
|
|
|
|
test.step | first inner @ a.test.ts:5
|
|
|
|
expect | expect.toBe @ a.test.ts:7
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
test.step |second outer @ a.test.ts:13
|
|
|
|
test.step |↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
test.step | second inner @ a.test.ts:14
|
|
|
|
test.step | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.toBe @ a.test.ts:15
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
`);
|
2023-07-05 15:30:53 -07:00
|
|
|
});
|
2023-09-29 09:44:00 -07:00
|
|
|
|
|
|
|
test('should step w/o box', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printErrorLocation: true }]], };`,
|
2023-09-29 09:44:00 -07:00
|
|
|
'a.test.ts':
|
|
|
|
` /*1*/ import { test, expect } from '@playwright/test';
|
|
|
|
/*2*/ test('fail', async () => {
|
|
|
|
/*3*/ await test.step('boxed step', async () => {
|
|
|
|
/*4*/ expect(1).toBe(2);
|
|
|
|
/*5*/ });
|
|
|
|
/*6*/ });
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |boxed step @ a.test.ts:3
|
|
|
|
test.step |↪ error: Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:4
|
|
|
|
test.step | at a.test.ts:4:27
|
|
|
|
test.step | at a.test.ts:3:26
|
|
|
|
expect | expect.toBe @ a.test.ts:4
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:4
|
|
|
|
expect | at a.test.ts:4:27
|
|
|
|
expect | at a.test.ts:3:26
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:4
|
|
|
|
| at a.test.ts:4:27
|
|
|
|
| at a.test.ts:3:26
|
|
|
|
`);
|
2023-09-29 09:44:00 -07:00
|
|
|
});
|
|
|
|
|
|
|
|
test('should step w/ box', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printErrorLocation: true }]], };`,
|
2023-09-29 09:44:00 -07:00
|
|
|
'a.test.ts':
|
|
|
|
` /*1*/ import { test, expect } from '@playwright/test';
|
|
|
|
/*2*/ test('fail', async () => {
|
|
|
|
/*3*/ const helper = async () => {
|
|
|
|
/*4*/ await test.step('boxed step', async () => {
|
2023-12-21 12:16:03 -08:00
|
|
|
/*5*/ expect(1).toBe(2);
|
|
|
|
/*6*/ }, { box: true });
|
2023-09-29 09:44:00 -07:00
|
|
|
/*7*/ };
|
|
|
|
/*8*/ await helper();
|
|
|
|
/*9*/ });
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |boxed step @ a.test.ts:8
|
|
|
|
test.step |↪ error: Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:8
|
|
|
|
test.step | at a.test.ts:8:21
|
|
|
|
expect | expect.toBe @ a.test.ts:5
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:8
|
|
|
|
expect | at a.test.ts:8:21
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:8
|
|
|
|
| at a.test.ts:8:21
|
|
|
|
`);
|
2023-12-21 12:16:03 -08:00
|
|
|
});
|
|
|
|
|
|
|
|
test('should soft step w/ box', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printErrorLocation: true }]], };`,
|
2023-12-21 12:16:03 -08:00
|
|
|
'a.test.ts':
|
|
|
|
` /*1*/ import { test, expect } from '@playwright/test';
|
|
|
|
/*2*/ test('fail', async () => {
|
|
|
|
/*3*/ const helper = async () => {
|
|
|
|
/*4*/ await test.step('boxed step', async () => {
|
|
|
|
/*5*/ expect.soft(1).toBe(2);
|
|
|
|
/*6*/ }, { box: true });
|
|
|
|
/*7*/ };
|
|
|
|
/*8*/ await helper();
|
|
|
|
/*9*/ });
|
|
|
|
`
|
|
|
|
}, { reporter: '' });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |boxed step @ a.test.ts:8
|
|
|
|
test.step |↪ error: Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:8
|
|
|
|
test.step | at a.test.ts:8:21
|
|
|
|
expect | expect.soft.toBe @ a.test.ts:5
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:8
|
|
|
|
expect | at a.test.ts:8:21
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBe(expected) // Object.is equality @ a.test.ts:8
|
|
|
|
| at a.test.ts:8:21
|
|
|
|
`);
|
2023-09-29 09:44:00 -07:00
|
|
|
});
|
2023-12-13 09:06:02 -08:00
|
|
|
|
|
|
|
test('should not generate dupes for named expects', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
2024-04-13 20:07:18 -07:00
|
|
|
'reporter.ts': stepIndentReporter,
|
2023-12-13 09:06:02 -08:00
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('timeout', async ({ page }) => {
|
|
|
|
await page.setContent('<div style="background:rgb(1,2,3)">hi</div>');
|
|
|
|
await expect(page.locator('div'), 'Checking color')
|
|
|
|
.toHaveCSS('background-color', 'rgb(1, 2, 3)');
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1, timeout: 2000 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(result.output).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
pw:api |page.setContent @ a.test.ts:4
|
|
|
|
expect |Checking color @ a.test.ts:6
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
2024-04-12 14:51:10 -07:00
|
|
|
});
|
|
|
|
|
|
|
|
test('step inside expect.toPass', async ({ runInlineTest }) => {
|
|
|
|
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/30322' });
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async ({}) => {
|
|
|
|
await test.step('step 1', async () => {
|
|
|
|
let counter = 0
|
|
|
|
await expect(async () => {
|
|
|
|
await test.step('step 2, attempt: ' + counter, async () => {
|
|
|
|
counter++;
|
|
|
|
expect(counter).toBe(2);
|
|
|
|
});
|
|
|
|
}).toPass();
|
|
|
|
await test.step('step 3', async () => {
|
|
|
|
await test.step('step 4', async () => {
|
|
|
|
expect(1).toBe(1);
|
|
|
|
});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |step 1 @ a.test.ts:4
|
|
|
|
expect | expect.toPass @ a.test.ts:11
|
|
|
|
test.step | step 2, attempt: 0 @ a.test.ts:7
|
|
|
|
test.step | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.toBe @ a.test.ts:9
|
2024-04-16 16:18:37 -07:00
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
2024-04-13 20:07:18 -07:00
|
|
|
test.step | step 2, attempt: 1 @ a.test.ts:7
|
|
|
|
expect | expect.toBe @ a.test.ts:9
|
|
|
|
test.step | step 3 @ a.test.ts:12
|
|
|
|
test.step | step 4 @ a.test.ts:13
|
|
|
|
expect | expect.toBe @ a.test.ts:14
|
|
|
|
hook |After Hooks
|
2024-04-12 14:51:10 -07:00
|
|
|
`);
|
|
|
|
});
|
|
|
|
|
|
|
|
test('library API call inside expect.toPass', async ({ runInlineTest }) => {
|
|
|
|
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/30322' });
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async ({page}) => {
|
|
|
|
let counter = 0
|
|
|
|
await expect(async () => {
|
|
|
|
await page.goto('about:blank');
|
|
|
|
await test.step('inner step attempt: ' + counter, async () => {
|
|
|
|
counter++;
|
|
|
|
expect(counter).toBe(2);
|
|
|
|
});
|
|
|
|
}).toPass();
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
2024-04-13 20:07:18 -07:00
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
expect |expect.toPass @ a.test.ts:11
|
|
|
|
pw:api | page.goto(about:blank) @ a.test.ts:6
|
|
|
|
test.step | inner step attempt: 0 @ a.test.ts:7
|
|
|
|
test.step | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
expect | expect.toBe @ a.test.ts:9
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
|
|
|
pw:api | page.goto(about:blank) @ a.test.ts:6
|
|
|
|
test.step | inner step attempt: 1 @ a.test.ts:7
|
|
|
|
expect | expect.toBe @ a.test.ts:9
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
2024-04-12 14:51:10 -07:00
|
|
|
`);
|
2024-04-16 16:18:37 -07:00
|
|
|
});
|
|
|
|
|
|
|
|
test('library API call inside expect.poll', async ({ runInlineTest }) => {
|
|
|
|
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/30322' });
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async ({page}) => {
|
|
|
|
let counter = 0
|
|
|
|
const a = [];
|
|
|
|
await expect.poll(async () => {
|
|
|
|
await page.goto('about:blank');
|
|
|
|
await test.step('inner step attempt: ' + counter, async () => {
|
|
|
|
counter++;
|
|
|
|
expect(1).toBe(1);
|
|
|
|
});
|
|
|
|
a.push(1);
|
|
|
|
return a;
|
|
|
|
}).toHaveLength(2);
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
expect |expect.poll.toHaveLength @ a.test.ts:14
|
|
|
|
pw:api | page.goto(about:blank) @ a.test.ts:7
|
|
|
|
test.step | inner step attempt: 0 @ a.test.ts:8
|
|
|
|
expect | expect.toBe @ a.test.ts:10
|
2024-09-16 00:10:06 -07:00
|
|
|
expect | expect.toHaveLength @ a.test.ts:6
|
|
|
|
expect | ↪ error: Error: expect(received).toHaveLength(expected)
|
2024-04-16 16:18:37 -07:00
|
|
|
pw:api | page.goto(about:blank) @ a.test.ts:7
|
|
|
|
test.step | inner step attempt: 1 @ a.test.ts:8
|
|
|
|
expect | expect.toBe @ a.test.ts:10
|
2024-09-16 00:10:06 -07:00
|
|
|
expect | expect.toHaveLength @ a.test.ts:6
|
2024-04-16 16:18:37 -07:00
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
|
|
|
});
|
2024-04-16 17:37:25 -07:00
|
|
|
|
|
|
|
test('web assertion inside expect.poll', async ({ runInlineTest }) => {
|
|
|
|
test.info().annotations.push({ type: 'issue', description: 'https://github.com/microsoft/playwright/issues/30322' });
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async ({ page }) => {
|
|
|
|
await page.setContent('<div>foo</div>');
|
|
|
|
let counter = 0
|
|
|
|
await expect.poll(async () => {
|
|
|
|
await expect(page.locator('div')).toHaveText('foo');
|
|
|
|
++counter;
|
|
|
|
await test.step('iteration ' + counter, async () => {
|
|
|
|
await expect(page.locator('div')).toBeVisible();
|
|
|
|
});
|
|
|
|
return counter;
|
|
|
|
}).toBe(2);
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
pw:api |page.setContent @ a.test.ts:4
|
|
|
|
expect |expect.poll.toBe @ a.test.ts:13
|
|
|
|
expect | expect.toHaveText @ a.test.ts:7
|
|
|
|
test.step | iteration 1 @ a.test.ts:9
|
|
|
|
expect | expect.toBeVisible @ a.test.ts:10
|
2024-09-16 00:10:06 -07:00
|
|
|
expect | expect.toBe @ a.test.ts:6
|
|
|
|
expect | ↪ error: Error: expect(received).toBe(expected) // Object.is equality
|
2024-04-16 17:37:25 -07:00
|
|
|
expect | expect.toHaveText @ a.test.ts:7
|
|
|
|
test.step | iteration 2 @ a.test.ts:9
|
|
|
|
expect | expect.toBeVisible @ a.test.ts:10
|
2024-09-16 00:10:06 -07:00
|
|
|
expect | expect.toBe @ a.test.ts:6
|
2024-04-16 17:37:25 -07:00
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
|
|
|
});
|
2024-04-22 15:30:51 -07:00
|
|
|
|
|
|
|
test('should report expect steps', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('fail', async ({}) => {
|
|
|
|
expect(true).toBeTruthy();
|
|
|
|
expect(false).toBeTruthy();
|
|
|
|
});
|
|
|
|
test('pass', async ({}) => {
|
|
|
|
expect(false).not.toBeTruthy();
|
|
|
|
});
|
|
|
|
test('async', async ({ page }) => {
|
|
|
|
await expect(page).not.toHaveTitle('False');
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
expect |expect.toBeTruthy @ a.test.ts:4
|
|
|
|
expect |expect.toBeTruthy @ a.test.ts:5
|
|
|
|
expect |↪ error: Error: expect(received).toBeTruthy()
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Worker Cleanup
|
|
|
|
|Error: expect(received).toBeTruthy()
|
|
|
|
hook |Before Hooks
|
|
|
|
expect |expect.not.toBeTruthy @ a.test.ts:8
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
expect |expect.not.toHaveTitle @ a.test.ts:11
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
|
|
|
});
|
|
|
|
|
|
|
|
test('should report api steps', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `module.exports = { reporter: [['./reporter', { skipErrorMessage: true }]] };`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async ({ page, request }) => {
|
|
|
|
await Promise.all([
|
|
|
|
page.waitForNavigation(),
|
|
|
|
page.goto('data:text/html,<button></button>'),
|
|
|
|
]);
|
|
|
|
await page.click('button');
|
|
|
|
await page.getByRole('button').click();
|
|
|
|
await page.request.get('http://localhost2').catch(() => {});
|
|
|
|
await request.get('http://localhost2').catch(() => {});
|
|
|
|
});
|
|
|
|
|
|
|
|
test.describe('suite', () => {
|
|
|
|
let myPage;
|
|
|
|
test.beforeAll(async ({ browser }) => {
|
|
|
|
myPage = await browser.newPage();
|
|
|
|
await myPage.setContent('<button></button>');
|
|
|
|
});
|
|
|
|
|
|
|
|
test('pass1', async () => {
|
|
|
|
await myPage.click('button');
|
|
|
|
});
|
|
|
|
test('pass2', async () => {
|
|
|
|
await myPage.click('button');
|
|
|
|
});
|
|
|
|
|
|
|
|
test.afterAll(async () => {
|
|
|
|
await myPage.close();
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
hook | beforeAll hook @ a.test.ts:16
|
|
|
|
pw:api | browser.newPage @ a.test.ts:17
|
|
|
|
pw:api | page.setContent @ a.test.ts:18
|
|
|
|
pw:api |page.click(button) @ a.test.ts:22
|
|
|
|
hook |After Hooks
|
|
|
|
hook |Before Hooks
|
|
|
|
pw:api |page.click(button) @ a.test.ts:25
|
|
|
|
hook |After Hooks
|
|
|
|
hook | afterAll hook @ a.test.ts:28
|
|
|
|
pw:api | page.close @ a.test.ts:29
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
fixture | fixture: request
|
|
|
|
pw:api | apiRequest.newContext
|
|
|
|
pw:api |page.waitForNavigation @ a.test.ts:5
|
2024-08-05 10:06:14 -07:00
|
|
|
pw:api |page.goto(data:text/html,<button></button>) @ a.test.ts:6
|
2024-04-22 15:30:51 -07:00
|
|
|
pw:api |page.click(button) @ a.test.ts:8
|
|
|
|
pw:api |locator.getByRole('button').click @ a.test.ts:9
|
|
|
|
pw:api |apiRequestContext.get(http://localhost2) @ a.test.ts:10
|
|
|
|
pw:api |↪ error: <error message>
|
|
|
|
pw:api |apiRequestContext.get(http://localhost2) @ a.test.ts:11
|
|
|
|
pw:api |↪ error: <error message>
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: request
|
|
|
|
pw:api | apiRequestContext.dispose
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
|
|
|
});
|
|
|
|
|
|
|
|
test('should report api step failure', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('fail', async ({ page }) => {
|
|
|
|
await page.setContent('<button></button>');
|
|
|
|
await page.click('input', { timeout: 1 });
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(1);
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
pw:api |page.setContent @ a.test.ts:4
|
|
|
|
pw:api |page.click(input) @ a.test.ts:5
|
|
|
|
pw:api |↪ error: TimeoutError: page.click: Timeout 1ms exceeded.
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
hook |Worker Cleanup
|
|
|
|
fixture | fixture: browser
|
|
|
|
|TimeoutError: page.click: Timeout 1ms exceeded.
|
|
|
|
`);
|
|
|
|
});
|
|
|
|
|
|
|
|
test('should show nice stacks for locators', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printErrorLocation: true }]] };`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async ({ page }) => {
|
|
|
|
await page.setContent('<button></button>');
|
|
|
|
const locator = page.locator('button');
|
|
|
|
await locator.evaluate(e => e.innerText);
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
|
|
expect(result.passed).toBe(0);
|
|
|
|
expect(result.output).not.toContain('Internal error');
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
pw:api |page.setContent @ a.test.ts:4
|
|
|
|
pw:api |locator.evaluate(button) @ a.test.ts:6
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
|
|
|
});
|
feat(test runner): allow to pass arbitrary location to test.step (#32504)
Fixes https://github.com/microsoft/playwright/issues/30160
### Description:
This pull request introduces the ability to specify custom locations for
test steps in Playwright. By enabling the provision of arbitrary
locations to the test.step method, it resolves the limitation where
helper methods obfuscate the original call site, providing more accurate
and meaningful location data in test reports.
### Motivation:
To enhance the utility and clarity of test reports in Playwright.
Specifically, it addresses the need to trace test steps back to their
precise location in the code, which is especially important when steps
are abstracted in helper functions. This feature is crucial for
maintaining accurate documentation and facilitating debugging processes.
### Changes:
Added functionality to pass a custom location object to test.step.
### Expected Outcome:
This PR is expected to significantly improve the precision and
usefulness of diagnostic data in test reports by allowing specific
locations within helper functions to be accurately documented. It
facilitates better tracking of test executions and simplifies the
debugging process, making it easier for developers to understand and
address issues within complex tests.
### References:
Closes https://github.com/microsoft/playwright/issues/30160 -
"[Feature]: allow to pass arbitrary location to test.step"
**Code Check**
I conducted tests on this new feature by integrating it into some
existing test codes, and it worked well. I will attach the code used for
testing and a screenshot showing the successful outcome.
<details>
<summary>toggle dropdown</summary>
<div markdown="1">
```
import type { Location } from '../../../packages/playwright/types/testReporter'
...
test('should respect the back button', async ({ page }) => {
await page.locator('.todo-list li .toggle').nth(1).check();
await checkNumberOfCompletedTodosInLocalStorage(page, 1);
...
await test.step('Showing active items', async () => {
await page.getByRole('link', { name: 'Active' }).click();
}, {location});
```
<img width="1109" alt="image"
src="https://github.com/user-attachments/assets/359feafa-0949-4c71-9426-46debef21bdd">
</div>
</details>
2024-09-18 00:11:21 +09:00
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
test('should allow passing location to test.step', async ({ runInlineTest, runTSC }) => {
|
feat(test runner): allow to pass arbitrary location to test.step (#32504)
Fixes https://github.com/microsoft/playwright/issues/30160
### Description:
This pull request introduces the ability to specify custom locations for
test steps in Playwright. By enabling the provision of arbitrary
locations to the test.step method, it resolves the limitation where
helper methods obfuscate the original call site, providing more accurate
and meaningful location data in test reports.
### Motivation:
To enhance the utility and clarity of test reports in Playwright.
Specifically, it addresses the need to trace test steps back to their
precise location in the code, which is especially important when steps
are abstracted in helper functions. This feature is crucial for
maintaining accurate documentation and facilitating debugging processes.
### Changes:
Added functionality to pass a custom location object to test.step.
### Expected Outcome:
This PR is expected to significantly improve the precision and
usefulness of diagnostic data in test reports by allowing specific
locations within helper functions to be accurately documented. It
facilitates better tracking of test executions and simplifies the
debugging process, making it easier for developers to understand and
address issues within complex tests.
### References:
Closes https://github.com/microsoft/playwright/issues/30160 -
"[Feature]: allow to pass arbitrary location to test.step"
**Code Check**
I conducted tests on this new feature by integrating it into some
existing test codes, and it worked well. I will attach the code used for
testing and a screenshot showing the successful outcome.
<details>
<summary>toggle dropdown</summary>
<div markdown="1">
```
import type { Location } from '../../../packages/playwright/types/testReporter'
...
test('should respect the back button', async ({ page }) => {
await page.locator('.todo-list li .toggle').nth(1).check();
await checkNumberOfCompletedTodosInLocalStorage(page, 1);
...
await test.step('Showing active items', async () => {
await page.getByRole('link', { name: 'Active' }).click();
}, {location});
```
<img width="1109" alt="image"
src="https://github.com/user-attachments/assets/359feafa-0949-4c71-9426-46debef21bdd">
</div>
</details>
2024-09-18 00:11:21 +09:00
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'helper.ts': `
|
2024-09-18 16:57:11 +02:00
|
|
|
import { Location, TestType } from '@playwright/test';
|
feat(test runner): allow to pass arbitrary location to test.step (#32504)
Fixes https://github.com/microsoft/playwright/issues/30160
### Description:
This pull request introduces the ability to specify custom locations for
test steps in Playwright. By enabling the provision of arbitrary
locations to the test.step method, it resolves the limitation where
helper methods obfuscate the original call site, providing more accurate
and meaningful location data in test reports.
### Motivation:
To enhance the utility and clarity of test reports in Playwright.
Specifically, it addresses the need to trace test steps back to their
precise location in the code, which is especially important when steps
are abstracted in helper functions. This feature is crucial for
maintaining accurate documentation and facilitating debugging processes.
### Changes:
Added functionality to pass a custom location object to test.step.
### Expected Outcome:
This PR is expected to significantly improve the precision and
usefulness of diagnostic data in test reports by allowing specific
locations within helper functions to be accurately documented. It
facilitates better tracking of test executions and simplifies the
debugging process, making it easier for developers to understand and
address issues within complex tests.
### References:
Closes https://github.com/microsoft/playwright/issues/30160 -
"[Feature]: allow to pass arbitrary location to test.step"
**Code Check**
I conducted tests on this new feature by integrating it into some
existing test codes, and it worked well. I will attach the code used for
testing and a screenshot showing the successful outcome.
<details>
<summary>toggle dropdown</summary>
<div markdown="1">
```
import type { Location } from '../../../packages/playwright/types/testReporter'
...
test('should respect the back button', async ({ page }) => {
await page.locator('.todo-list li .toggle').nth(1).check();
await checkNumberOfCompletedTodosInLocalStorage(page, 1);
...
await test.step('Showing active items', async () => {
await page.getByRole('link', { name: 'Active' }).click();
}, {location});
```
<img width="1109" alt="image"
src="https://github.com/user-attachments/assets/359feafa-0949-4c71-9426-46debef21bdd">
</div>
</details>
2024-09-18 00:11:21 +09:00
|
|
|
|
2024-09-18 16:57:11 +02:00
|
|
|
export async function dummyStep(test: TestType<{}, {}>, title: string, action: () => void, location: Location) {
|
|
|
|
await test.step(title, action, { location });
|
feat(test runner): allow to pass arbitrary location to test.step (#32504)
Fixes https://github.com/microsoft/playwright/issues/30160
### Description:
This pull request introduces the ability to specify custom locations for
test steps in Playwright. By enabling the provision of arbitrary
locations to the test.step method, it resolves the limitation where
helper methods obfuscate the original call site, providing more accurate
and meaningful location data in test reports.
### Motivation:
To enhance the utility and clarity of test reports in Playwright.
Specifically, it addresses the need to trace test steps back to their
precise location in the code, which is especially important when steps
are abstracted in helper functions. This feature is crucial for
maintaining accurate documentation and facilitating debugging processes.
### Changes:
Added functionality to pass a custom location object to test.step.
### Expected Outcome:
This PR is expected to significantly improve the precision and
usefulness of diagnostic data in test reports by allowing specific
locations within helper functions to be accurately documented. It
facilitates better tracking of test executions and simplifies the
debugging process, making it easier for developers to understand and
address issues within complex tests.
### References:
Closes https://github.com/microsoft/playwright/issues/30160 -
"[Feature]: allow to pass arbitrary location to test.step"
**Code Check**
I conducted tests on this new feature by integrating it into some
existing test codes, and it worked well. I will attach the code used for
testing and a screenshot showing the successful outcome.
<details>
<summary>toggle dropdown</summary>
<div markdown="1">
```
import type { Location } from '../../../packages/playwright/types/testReporter'
...
test('should respect the back button', async ({ page }) => {
await page.locator('.todo-list li .toggle').nth(1).check();
await checkNumberOfCompletedTodosInLocalStorage(page, 1);
...
await test.step('Showing active items', async () => {
await page.getByRole('link', { name: 'Active' }).click();
}, {location});
```
<img width="1109" alt="image"
src="https://github.com/user-attachments/assets/359feafa-0949-4c71-9426-46debef21bdd">
</div>
</details>
2024-09-18 00:11:21 +09:00
|
|
|
}
|
|
|
|
|
|
|
|
export function getCustomLocation() {
|
|
|
|
return { file: 'dummy-file.ts', line: 123, column: 45 };
|
|
|
|
}
|
|
|
|
`,
|
|
|
|
'playwright.config.ts': `
|
|
|
|
module.exports = {
|
|
|
|
reporter: './reporter',
|
|
|
|
};
|
|
|
|
`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test } from '@playwright/test';
|
|
|
|
import { dummyStep, getCustomLocation } from './helper';
|
|
|
|
|
|
|
|
test('custom location test', async () => {
|
|
|
|
const location = getCustomLocation();
|
2024-09-18 16:57:11 +02:00
|
|
|
await dummyStep(test, 'Perform a dummy step', async () => {}, location);
|
feat(test runner): allow to pass arbitrary location to test.step (#32504)
Fixes https://github.com/microsoft/playwright/issues/30160
### Description:
This pull request introduces the ability to specify custom locations for
test steps in Playwright. By enabling the provision of arbitrary
locations to the test.step method, it resolves the limitation where
helper methods obfuscate the original call site, providing more accurate
and meaningful location data in test reports.
### Motivation:
To enhance the utility and clarity of test reports in Playwright.
Specifically, it addresses the need to trace test steps back to their
precise location in the code, which is especially important when steps
are abstracted in helper functions. This feature is crucial for
maintaining accurate documentation and facilitating debugging processes.
### Changes:
Added functionality to pass a custom location object to test.step.
### Expected Outcome:
This PR is expected to significantly improve the precision and
usefulness of diagnostic data in test reports by allowing specific
locations within helper functions to be accurately documented. It
facilitates better tracking of test executions and simplifies the
debugging process, making it easier for developers to understand and
address issues within complex tests.
### References:
Closes https://github.com/microsoft/playwright/issues/30160 -
"[Feature]: allow to pass arbitrary location to test.step"
**Code Check**
I conducted tests on this new feature by integrating it into some
existing test codes, and it worked well. I will attach the code used for
testing and a screenshot showing the successful outcome.
<details>
<summary>toggle dropdown</summary>
<div markdown="1">
```
import type { Location } from '../../../packages/playwright/types/testReporter'
...
test('should respect the back button', async ({ page }) => {
await page.locator('.todo-list li .toggle').nth(1).check();
await checkNumberOfCompletedTodosInLocalStorage(page, 1);
...
await test.step('Showing active items', async () => {
await page.getByRole('link', { name: 'Active' }).click();
}, {location});
```
<img width="1109" alt="image"
src="https://github.com/user-attachments/assets/359feafa-0949-4c71-9426-46debef21bdd">
</div>
</details>
2024-09-18 00:11:21 +09:00
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
test.step |Perform a dummy step @ dummy-file.ts:123
|
|
|
|
hook |After Hooks
|
|
|
|
`);
|
2024-09-18 16:57:11 +02:00
|
|
|
|
|
|
|
const { exitCode } = await runTSC({
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('should work', async () => {
|
|
|
|
const location = { file: 'dummy-file.ts', line: 123, column: 45 };
|
|
|
|
await test.step('step1', () => {}, { location });
|
|
|
|
});
|
|
|
|
`
|
|
|
|
});
|
|
|
|
expect(exitCode).toBe(0);
|
|
|
|
});
|
2024-11-05 04:45:54 -08:00
|
|
|
|
|
|
|
test('should show tracing.group nested inside test.step', async ({ runInlineTest }) => {
|
|
|
|
const result = await runInlineTest({
|
|
|
|
'reporter.ts': stepIndentReporter,
|
|
|
|
'playwright.config.ts': `module.exports = { reporter: [['./reporter', { printErrorLocation: true }]] };`,
|
|
|
|
'a.test.ts': `
|
|
|
|
import { test, expect } from '@playwright/test';
|
|
|
|
test('pass', async ({ page }) => {
|
|
|
|
await test.step('my step 1', async () => {
|
|
|
|
await test.step('my step 2', async () => {
|
|
|
|
await page.context().tracing.group('my group 1');
|
|
|
|
await page.context().tracing.group('my group 2');
|
|
|
|
await page.setContent('<button></button>');
|
|
|
|
await page.context().tracing.groupEnd();
|
|
|
|
await page.context().tracing.groupEnd();
|
|
|
|
});
|
|
|
|
});
|
|
|
|
});
|
|
|
|
`
|
|
|
|
}, { reporter: '', workers: 1 });
|
|
|
|
|
|
|
|
expect(result.exitCode).toBe(0);
|
|
|
|
expect(stripAnsi(result.output)).toBe(`
|
|
|
|
hook |Before Hooks
|
|
|
|
fixture | fixture: browser
|
|
|
|
pw:api | browserType.launch
|
|
|
|
fixture | fixture: context
|
|
|
|
pw:api | browser.newContext
|
|
|
|
fixture | fixture: page
|
|
|
|
pw:api | browserContext.newPage
|
|
|
|
test.step |my step 1 @ a.test.ts:4
|
|
|
|
test.step | my step 2 @ a.test.ts:5
|
|
|
|
pw:api | my group 1 @ a.test.ts:6
|
|
|
|
pw:api | my group 2 @ a.test.ts:7
|
|
|
|
pw:api | page.setContent @ a.test.ts:8
|
|
|
|
hook |After Hooks
|
|
|
|
fixture | fixture: page
|
|
|
|
fixture | fixture: context
|
|
|
|
`);
|
|
|
|
});
|