yujunjun/playwright
1
0
Fork 0
mirror of https://github.com/microsoft/playwright.git synced 2025-06-26 21:40:17 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(list reporter): print step ends in non-TTY mode (#31703)

Browse Source 
When used in a terminal, the `list` reporter prints out information
about test steps to help debugging. In non-TTY environments like GitHub
Actions, currently it doesn't.

This PR changes that, so that in non-TTY environments you'll see the
"step end" messages appearing, but not the "step begin" messages. This
is a good middleground, because it helps the user understand test
progress, without being too verbose.

Closes https://github.com/microsoft/playwright/issues/31674
This commit is contained in:
Simon Knott 2024-07-17 13:36:37 +02:00 committed by GitHub
parent ed6abf86c7
commit 8eab28d858
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 118 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
packages/playwright/src/reporters/list.ts
View File

@ -36,7 +36,7 @@ class ListReporter extends BaseReporter {
constructor(options: { printSteps?: boolean } = {}) { constructor(options: { printSteps?: boolean } = {}) {
super(); super();
this._printSteps = isTTY && getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options.printSteps); this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options.printSteps);
} }
override printsToStdio() { override printsToStdio() {
@ -54,11 +54,13 @@ class ListReporter extends BaseReporter {
override onTestBegin(test: TestCase, result: TestResult) { override onTestBegin(test: TestCase, result: TestResult) {
super.onTestBegin(test, result); super.onTestBegin(test, result);
const index = String(this._resultIndex.size + 1);
this._resultIndex.set(result, index);
if (!isTTY) if (!isTTY)
return; return;
this._maybeWriteNewLine(); this._maybeWriteNewLine();
const index = String(this._resultIndex.size + 1);
this._resultIndex.set(result, index);
this._testRows.set(test, this._lastRow); this._testRows.set(test, this._lastRow);
const prefix = this._testPrefix(index, ''); const prefix = this._testPrefix(index, '');
const line = colors.dim(formatTestTitle(this.config, test)) + this._retrySuffix(result); const line = colors.dim(formatTestTitle(this.config, test)) + this._retrySuffix(result);
@ -75,28 +77,34 @@ class ListReporter extends BaseReporter {
this._dumpToStdio(test, chunk, process.stderr); this._dumpToStdio(test, chunk, process.stderr);
} }
override onStepBegin(test: TestCase, result: TestResult, step: TestStep) { private getStepIndex(testIndex: string, result: TestResult, step: TestStep): string {
super.onStepBegin(test, result, step); if (this._stepIndex.has(step))
if (step.category !== 'test.step') return this._stepIndex.get(step)!;
return;
const testIndex = this._resultIndex.get(result) || '';
if (!this._printSteps) {
if (isTTY)
this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
return;
}
const ordinal = ((result as any)[lastStepOrdinalSymbol] || 0) + 1; const ordinal = ((result as any)[lastStepOrdinalSymbol] || 0) + 1;
(result as any)[lastStepOrdinalSymbol] = ordinal; (result as any)[lastStepOrdinalSymbol] = ordinal;
const stepIndex = `${testIndex}.${ordinal}`; const stepIndex = `${testIndex}.${ordinal}`;
this._stepIndex.set(step, stepIndex); this._stepIndex.set(step, stepIndex);
return stepIndex;
}
if (isTTY) { override onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
super.onStepBegin(test, result, step);
if (step.category !== 'test.step')
return;
const testIndex = this._resultIndex.get(result) || '';
if (!isTTY)
return;
if (this._printSteps) {
this._maybeWriteNewLine(); this._maybeWriteNewLine();
this._stepRows.set(step, this._lastRow); this._stepRows.set(step, this._lastRow);
const prefix = this._testPrefix(stepIndex, ''); const prefix = this._testPrefix(this.getStepIndex(testIndex, result, step), '');
const line = test.title + colors.dim(stepSuffix(step)); const line = test.title + colors.dim(stepSuffix(step));
this._appendLine(line, prefix); this._appendLine(line, prefix);
} else {
this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
} }
} }
@ -112,8 +120,8 @@ class ListReporter extends BaseReporter {
return; return;
} }
const index = this._stepIndex.get(step)!; const index = this.getStepIndex(testIndex, result, step);
const title = test.title + colors.dim(stepSuffix(step)); const title = isTTY ? test.title + colors.dim(stepSuffix(step)) : formatTestTitle(this.config, test, step);
const prefix = this._testPrefix(index, ''); const prefix = this._testPrefix(index, '');
let text = ''; let text = '';
if (step.error) if (step.error)
@ -204,7 +212,7 @@ class ListReporter extends BaseReporter {
private _appendLine(text: string, prefix: string) { private _appendLine(text: string, prefix: string) {
const line = prefix + this.fitToScreen(text, prefix); const line = prefix + this.fitToScreen(text, prefix);
if (process.env.PW_TEST_DEBUG_REPORTERS) { if (process.env.PW_TEST_DEBUG_REPORTERS) {
process.stdout.write(this._lastRow + ' : ' + line + '\n'); process.stdout.write('#' + this._lastRow + ' : ' + line + '\n');
} else { } else {
process.stdout.write(line); process.stdout.write(line);
process.stdout.write('\n'); process.stdout.write('\n');
@ -215,7 +223,7 @@ class ListReporter extends BaseReporter {
private _updateLine(row: number, text: string, prefix: string) { private _updateLine(row: number, text: string, prefix: string) {
const line = prefix + this.fitToScreen(text, prefix); const line = prefix + this.fitToScreen(text, prefix);
if (process.env.PW_TEST_DEBUG_REPORTERS) if (process.env.PW_TEST_DEBUG_REPORTERS)
process.stdout.write(row + ' : ' + line + '\n'); process.stdout.write('#' + row + ' : ' + line + '\n');
else else
this._updateLineForTTY(row, line); this._updateLineForTTY(row, line);
} }

50
tests/playwright-test/reporter-blob.spec.ts
View File

@ -478,28 +478,36 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
const text = stripAnsi(output); const text = stripAnsi(output);
expect(text).toContain('Running 10 tests using 3 workers'); expect(text).toContain('Running 10 tests using 3 workers');
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
expect(lines).toEqual([ expect(lines).toEqual([
`0 : 1 a.test.js:3:11 › math 1`, `#0 : 1 a.test.js:3:11 › math 1`,
`0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`, `#0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`,
`1 : 2 a.test.js:6:11 › failing 1`, `#1 : 2 a.test.js:6:11 › failing 1`,
`1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`, `#1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`,
`2 : 3 a.test.js:6:11 › failing 1 (retry #1)`, `#2 : 3 a.test.js:6:11 › failing 1 (retry #1)`,
`2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`, `#2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`,
`3 : 4 a.test.js:9:11 › flaky 1`, `#3 : 4 a.test.js:9:11 › flaky 1`,
`3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`, `#3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`,
`4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`, `#4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`,
`4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`, `#4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`,
`5 : 6 a.test.js:12:12 › skipped 1`, `#5 : 6 a.test.js:12:12 › skipped 1`,
`5 : - 6 a.test.js:12:12 › skipped 1`, `#5 : - 6 a.test.js:12:12 › skipped 1`,
`6 : 7 b.test.js:3:11 › math 2`, `#6 : 7 b.test.js:3:11 › math 2`,
`6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`, `#6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`,
`7 : 8 b.test.js:6:11 › failing 2`, `#7 : 8 b.test.js:6:11 › failing 2`,
`7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`, `#7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`,
`8 : 9 b.test.js:6:11 › failing 2 (retry #1)`, `#8 : 9 b.test.js:6:11 › failing 2 (retry #1)`,
`8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`, `#8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`,
`9 : 10 b.test.js:9:12 › skipped 2`, `#9 : 10 b.test.js:9:12 › skipped 2`,
`9 : - 10 b.test.js:9:12 › skipped 2` `#9 : - 10 b.test.js:9:12 › skipped 2`,
`#10 : 11 c.test.js:3:11 › math 3`,
`#10 : ${POSITIVE_STATUS_MARK} 11 c.test.js:3:11 › math 3 (Xms)`,
`#11 : 12 c.test.js:6:11 › flaky 2`,
`#11 : ${NEGATIVE_STATUS_MARK} 12 c.test.js:6:11 › flaky 2 (Xms)`,
`#12 : 13 c.test.js:6:11 › flaky 2 (retry #1)`,
`#12 : ${POSITIVE_STATUS_MARK} 13 c.test.js:6:11 › flaky 2 (retry #1) (Xms)`,
`#13 : 14 c.test.js:9:12 › skipped 3`,
`#13 : - 14 c.test.js:9:12 › skipped 3`,
]); ]);
}); });

95
tests/playwright-test/reporter-list.spec.ts
View File

@ -72,22 +72,22 @@ for (const useIntermediateMergeReport of [false, true] as const) {
`, `,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIST_PRINT_STEPS: '1', PLAYWRIGHT_FORCE_TTY: '80' }); }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIST_PRINT_STEPS: '1', PLAYWRIGHT_FORCE_TTY: '80' });
const text = result.output; const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms. lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([ expect(lines).toEqual([
'0 : 1 a.test.ts:3:15 › passes', '#0 : 1 a.test.ts:3:15 › passes',
'1 : 1.1 passes › outer 1.0', '#1 : 1.1 passes › outer 1.0',
'2 : 1.2 passes › outer 1.0 › inner 1.1', '#2 : 1.2 passes › outer 1.0 › inner 1.1',
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)', '#2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
'3 : 1.3 passes › outer 1.0 › inner 1.2', '#3 : 1.3 passes › outer 1.0 › inner 1.2',
'3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)', '#3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
'1 : 1.1 passes › outer 1.0 (Xms)', '#1 : 1.1 passes › outer 1.0 (Xms)',
'4 : 1.4 passes › outer 2.0', '#4 : 1.4 passes › outer 2.0',
'5 : 1.5 passes › outer 2.0 › inner 2.1', '#5 : 1.5 passes › outer 2.0 › inner 2.1',
'5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)', '#5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
'6 : 1.6 passes › outer 2.0 › inner 2.2', '#6 : 1.6 passes › outer 2.0 › inner 2.2',
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)', '#6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
'4 : 1.4 passes › outer 2.0 (Xms)', '#4 : 1.4 passes › outer 2.0 (Xms)',
]); ]);
}); });
@ -107,22 +107,51 @@ for (const useIntermediateMergeReport of [false, true] as const) {
});`, });`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' }); }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' });
const text = result.output; const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
lines.pop(); // Remove last item that contains [v] and time in ms. lines.pop(); // Remove last item that contains [v] and time in ms.
expect(lines).toEqual([ expect(lines).toEqual([
'0 : 1 a.test.ts:3:11 › passes', '#0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0', '#0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1', '#0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0', '#0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2', '#0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2',
'0 : 1 a.test.ts:4:20 › passes › outer 1.0', '#0 : 1 a.test.ts:4:20 › passes › outer 1.0',
'0 : 1 a.test.ts:3:11 › passes', '#0 : 1 a.test.ts:3:11 › passes',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0', '#0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1', '#0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0', '#0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2', '#0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2',
'0 : 1 a.test.ts:8:20 › passes › outer 2.0', '#0 : 1 a.test.ts:8:20 › passes › outer 2.0',
'0 : 1 a.test.ts:3:11 › passes', '#0 : 1 a.test.ts:3:11 › passes',
]);
});
test('render steps in non-TTY mode', async ({ runInlineTest }) => {
const result = await runInlineTest({
'a.test.ts': `
import { test, expect } from '@playwright/test';
test('passes', async ({}) => {
await test.step('outer 1.0', async () => {
await test.step('inner 1.1', async () => {});
await test.step('inner 1.2', async () => {});
});
await test.step('outer 2.0', async () => {
await test.step('inner 2.1', async () => {});
await test.step('inner 2.2', async () => {});
});
});
`,
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIST_PRINT_STEPS: '1' });
const text = result.output;
const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
expect(lines).toEqual([
'#0 : 1.1 a.test.ts:5:26 › passes › outer 1.0 › inner 1.1 (Xms)',
'#1 : 1.2 a.test.ts:6:26 › passes › outer 1.0 › inner 1.2 (Xms)',
'#2 : 1.3 a.test.ts:4:24 › passes › outer 1.0 (Xms)',
'#3 : 1.4 a.test.ts:9:26 › passes › outer 2.0 › inner 2.1 (Xms)',
'#4 : 1.5 a.test.ts:10:26 › passes › outer 2.0 › inner 2.2 (Xms)',
'#5 : 1.6 a.test.ts:8:24 › passes › outer 2.0 (Xms)',
`#6 : ${POSITIVE_STATUS_MARK} 1 a.test.ts:3:15 › passes (Xms)`,
]); ]);
}); });
@ -156,13 +185,13 @@ for (const useIntermediateMergeReport of [false, true] as const) {
`, `,
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' }); }, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' });
const text = result.output; const text = result.output;
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms')); const lines = text.split('\n').filter(l => l.startsWith('#0 :') || l.startsWith('#1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
expect(lines).toEqual([ expect(lines).toEqual([
`0 : 1 a.test.ts:3:15 › flaky`, `#0 : 1 a.test.ts:3:15 › flaky`,
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`, `#0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`,
`1 : 2 a.test.ts:3:15 › flaky (retry #1)`, `#1 : 2 a.test.ts:3:15 › flaky (retry #1)`,
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`, `#1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`,
]); ]);
}); });