mirror of
https://github.com/microsoft/playwright.git
synced 2025-06-26 21:40:17 +00:00
fix(list reporter): print step ends in non-TTY mode (#31703)
When used in a terminal, the `list` reporter prints out information about test steps to help debugging. In non-TTY environments like GitHub Actions, currently it doesn't. This PR changes that, so that in non-TTY environments you'll see the "step end" messages appearing, but not the "step begin" messages. This is a good middleground, because it helps the user understand test progress, without being too verbose. Closes https://github.com/microsoft/playwright/issues/31674
This commit is contained in:
Simon Knott
GitHub
parent
ed6abf86c7
commit
8eab28d858
@ -36,7 +36,7 @@ class ListReporter extends BaseReporter {
|
||||
|
||||
constructor(options: { printSteps?: boolean } = {}) {
|
||||
super();
|
||||
this._printSteps = isTTY && getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options.printSteps);
|
||||
this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options.printSteps);
|
||||
}
|
||||
|
||||
override printsToStdio() {
|
||||
@ -54,11 +54,13 @@ class ListReporter extends BaseReporter {
|
||||
|
||||
override onTestBegin(test: TestCase, result: TestResult) {
|
||||
super.onTestBegin(test, result);
|
||||
|
||||
const index = String(this._resultIndex.size + 1);
|
||||
this._resultIndex.set(result, index);
|
||||
|
||||
if (!isTTY)
|
||||
return;
|
||||
this._maybeWriteNewLine();
|
||||
const index = String(this._resultIndex.size + 1);
|
||||
this._resultIndex.set(result, index);
|
||||
this._testRows.set(test, this._lastRow);
|
||||
const prefix = this._testPrefix(index, '');
|
||||
const line = colors.dim(formatTestTitle(this.config, test)) + this._retrySuffix(result);
|
||||
@ -75,28 +77,34 @@ class ListReporter extends BaseReporter {
|
||||
this._dumpToStdio(test, chunk, process.stderr);
|
||||
}
|
||||
|
||||
override onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
|
||||
super.onStepBegin(test, result, step);
|
||||
if (step.category !== 'test.step')
|
||||
return;
|
||||
const testIndex = this._resultIndex.get(result) || '';
|
||||
if (!this._printSteps) {
|
||||
if (isTTY)
|
||||
this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
|
||||
return;
|
||||
}
|
||||
private getStepIndex(testIndex: string, result: TestResult, step: TestStep): string {
|
||||
if (this._stepIndex.has(step))
|
||||
return this._stepIndex.get(step)!;
|
||||
|
||||
const ordinal = ((result as any)[lastStepOrdinalSymbol] || 0) + 1;
|
||||
(result as any)[lastStepOrdinalSymbol] = ordinal;
|
||||
const stepIndex = `${testIndex}.${ordinal}`;
|
||||
this._stepIndex.set(step, stepIndex);
|
||||
return stepIndex;
|
||||
}
|
||||
|
||||
if (isTTY) {
|
||||
override onStepBegin(test: TestCase, result: TestResult, step: TestStep) {
|
||||
super.onStepBegin(test, result, step);
|
||||
if (step.category !== 'test.step')
|
||||
return;
|
||||
const testIndex = this._resultIndex.get(result) || '';
|
||||
|
||||
if (!isTTY)
|
||||
return;
|
||||
|
||||
if (this._printSteps) {
|
||||
this._maybeWriteNewLine();
|
||||
this._stepRows.set(step, this._lastRow);
|
||||
const prefix = this._testPrefix(stepIndex, '');
|
||||
const prefix = this._testPrefix(this.getStepIndex(testIndex, result, step), '');
|
||||
const line = test.title + colors.dim(stepSuffix(step));
|
||||
this._appendLine(line, prefix);
|
||||
} else {
|
||||
this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, ''));
|
||||
}
|
||||
}
|
||||
|
||||
@ -112,8 +120,8 @@ class ListReporter extends BaseReporter {
|
||||
return;
|
||||
}
|
||||
|
||||
const index = this._stepIndex.get(step)!;
|
||||
const title = test.title + colors.dim(stepSuffix(step));
|
||||
const index = this.getStepIndex(testIndex, result, step);
|
||||
const title = isTTY ? test.title + colors.dim(stepSuffix(step)) : formatTestTitle(this.config, test, step);
|
||||
const prefix = this._testPrefix(index, '');
|
||||
let text = '';
|
||||
if (step.error)
|
||||
@ -204,7 +212,7 @@ class ListReporter extends BaseReporter {
|
||||
private _appendLine(text: string, prefix: string) {
|
||||
const line = prefix + this.fitToScreen(text, prefix);
|
||||
if (process.env.PW_TEST_DEBUG_REPORTERS) {
|
||||
process.stdout.write(this._lastRow + ' : ' + line + '\n');
|
||||
process.stdout.write('#' + this._lastRow + ' : ' + line + '\n');
|
||||
} else {
|
||||
process.stdout.write(line);
|
||||
process.stdout.write('\n');
|
||||
@ -215,7 +223,7 @@ class ListReporter extends BaseReporter {
|
||||
private _updateLine(row: number, text: string, prefix: string) {
|
||||
const line = prefix + this.fitToScreen(text, prefix);
|
||||
if (process.env.PW_TEST_DEBUG_REPORTERS)
|
||||
process.stdout.write(row + ' : ' + line + '\n');
|
||||
process.stdout.write('#' + row + ' : ' + line + '\n');
|
||||
else
|
||||
this._updateLineForTTY(row, line);
|
||||
}
|
||||
|
||||
@ -478,28 +478,36 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports }
|
||||
|
||||
const text = stripAnsi(output);
|
||||
expect(text).toContain('Running 10 tests using 3 workers');
|
||||
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
|
||||
const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
|
||||
expect(lines).toEqual([
|
||||
`0 : 1 a.test.js:3:11 › math 1`,
|
||||
`0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`,
|
||||
`1 : 2 a.test.js:6:11 › failing 1`,
|
||||
`1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`,
|
||||
`2 : 3 a.test.js:6:11 › failing 1 (retry #1)`,
|
||||
`2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`,
|
||||
`3 : 4 a.test.js:9:11 › flaky 1`,
|
||||
`3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`,
|
||||
`4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`,
|
||||
`4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`,
|
||||
`5 : 6 a.test.js:12:12 › skipped 1`,
|
||||
`5 : - 6 a.test.js:12:12 › skipped 1`,
|
||||
`6 : 7 b.test.js:3:11 › math 2`,
|
||||
`6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`,
|
||||
`7 : 8 b.test.js:6:11 › failing 2`,
|
||||
`7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`,
|
||||
`8 : 9 b.test.js:6:11 › failing 2 (retry #1)`,
|
||||
`8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`,
|
||||
`9 : 10 b.test.js:9:12 › skipped 2`,
|
||||
`9 : - 10 b.test.js:9:12 › skipped 2`
|
||||
`#0 : 1 a.test.js:3:11 › math 1`,
|
||||
`#0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`,
|
||||
`#1 : 2 a.test.js:6:11 › failing 1`,
|
||||
`#1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`,
|
||||
`#2 : 3 a.test.js:6:11 › failing 1 (retry #1)`,
|
||||
`#2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`,
|
||||
`#3 : 4 a.test.js:9:11 › flaky 1`,
|
||||
`#3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`,
|
||||
`#4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`,
|
||||
`#4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`,
|
||||
`#5 : 6 a.test.js:12:12 › skipped 1`,
|
||||
`#5 : - 6 a.test.js:12:12 › skipped 1`,
|
||||
`#6 : 7 b.test.js:3:11 › math 2`,
|
||||
`#6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`,
|
||||
`#7 : 8 b.test.js:6:11 › failing 2`,
|
||||
`#7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`,
|
||||
`#8 : 9 b.test.js:6:11 › failing 2 (retry #1)`,
|
||||
`#8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`,
|
||||
`#9 : 10 b.test.js:9:12 › skipped 2`,
|
||||
`#9 : - 10 b.test.js:9:12 › skipped 2`,
|
||||
`#10 : 11 c.test.js:3:11 › math 3`,
|
||||
`#10 : ${POSITIVE_STATUS_MARK} 11 c.test.js:3:11 › math 3 (Xms)`,
|
||||
`#11 : 12 c.test.js:6:11 › flaky 2`,
|
||||
`#11 : ${NEGATIVE_STATUS_MARK} 12 c.test.js:6:11 › flaky 2 (Xms)`,
|
||||
`#12 : 13 c.test.js:6:11 › flaky 2 (retry #1)`,
|
||||
`#12 : ${POSITIVE_STATUS_MARK} 13 c.test.js:6:11 › flaky 2 (retry #1) (Xms)`,
|
||||
`#13 : 14 c.test.js:9:12 › skipped 3`,
|
||||
`#13 : - 14 c.test.js:9:12 › skipped 3`,
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
@ -72,22 +72,22 @@ for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
`,
|
||||
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIST_PRINT_STEPS: '1', PLAYWRIGHT_FORCE_TTY: '80' });
|
||||
const text = result.output;
|
||||
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
|
||||
const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
|
||||
lines.pop(); // Remove last item that contains [v] and time in ms.
|
||||
expect(lines).toEqual([
|
||||
'0 : 1 a.test.ts:3:15 › passes',
|
||||
'1 : 1.1 passes › outer 1.0',
|
||||
'2 : 1.2 passes › outer 1.0 › inner 1.1',
|
||||
'2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
|
||||
'3 : 1.3 passes › outer 1.0 › inner 1.2',
|
||||
'3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
|
||||
'1 : 1.1 passes › outer 1.0 (Xms)',
|
||||
'4 : 1.4 passes › outer 2.0',
|
||||
'5 : 1.5 passes › outer 2.0 › inner 2.1',
|
||||
'5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
|
||||
'6 : 1.6 passes › outer 2.0 › inner 2.2',
|
||||
'6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
|
||||
'4 : 1.4 passes › outer 2.0 (Xms)',
|
||||
'#0 : 1 a.test.ts:3:15 › passes',
|
||||
'#1 : 1.1 passes › outer 1.0',
|
||||
'#2 : 1.2 passes › outer 1.0 › inner 1.1',
|
||||
'#2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)',
|
||||
'#3 : 1.3 passes › outer 1.0 › inner 1.2',
|
||||
'#3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)',
|
||||
'#1 : 1.1 passes › outer 1.0 (Xms)',
|
||||
'#4 : 1.4 passes › outer 2.0',
|
||||
'#5 : 1.5 passes › outer 2.0 › inner 2.1',
|
||||
'#5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)',
|
||||
'#6 : 1.6 passes › outer 2.0 › inner 2.2',
|
||||
'#6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)',
|
||||
'#4 : 1.4 passes › outer 2.0 (Xms)',
|
||||
]);
|
||||
});
|
||||
|
||||
@ -107,22 +107,51 @@ for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
});`,
|
||||
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' });
|
||||
const text = result.output;
|
||||
const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
|
||||
const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
|
||||
lines.pop(); // Remove last item that contains [v] and time in ms.
|
||||
expect(lines).toEqual([
|
||||
'0 : 1 a.test.ts:3:11 › passes',
|
||||
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
|
||||
'0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1',
|
||||
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
|
||||
'0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2',
|
||||
'0 : 1 a.test.ts:4:20 › passes › outer 1.0',
|
||||
'0 : 1 a.test.ts:3:11 › passes',
|
||||
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
|
||||
'0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1',
|
||||
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
|
||||
'0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2',
|
||||
'0 : 1 a.test.ts:8:20 › passes › outer 2.0',
|
||||
'0 : 1 a.test.ts:3:11 › passes',
|
||||
'#0 : 1 a.test.ts:3:11 › passes',
|
||||
'#0 : 1 a.test.ts:4:20 › passes › outer 1.0',
|
||||
'#0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1',
|
||||
'#0 : 1 a.test.ts:4:20 › passes › outer 1.0',
|
||||
'#0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2',
|
||||
'#0 : 1 a.test.ts:4:20 › passes › outer 1.0',
|
||||
'#0 : 1 a.test.ts:3:11 › passes',
|
||||
'#0 : 1 a.test.ts:8:20 › passes › outer 2.0',
|
||||
'#0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1',
|
||||
'#0 : 1 a.test.ts:8:20 › passes › outer 2.0',
|
||||
'#0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2',
|
||||
'#0 : 1 a.test.ts:8:20 › passes › outer 2.0',
|
||||
'#0 : 1 a.test.ts:3:11 › passes',
|
||||
]);
|
||||
});
|
||||
|
||||
test('render steps in non-TTY mode', async ({ runInlineTest }) => {
|
||||
const result = await runInlineTest({
|
||||
'a.test.ts': `
|
||||
import { test, expect } from '@playwright/test';
|
||||
test('passes', async ({}) => {
|
||||
await test.step('outer 1.0', async () => {
|
||||
await test.step('inner 1.1', async () => {});
|
||||
await test.step('inner 1.2', async () => {});
|
||||
});
|
||||
await test.step('outer 2.0', async () => {
|
||||
await test.step('inner 2.1', async () => {});
|
||||
await test.step('inner 2.2', async () => {});
|
||||
});
|
||||
});
|
||||
`,
|
||||
}, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIST_PRINT_STEPS: '1' });
|
||||
const text = result.output;
|
||||
const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms'));
|
||||
expect(lines).toEqual([
|
||||
'#0 : 1.1 a.test.ts:5:26 › passes › outer 1.0 › inner 1.1 (Xms)',
|
||||
'#1 : 1.2 a.test.ts:6:26 › passes › outer 1.0 › inner 1.2 (Xms)',
|
||||
'#2 : 1.3 a.test.ts:4:24 › passes › outer 1.0 (Xms)',
|
||||
'#3 : 1.4 a.test.ts:9:26 › passes › outer 2.0 › inner 2.1 (Xms)',
|
||||
'#4 : 1.5 a.test.ts:10:26 › passes › outer 2.0 › inner 2.2 (Xms)',
|
||||
'#5 : 1.6 a.test.ts:8:24 › passes › outer 2.0 (Xms)',
|
||||
`#6 : ${POSITIVE_STATUS_MARK} 1 a.test.ts:3:15 › passes (Xms)`,
|
||||
]);
|
||||
});
|
||||
|
||||
@ -156,13 +185,13 @@ for (const useIntermediateMergeReport of [false, true] as const) {
|
||||
`,
|
||||
}, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' });
|
||||
const text = result.output;
|
||||
const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
|
||||
const lines = text.split('\n').filter(l => l.startsWith('#0 :') || l.startsWith('#1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms'));
|
||||
|
||||
expect(lines).toEqual([
|
||||
`0 : 1 a.test.ts:3:15 › flaky`,
|
||||
`0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`,
|
||||
`1 : 2 a.test.ts:3:15 › flaky (retry #1)`,
|
||||
`1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`,
|
||||
`#0 : 1 a.test.ts:3:15 › flaky`,
|
||||
`#0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`,
|
||||
`#1 : 2 a.test.ts:3:15 › flaky (retry #1)`,
|
||||
`#1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`,
|
||||
]);
|
||||
});
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user