diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts index 94e507fd4b..f87a7d4fe8 100644 --- a/packages/playwright/src/reporters/list.ts +++ b/packages/playwright/src/reporters/list.ts @@ -36,7 +36,7 @@ class ListReporter extends BaseReporter { constructor(options: { printSteps?: boolean } = {}) { super(); - this._printSteps = isTTY && getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options.printSteps); + this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options.printSteps); } override printsToStdio() { @@ -54,11 +54,13 @@ class ListReporter extends BaseReporter { override onTestBegin(test: TestCase, result: TestResult) { super.onTestBegin(test, result); + + const index = String(this._resultIndex.size + 1); + this._resultIndex.set(result, index); + if (!isTTY) return; this._maybeWriteNewLine(); - const index = String(this._resultIndex.size + 1); - this._resultIndex.set(result, index); this._testRows.set(test, this._lastRow); const prefix = this._testPrefix(index, ''); const line = colors.dim(formatTestTitle(this.config, test)) + this._retrySuffix(result); @@ -75,28 +77,34 @@ class ListReporter extends BaseReporter { this._dumpToStdio(test, chunk, process.stderr); } - override onStepBegin(test: TestCase, result: TestResult, step: TestStep) { - super.onStepBegin(test, result, step); - if (step.category !== 'test.step') - return; - const testIndex = this._resultIndex.get(result) || ''; - if (!this._printSteps) { - if (isTTY) - this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); - return; - } + private getStepIndex(testIndex: string, result: TestResult, step: TestStep): string { + if (this._stepIndex.has(step)) + return this._stepIndex.get(step)!; const ordinal = ((result as any)[lastStepOrdinalSymbol] || 0) + 1; (result as any)[lastStepOrdinalSymbol] = ordinal; const stepIndex = `${testIndex}.${ordinal}`; this._stepIndex.set(step, stepIndex); + return stepIndex; + } - if (isTTY) { + override onStepBegin(test: TestCase, result: TestResult, step: TestStep) { + super.onStepBegin(test, result, step); + if (step.category !== 'test.step') + return; + const testIndex = this._resultIndex.get(result) || ''; + + if (!isTTY) + return; + + if (this._printSteps) { this._maybeWriteNewLine(); this._stepRows.set(step, this._lastRow); - const prefix = this._testPrefix(stepIndex, ''); + const prefix = this._testPrefix(this.getStepIndex(testIndex, result, step), ''); const line = test.title + colors.dim(stepSuffix(step)); this._appendLine(line, prefix); + } else { + this._updateLine(this._testRows.get(test)!, colors.dim(formatTestTitle(this.config, test, step)) + this._retrySuffix(result), this._testPrefix(testIndex, '')); } } @@ -112,8 +120,8 @@ class ListReporter extends BaseReporter { return; } - const index = this._stepIndex.get(step)!; - const title = test.title + colors.dim(stepSuffix(step)); + const index = this.getStepIndex(testIndex, result, step); + const title = isTTY ? test.title + colors.dim(stepSuffix(step)) : formatTestTitle(this.config, test, step); const prefix = this._testPrefix(index, ''); let text = ''; if (step.error) @@ -204,7 +212,7 @@ class ListReporter extends BaseReporter { private _appendLine(text: string, prefix: string) { const line = prefix + this.fitToScreen(text, prefix); if (process.env.PW_TEST_DEBUG_REPORTERS) { - process.stdout.write(this._lastRow + ' : ' + line + '\n'); + process.stdout.write('#' + this._lastRow + ' : ' + line + '\n'); } else { process.stdout.write(line); process.stdout.write('\n'); @@ -215,7 +223,7 @@ class ListReporter extends BaseReporter { private _updateLine(row: number, text: string, prefix: string) { const line = prefix + this.fitToScreen(text, prefix); if (process.env.PW_TEST_DEBUG_REPORTERS) - process.stdout.write(row + ' : ' + line + '\n'); + process.stdout.write('#' + row + ' : ' + line + '\n'); else this._updateLineForTTY(row, line); } diff --git a/tests/playwright-test/reporter-blob.spec.ts b/tests/playwright-test/reporter-blob.spec.ts index 99882777a8..8f3064a073 100644 --- a/tests/playwright-test/reporter-blob.spec.ts +++ b/tests/playwright-test/reporter-blob.spec.ts @@ -478,28 +478,36 @@ test('merge into list report by default', async ({ runInlineTest, mergeReports } const text = stripAnsi(output); expect(text).toContain('Running 10 tests using 3 workers'); - const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); + const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); expect(lines).toEqual([ - `0 : 1 a.test.js:3:11 › math 1`, - `0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`, - `1 : 2 a.test.js:6:11 › failing 1`, - `1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`, - `2 : 3 a.test.js:6:11 › failing 1 (retry #1)`, - `2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`, - `3 : 4 a.test.js:9:11 › flaky 1`, - `3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`, - `4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`, - `4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`, - `5 : 6 a.test.js:12:12 › skipped 1`, - `5 : - 6 a.test.js:12:12 › skipped 1`, - `6 : 7 b.test.js:3:11 › math 2`, - `6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`, - `7 : 8 b.test.js:6:11 › failing 2`, - `7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`, - `8 : 9 b.test.js:6:11 › failing 2 (retry #1)`, - `8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`, - `9 : 10 b.test.js:9:12 › skipped 2`, - `9 : - 10 b.test.js:9:12 › skipped 2` + `#0 : 1 a.test.js:3:11 › math 1`, + `#0 : ${POSITIVE_STATUS_MARK} 1 a.test.js:3:11 › math 1 (Xms)`, + `#1 : 2 a.test.js:6:11 › failing 1`, + `#1 : ${NEGATIVE_STATUS_MARK} 2 a.test.js:6:11 › failing 1 (Xms)`, + `#2 : 3 a.test.js:6:11 › failing 1 (retry #1)`, + `#2 : ${NEGATIVE_STATUS_MARK} 3 a.test.js:6:11 › failing 1 (retry #1) (Xms)`, + `#3 : 4 a.test.js:9:11 › flaky 1`, + `#3 : ${NEGATIVE_STATUS_MARK} 4 a.test.js:9:11 › flaky 1 (Xms)`, + `#4 : 5 a.test.js:9:11 › flaky 1 (retry #1)`, + `#4 : ${POSITIVE_STATUS_MARK} 5 a.test.js:9:11 › flaky 1 (retry #1) (Xms)`, + `#5 : 6 a.test.js:12:12 › skipped 1`, + `#5 : - 6 a.test.js:12:12 › skipped 1`, + `#6 : 7 b.test.js:3:11 › math 2`, + `#6 : ${POSITIVE_STATUS_MARK} 7 b.test.js:3:11 › math 2 (Xms)`, + `#7 : 8 b.test.js:6:11 › failing 2`, + `#7 : ${NEGATIVE_STATUS_MARK} 8 b.test.js:6:11 › failing 2 (Xms)`, + `#8 : 9 b.test.js:6:11 › failing 2 (retry #1)`, + `#8 : ${NEGATIVE_STATUS_MARK} 9 b.test.js:6:11 › failing 2 (retry #1) (Xms)`, + `#9 : 10 b.test.js:9:12 › skipped 2`, + `#9 : - 10 b.test.js:9:12 › skipped 2`, + `#10 : 11 c.test.js:3:11 › math 3`, + `#10 : ${POSITIVE_STATUS_MARK} 11 c.test.js:3:11 › math 3 (Xms)`, + `#11 : 12 c.test.js:6:11 › flaky 2`, + `#11 : ${NEGATIVE_STATUS_MARK} 12 c.test.js:6:11 › flaky 2 (Xms)`, + `#12 : 13 c.test.js:6:11 › flaky 2 (retry #1)`, + `#12 : ${POSITIVE_STATUS_MARK} 13 c.test.js:6:11 › flaky 2 (retry #1) (Xms)`, + `#13 : 14 c.test.js:9:12 › skipped 3`, + `#13 : - 14 c.test.js:9:12 › skipped 3`, ]); }); diff --git a/tests/playwright-test/reporter-list.spec.ts b/tests/playwright-test/reporter-list.spec.ts index 489e97feb7..c57190a5d7 100644 --- a/tests/playwright-test/reporter-list.spec.ts +++ b/tests/playwright-test/reporter-list.spec.ts @@ -72,22 +72,22 @@ for (const useIntermediateMergeReport of [false, true] as const) { `, }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIST_PRINT_STEPS: '1', PLAYWRIGHT_FORCE_TTY: '80' }); const text = result.output; - const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); + const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); lines.pop(); // Remove last item that contains [v] and time in ms. expect(lines).toEqual([ - '0 : 1 a.test.ts:3:15 › passes', - '1 : 1.1 passes › outer 1.0', - '2 : 1.2 passes › outer 1.0 › inner 1.1', - '2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)', - '3 : 1.3 passes › outer 1.0 › inner 1.2', - '3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)', - '1 : 1.1 passes › outer 1.0 (Xms)', - '4 : 1.4 passes › outer 2.0', - '5 : 1.5 passes › outer 2.0 › inner 2.1', - '5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)', - '6 : 1.6 passes › outer 2.0 › inner 2.2', - '6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)', - '4 : 1.4 passes › outer 2.0 (Xms)', + '#0 : 1 a.test.ts:3:15 › passes', + '#1 : 1.1 passes › outer 1.0', + '#2 : 1.2 passes › outer 1.0 › inner 1.1', + '#2 : 1.2 passes › outer 1.0 › inner 1.1 (Xms)', + '#3 : 1.3 passes › outer 1.0 › inner 1.2', + '#3 : 1.3 passes › outer 1.0 › inner 1.2 (Xms)', + '#1 : 1.1 passes › outer 1.0 (Xms)', + '#4 : 1.4 passes › outer 2.0', + '#5 : 1.5 passes › outer 2.0 › inner 2.1', + '#5 : 1.5 passes › outer 2.0 › inner 2.1 (Xms)', + '#6 : 1.6 passes › outer 2.0 › inner 2.2', + '#6 : 1.6 passes › outer 2.0 › inner 2.2 (Xms)', + '#4 : 1.4 passes › outer 2.0 (Xms)', ]); }); @@ -107,22 +107,51 @@ for (const useIntermediateMergeReport of [false, true] as const) { });`, }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' }); const text = result.output; - const lines = text.split('\n').filter(l => l.match(/^\d :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); + const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); lines.pop(); // Remove last item that contains [v] and time in ms. expect(lines).toEqual([ - '0 : 1 a.test.ts:3:11 › passes', - '0 : 1 a.test.ts:4:20 › passes › outer 1.0', - '0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1', - '0 : 1 a.test.ts:4:20 › passes › outer 1.0', - '0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2', - '0 : 1 a.test.ts:4:20 › passes › outer 1.0', - '0 : 1 a.test.ts:3:11 › passes', - '0 : 1 a.test.ts:8:20 › passes › outer 2.0', - '0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1', - '0 : 1 a.test.ts:8:20 › passes › outer 2.0', - '0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2', - '0 : 1 a.test.ts:8:20 › passes › outer 2.0', - '0 : 1 a.test.ts:3:11 › passes', + '#0 : 1 a.test.ts:3:11 › passes', + '#0 : 1 a.test.ts:4:20 › passes › outer 1.0', + '#0 : 1 a.test.ts:5:22 › passes › outer 1.0 › inner 1.1', + '#0 : 1 a.test.ts:4:20 › passes › outer 1.0', + '#0 : 1 a.test.ts:6:22 › passes › outer 1.0 › inner 1.2', + '#0 : 1 a.test.ts:4:20 › passes › outer 1.0', + '#0 : 1 a.test.ts:3:11 › passes', + '#0 : 1 a.test.ts:8:20 › passes › outer 2.0', + '#0 : 1 a.test.ts:9:22 › passes › outer 2.0 › inner 2.1', + '#0 : 1 a.test.ts:8:20 › passes › outer 2.0', + '#0 : 1 a.test.ts:10:22 › passes › outer 2.0 › inner 2.2', + '#0 : 1 a.test.ts:8:20 › passes › outer 2.0', + '#0 : 1 a.test.ts:3:11 › passes', + ]); + }); + + test('render steps in non-TTY mode', async ({ runInlineTest }) => { + const result = await runInlineTest({ + 'a.test.ts': ` + import { test, expect } from '@playwright/test'; + test('passes', async ({}) => { + await test.step('outer 1.0', async () => { + await test.step('inner 1.1', async () => {}); + await test.step('inner 1.2', async () => {}); + }); + await test.step('outer 2.0', async () => { + await test.step('inner 2.1', async () => {}); + await test.step('inner 2.2', async () => {}); + }); + }); + `, + }, { reporter: 'list' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_LIST_PRINT_STEPS: '1' }); + const text = result.output; + const lines = text.split('\n').filter(l => l.match(/^#.* :/)).map(l => l.replace(/[.\d]+m?s/, 'Xms')); + expect(lines).toEqual([ + '#0 : 1.1 a.test.ts:5:26 › passes › outer 1.0 › inner 1.1 (Xms)', + '#1 : 1.2 a.test.ts:6:26 › passes › outer 1.0 › inner 1.2 (Xms)', + '#2 : 1.3 a.test.ts:4:24 › passes › outer 1.0 (Xms)', + '#3 : 1.4 a.test.ts:9:26 › passes › outer 2.0 › inner 2.1 (Xms)', + '#4 : 1.5 a.test.ts:10:26 › passes › outer 2.0 › inner 2.2 (Xms)', + '#5 : 1.6 a.test.ts:8:24 › passes › outer 2.0 (Xms)', + `#6 : ${POSITIVE_STATUS_MARK} 1 a.test.ts:3:15 › passes (Xms)`, ]); }); @@ -156,13 +185,13 @@ for (const useIntermediateMergeReport of [false, true] as const) { `, }, { reporter: 'list', retries: '1' }, { PW_TEST_DEBUG_REPORTERS: '1', PLAYWRIGHT_FORCE_TTY: '80' }); const text = result.output; - const lines = text.split('\n').filter(l => l.startsWith('0 :') || l.startsWith('1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms')); + const lines = text.split('\n').filter(l => l.startsWith('#0 :') || l.startsWith('#1 :')).map(l => l.replace(/\d+(\.\d+)?m?s/, 'XXms')); expect(lines).toEqual([ - `0 : 1 a.test.ts:3:15 › flaky`, - `0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`, - `1 : 2 a.test.ts:3:15 › flaky (retry #1)`, - `1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`, + `#0 : 1 a.test.ts:3:15 › flaky`, + `#0 : ${NEGATIVE_STATUS_MARK} 1 a.test.ts:3:15 › flaky (XXms)`, + `#1 : 2 a.test.ts:3:15 › flaky (retry #1)`, + `#1 : ${POSITIVE_STATUS_MARK} 2 a.test.ts:3:15 › flaky (retry #1) (XXms)`, ]); });