diff --git a/packages/playwright/src/common/suiteUtils.ts b/packages/playwright/src/common/suiteUtils.ts index c2140a8a96..81dc9d5465 100644 --- a/packages/playwright/src/common/suiteUtils.ts +++ b/packages/playwright/src/common/suiteUtils.ts @@ -90,7 +90,8 @@ export function applyRepeatEachIndex(project: FullProjectInternal, fileSuite: Su // Assign test properties with project-specific values. fileSuite.forEachTest((test, suite) => { if (repeatEachIndex) { - const testIdExpression = `[project=${project.id}]${test.titlePath().join('\x1e')} (repeat:${repeatEachIndex})`; + const [file, ...titles] = test.titlePath(); + const testIdExpression = `[project=${project.id}]${toPosixPath(file)}\x1e${titles.join('\x1e')} (repeat:${repeatEachIndex})`; const testId = suite._fileId + '-' + calculateSha1(testIdExpression).slice(0, 20); test.id = testId; test.repeatEachIndex = repeatEachIndex; diff --git a/packages/playwright/src/common/test.ts b/packages/playwright/src/common/test.ts index ec67168826..0d05ac5dfd 100644 --- a/packages/playwright/src/common/test.ts +++ b/packages/playwright/src/common/test.ts @@ -16,7 +16,6 @@ import type { FixturePool } from './fixtures'; import type * as reporterTypes from '../../types/testReporter'; -import type { SuitePrivate } from '../../types/reporterPrivate'; import type { TestTypeImpl } from './testType'; import { rootTestType } from './testType'; import type { Annotation, FixturesWithLocation, FullProjectInternal } from './config'; @@ -40,7 +39,7 @@ export type Modifier = { description: string | undefined }; -export class Suite extends Base implements SuitePrivate { +export class Suite extends Base { location?: Location; parent?: Suite; _use: FixturesWithLocation[] = []; diff --git a/packages/playwright/src/isomorphic/teleReceiver.ts b/packages/playwright/src/isomorphic/teleReceiver.ts index ce6834844e..9624a76876 100644 --- a/packages/playwright/src/isomorphic/teleReceiver.ts +++ b/packages/playwright/src/isomorphic/teleReceiver.ts @@ -17,7 +17,6 @@ import type { Annotation } from '../common/config'; import type { FullProject, Metadata } from '../../types/test'; import type * as reporterTypes from '../../types/testReporter'; -import type { SuitePrivate } from '../../types/reporterPrivate'; import type { ReporterV2 } from '../reporters/reporterV2'; import { StringInternPool } from './stringInternPool'; @@ -45,12 +44,15 @@ export type JsonProject = { metadata: Metadata; name: string; dependencies: string[]; + // This is relative to root dir. snapshotDir: string; + // This is relative to root dir. outputDir: string; repeatEach: number; retries: number; suites: JsonSuite[]; teardown?: string; + // This is relative to root dir. testDir: string; testIgnore: JsonPattern[]; testMatch: JsonPattern[]; @@ -58,13 +60,10 @@ export type JsonProject = { }; export type JsonSuite = { - type: 'root' | 'project' | 'file' | 'describe'; title: string; location?: JsonLocation; suites: JsonSuite[]; tests: JsonTestCase[]; - fileId: string | undefined; - parallelMode: 'none' | 'default' | 'serial' | 'parallel'; }; export type JsonTestCase = { @@ -73,6 +72,7 @@ export type JsonTestCase = { location: JsonLocation; retries: number; tags?: string[]; + repeatEachIndex: number; }; export type JsonTestEnd = { @@ -365,13 +365,11 @@ export class TeleReporterReceiver { for (const jsonSuite of jsonSuites) { let targetSuite = parent.suites.find(s => s.title === jsonSuite.title); if (!targetSuite) { - targetSuite = new TeleSuite(jsonSuite.title, jsonSuite.type); + targetSuite = new TeleSuite(jsonSuite.title, parent._type === 'project' ? 'file' : 'describe'); targetSuite.parent = parent; parent.suites.push(targetSuite); } targetSuite.location = this._absoluteLocation(jsonSuite.location); - targetSuite._fileId = jsonSuite.fileId; - targetSuite._parallelMode = jsonSuite.parallelMode; this._mergeSuitesInto(jsonSuite.suites, targetSuite); this._mergeTestsInto(jsonSuite.tests, targetSuite); } @@ -379,9 +377,9 @@ export class TeleReporterReceiver { private _mergeTestsInto(jsonTests: JsonTestCase[], parent: TeleSuite) { for (const jsonTest of jsonTests) { - let targetTest = this._reuseTestCases ? parent.tests.find(s => s.title === jsonTest.title) : undefined; + let targetTest = this._reuseTestCases ? parent.tests.find(s => s.title === jsonTest.title && s.repeatEachIndex === jsonTest.repeatEachIndex) : undefined; if (!targetTest) { - targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location)); + targetTest = new TeleTestCase(jsonTest.testId, jsonTest.title, this._absoluteLocation(jsonTest.location), jsonTest.repeatEachIndex); targetTest.parent = parent; parent.tests.push(targetTest); this._tests.set(targetTest.id, targetTest); @@ -412,14 +410,14 @@ export class TeleReporterReceiver { private _absolutePath(relativePath: string): string; private _absolutePath(relativePath?: string): string | undefined; private _absolutePath(relativePath?: string): string | undefined { - if (!relativePath) - return relativePath; + if (relativePath === undefined) + return; return this._stringPool.internString(this._rootDir + this._pathSeparator + relativePath); } } -export class TeleSuite implements SuitePrivate { +export class TeleSuite { title: string; location?: reporterTypes.Location; parent?: TeleSuite; @@ -428,7 +426,6 @@ export class TeleSuite implements SuitePrivate { tests: TeleTestCase[] = []; _timeout: number | undefined; _retries: number | undefined; - _fileId: string | undefined; _project: TeleFullProject | undefined; _parallelMode: 'none' | 'default' | 'serial' | 'parallel' = 'none'; readonly _type: 'root' | 'project' | 'file' | 'describe'; @@ -482,10 +479,11 @@ export class TeleTestCase implements reporterTypes.TestCase { resultsMap = new Map(); - constructor(id: string, title: string, location: reporterTypes.Location) { + constructor(id: string, title: string, location: reporterTypes.Location, repeatEachIndex: number) { this.id = id; this.title = title; this.location = location; + this.repeatEachIndex = repeatEachIndex; } titlePath(): string[] { diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts index 91f34faf74..fff2703208 100644 --- a/packages/playwright/src/reporters/base.ts +++ b/packages/playwright/src/reporters/base.ts @@ -17,7 +17,6 @@ import { colors as realColors, ms as milliseconds, parseStackTraceLine } from 'playwright-core/lib/utilsBundle'; import path from 'path'; import type { FullConfig, TestCase, Suite, TestResult, TestError, FullResult, TestStep, Location } from '../../types/testReporter'; -import type { SuitePrivate } from '../../types/reporterPrivate'; import { getPackageManagerExecCommand } from 'playwright-core/lib/utils'; import type { ReporterV2 } from './reporterV2'; export type TestResultOutput = { chunk: string | Buffer, type: 'stdout' | 'stderr' }; @@ -72,7 +71,7 @@ export class BaseReporter implements ReporterV2 { suite!: Suite; totalTestCount = 0; result!: FullResult; - private fileDurations = new Map(); + private fileDurations = new Map }>(); private _omitFailures: boolean; private _fatalErrors: TestError[] = []; private _failureCount: number = 0; @@ -115,16 +114,13 @@ export class BaseReporter implements ReporterV2 { onTestEnd(test: TestCase, result: TestResult) { if (result.status !== 'skipped' && result.status !== test.expectedStatus) ++this._failureCount; - // Ignore any tests that are run in parallel. - for (let suite: Suite | undefined = test.parent; suite; suite = suite.parent) { - if ((suite as SuitePrivate)._parallelMode === 'parallel') - return; - } const projectName = test.titlePath()[1]; const relativePath = relativeTestPath(this.config, test); const fileAndProject = (projectName ? `[${projectName}] › ` : '') + relativePath; - const duration = this.fileDurations.get(fileAndProject) || 0; - this.fileDurations.set(fileAndProject, duration + result.duration); + const entry = this.fileDurations.get(fileAndProject) || { duration: 0, workers: new Set() }; + entry.duration += result.duration; + entry.workers.add(result.workerIndex); + this.fileDurations.set(fileAndProject, entry); } onError(error: TestError) { @@ -167,7 +163,8 @@ export class BaseReporter implements ReporterV2 { protected getSlowTests(): [string, number][] { if (!this.config.reportSlowTests) return []; - const fileDurations = [...this.fileDurations.entries()]; + // Only pick durations that were served by single worker. + const fileDurations = [...this.fileDurations.entries()].filter(([key, value]) => value.workers.size === 1).map(([key, value]) => [key, value.duration]) as [string, number][]; fileDurations.sort((a, b) => b[1] - a[1]); const count = Math.min(fileDurations.length, this.config.reportSlowTests.max || Number.POSITIVE_INFINITY); const threshold = this.config.reportSlowTests.threshold; diff --git a/packages/playwright/src/reporters/html.ts b/packages/playwright/src/reporters/html.ts index 3f21de3cde..4184273e1d 100644 --- a/packages/playwright/src/reporters/html.ts +++ b/packages/playwright/src/reporters/html.ts @@ -22,7 +22,6 @@ import type { TransformCallback } from 'stream'; import { Transform } from 'stream'; import { codeFrameColumns } from '../transform/babelBundle'; import type { FullResult, FullConfig, Location, Suite, TestCase as TestCasePublic, TestResult as TestResultPublic, TestStep as TestStepPublic, TestError } from '../../types/testReporter'; -import type { SuitePrivate } from '../../types/reporterPrivate'; import { HttpServer, assert, calculateSha1, copyFileAndMakeWritable, gracefullyProcessExitDoNotHang, removeFolders, sanitizeForFilePath, toPosixPath } from 'playwright-core/lib/utils'; import { colors, formatError, formatResultFailure, stripAnsiEscapes } from './base'; import { resolveReporterOutputPath } from '../util'; @@ -227,9 +226,12 @@ class HtmlBuilder { async build(metadata: Metadata, projectSuites: Suite[], result: FullResult, topLevelErrors: TestError[]): Promise<{ ok: boolean, singleTestId: string | undefined }> { const data = new Map(); for (const projectSuite of projectSuites) { + const testDir = projectSuite.project()!.testDir; for (const fileSuite of projectSuite.suites) { const fileName = this._relativeLocation(fileSuite.location)!.file; - const fileId = (fileSuite as SuitePrivate)._fileId!; + // Preserve file ids computed off the testDir. + const relativeFile = path.relative(testDir, fileSuite.location!.file); + const fileId = calculateSha1(toPosixPath(relativeFile)).slice(0, 20); let fileEntry = data.get(fileId); if (!fileEntry) { fileEntry = { @@ -343,19 +345,23 @@ class HtmlBuilder { private _processJsonSuite(suite: Suite, fileId: string, projectName: string, botName: string | undefined, path: string[], outTests: TestEntry[]) { const newPath = [...path, suite.title]; suite.suites.forEach(s => this._processJsonSuite(s, fileId, projectName, botName, newPath, outTests)); - suite.tests.forEach(t => outTests.push(this._createTestEntry(t, projectName, botName, newPath))); + suite.tests.forEach(t => outTests.push(this._createTestEntry(fileId, t, projectName, botName, newPath))); } - private _createTestEntry(test: TestCasePublic, projectName: string, botName: string | undefined, path: string[]): TestEntry { + private _createTestEntry(fileId: string, test: TestCasePublic, projectName: string, botName: string | undefined, path: string[]): TestEntry { const duration = test.results.reduce((a, r) => a + r.duration, 0); const location = this._relativeLocation(test.location)!; path = path.slice(1); + const [file, ...titles] = test.titlePath(); + const testIdExpression = `[project=${projectName}]${toPosixPath(file)}\x1e${titles.join('\x1e')} (repeat:${test.repeatEachIndex})`; + const testId = fileId + '-' + calculateSha1(testIdExpression).slice(0, 20); + const results = test.results.map(r => this._createTestResult(test, r)); return { testCase: { - testId: test.id, + testId, title: test.title, projectName, botName, @@ -370,7 +376,7 @@ class HtmlBuilder { ok: test.outcome() === 'expected' || test.outcome() === 'flaky', }, testCaseSummary: { - testId: test.id, + testId, title: test.title, projectName, botName, diff --git a/packages/playwright/src/reporters/teleEmitter.ts b/packages/playwright/src/reporters/teleEmitter.ts index 7f4caf6ad5..e6bc49ee3e 100644 --- a/packages/playwright/src/reporters/teleEmitter.ts +++ b/packages/playwright/src/reporters/teleEmitter.ts @@ -16,10 +16,8 @@ import path from 'path'; import { createGuid } from 'playwright-core/lib/utils'; -import type { SuitePrivate } from '../../types/reporterPrivate'; -import type { FullConfig, FullResult, Location, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; +import type { FullConfig, FullResult, Location, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter'; import { FullConfigInternal, getProjectId } from '../common/config'; -import type { Suite } from '../common/test'; import type { JsonAttachment, JsonConfig, JsonEvent, JsonFullResult, JsonProject, JsonStdIOType, JsonSuite, JsonTestCase, JsonTestEnd, JsonTestResultEnd, JsonTestResultStart, JsonTestStepEnd, JsonTestStepStart } from '../isomorphic/teleReceiver'; import { serializeRegexPatterns } from '../isomorphic/teleReceiver'; import type { ReporterV2 } from './reporterV2'; @@ -185,10 +183,7 @@ export class TeleReporterEmitter implements ReporterV2 { private _serializeSuite(suite: Suite): JsonSuite { const result = { - type: suite._type, title: suite.title, - fileId: (suite as SuitePrivate)._fileId, - parallelMode: (suite as SuitePrivate)._parallelMode, location: this._relativeLocation(suite.location), suites: suite.suites.map(s => this._serializeSuite(s)), tests: suite.tests.map(t => this._serializeTest(t)), @@ -203,6 +198,7 @@ export class TeleReporterEmitter implements ReporterV2 { location: this._relativeLocation(test.location), retries: test.retries, tags: test.tags, + repeatEachIndex: test.repeatEachIndex, }; } diff --git a/packages/playwright/types/reporterPrivate.ts b/packages/playwright/types/reporterPrivate.ts deleted file mode 100644 index 198b2fe9ea..0000000000 --- a/packages/playwright/types/reporterPrivate.ts +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import type { Suite } from './testReporter'; - -export interface SuitePrivate extends Suite { - _fileId: string | undefined; - _parallelMode: 'none' | 'default' | 'serial' | 'parallel'; -}