From aa929bff3bf9a10ec08689e74e1bddbf4d10f77b Mon Sep 17 00:00:00 2001
From: Max Schmitt <max@schmitt.mx>
Date: Thu, 14 Sep 2023 19:51:05 +0200
Subject: [PATCH] test: unflake 'filter should update stats' test (#27085)

It's not possible to calculate the total test duration of the tests in
the UI, since our `msToString` function is lossy. This patch unflakes
the
[test](https://github.com/microsoft/playwright/actions/runs/6183832112/job/16787806162?pr=27074#step:7:254)
when it took longer than a second to start, by writing the test-ids to
the DOM, getting them into the Node.js process, and calculating their
test duration sum based on the JSON report.

Drive-by: I found a bug that we used the JSON report from the main
test-runner process rather than from the merge process, so the test IDs
were still old. (Without that change the `useIntermediateMergeReport`
tests were not passing.
---
 tests/playwright-test/reporter-html.spec.ts | 44 ++++++++++++++-------
 1 file changed, 30 insertions(+), 14 deletions(-)

diff --git a/tests/playwright-test/reporter-html.spec.ts b/tests/playwright-test/reporter-html.spec.ts
index e1391e0327..acebf3cb75 100644
--- a/tests/playwright-test/reporter-html.spec.ts
+++ b/tests/playwright-test/reporter-html.spec.ts
@@ -20,6 +20,7 @@ import url from 'url';
 import { test as baseTest, expect as baseExpect, createImage } from './playwright-test-fixtures';
 import type { HttpServer } from '../../packages/playwright-core/src/utils';
 import { startHtmlReportServer } from '../../packages/playwright/lib/reporters/html';
+import { msToString } from '../../packages/web/src/uiUtils';
 const { spawnAsync } = require('../../packages/playwright-core/lib/utils');
 
 const test = baseTest.extend<{ showReport: (reportFolder?: string) => Promise<void> }>({
@@ -1493,20 +1494,22 @@ for (const useIntermediateMergeReport of [false, true] as const) {
           'a.test.js': `
             const { expect, test } = require('@playwright/test');
             const names = ['one foo', 'two foo', 'three bar', 'four bar', 'five baz'];
-            names.forEach(name => {
-              test(name, async ({}) => {
+            for (const name of names) {
+              test('a-' + name, async ({}) => {
                 expect(name).not.toContain('foo');
+                await new Promise(f => setTimeout(f, 1100));
               });
-            });
+            }
           `,
           'b.test.js': `
             const { expect, test } = require('@playwright/test');
             const names = ['one foo', 'two foo', 'three bar', 'four bar', 'five baz'];
-            names.forEach(name => {
-              test(name, async ({}) => {
+            for (const name of names) {
+              test('b-' + name, async ({}) => {
                 expect(name).not.toContain('one');
+                await new Promise(f => setTimeout(f, 1100));
               });
-            });
+            }
           `,
         }, { reporter: 'dot,html' }, { PW_TEST_HTML_REPORT_OPEN: 'never' });
 
@@ -1516,14 +1519,27 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
         await showReport();
 
-        async function checkTotalDuration() {
+        function calculateTotalTestDuration(testNames: string[]) {
           let total = 0;
-          for (const text of await page.getByTestId('test-duration').allTextContents()) {
-            expect(text).toMatch(/\d+ms$/);
-            total += parseInt(text.substring(0, text.length - 2), 10);
+          for (const suite of result.report.suites) {
+            for (const spec of suite.specs) {
+              if (!testNames.includes(spec.title))
+                continue;
+              for (const test of spec.tests) {
+                for (const result of test.results)
+                  total += result.duration;
+              }
+            }
           }
-          const totalDuration = await page.getByTestId('overall-duration').textContent();
-          expect(totalDuration).toBe(`Total time: ${total}ms`);
+          return total;
+        }
+
+        async function checkTotalDuration(testNames: string[]) {
+          for (const testDuration of await page.getByTestId('test-duration').allTextContents())
+            expect(testDuration).toMatch(/\d+m?s$/);
+
+          const expectedTotalTimeInMs = calculateTotalTestDuration(testNames);
+          await expect(page.getByTestId('overall-duration')).toHaveText(`Total time: ${msToString(expectedTotalTimeInMs)}`);
         }
 
         const searchInput = page.locator('.subnav-search-input');
@@ -1532,7 +1548,7 @@ for (const useIntermediateMergeReport of [false, true] as const) {
         await searchInput.fill('s:failed');
 
         await expect(page.getByTestId('filtered-tests-count')).toHaveText('Filtered: 3');
-        await checkTotalDuration();
+        await checkTotalDuration(['a-one foo', 'a-two foo', 'b-one foo']);
         await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('10');
         await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('7');
         await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('3');
@@ -1544,7 +1560,7 @@ for (const useIntermediateMergeReport of [false, true] as const) {
 
         await searchInput.fill('foo');
         await expect(page.getByTestId('filtered-tests-count')).toHaveText('Filtered: 4');
-        await checkTotalDuration();
+        await checkTotalDuration(['a-one foo', 'a-two foo', 'b-one foo', 'b-two foo']);
         await expect(page.locator('.subnav-item:has-text("All") .counter')).toHaveText('10');
         await expect(page.locator('.subnav-item:has-text("Passed") .counter')).toHaveText('7');
         await expect(page.locator('.subnav-item:has-text("Failed") .counter')).toHaveText('3');