chore: run primary tests with blob reporter (#23076)

.github/workflows/tests_primary.yml

@@ -55,6 +55,11 @@ jobs:
     - run: npm run build
     - run: npx playwright install --with-deps ${{ matrix.browser }} chromium
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }}
+      env:
+        PWTEST_BLOB_REPORT: 1
+    - name: Upload artifacts to Azure Blob Storage
+      if: always() && github.event_name != 'pull_request'
+      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
     - run: node tests/config/checkCoverage.js ${{ matrix.browser }}
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
@@ -65,6 +70,29 @@ jobs:
         name: ${{ matrix.browser }}-${{ matrix.os }}-test-results
         path: test-results
 
+  merge_test_linux:
+    if: ${{ always() && github.event_name != 'pull_request' }}
+    needs: [test_linux]
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+    - name: Install dependencies
+      run: npm ci
+    - name: Download Blob Reports from Azure Blob Storage
+      run: |
+        az storage blob download-batch -d . -s '$web'  --pattern 'run-${{ github.run_id }}-${{ github.sha }}-test_linux/*.jsonl' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+    - name: Merge into HTML Report
+      run: |
+        npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_linux'
+    - name: Upload HTML Report to Azure Blob Storage
+      run: |
+        az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_linux/index.html"
+
   test_linux_chromium_tot:
     name: ${{ matrix.os }} (chromium tip-of-tree)
     strategy:
@@ -103,9 +131,14 @@ jobs:
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
         node-version: [16]
+        shard: [1/2, 2/2]
         include:
           - os: ubuntu-latest
             node-version: 18
+            shard: [1/2]
+          - os: ubuntu-latest
+            node-version: 18
+            shard: [2/2]
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v3
@@ -118,13 +151,43 @@ jobs:
         DEBUG: pw:install
     - run: npm run build
     - run: npx playwright install --with-deps
-    - run: npm run ttest
+    - run: npm run ttest -- --shard ${{ matrix.shard }}
+      env:
+        PWTEST_BLOB_REPORT: 1
       if: matrix.os != 'ubuntu-latest'
-    - run: xvfb-run npm run ttest
+    - run: xvfb-run npm run ttest --  --shard ${{ matrix.shard }}
+      env:
+        PWTEST_BLOB_REPORT: 1
       if: matrix.os == 'ubuntu-latest'
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always()
       shell: bash
+    - name: Upload artifacts to Azure Blob Storage
+      if: always() && github.event_name != 'pull_request'
+      run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+
+  merge_test_test_runner:
+    if: ${{ always() && github.event_name != 'pull_request' }}
+    needs: [test_test_runner]
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+    - name: Install dependencies
+      run: npm ci
+    - name: Download Blob Reports from Azure Blob Storage
+      run: |
+        az storage blob download-batch -d . -s '$web'  --pattern 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner/*.jsonl' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+    - name: Merge into HTML Report
+      run: |
+        npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner'
+    - name: Upload HTML Report to Azure Blob Storage
+      run: |
+        az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}"
+        echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner/index.html"
 
   test_web_components:
     name: Web Components

tests/library/playwright.config.ts

@@ -17,7 +17,7 @@
 import { config as loadEnv } from 'dotenv';
 loadEnv({ path: path.join(__dirname, '..', '..', '.env') });
 
-import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test';
+import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions, ReporterDescription } from '@playwright/test';
 import * as path from 'path';
 import type { TestModeWorkerOptions } from '../config/testModeFixtures';
 import type { TestModeName } from '../config/testMode';
@@ -42,6 +42,17 @@ const trace = !!process.env.PWTEST_TRACE;
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
 const testDir = path.join(__dirname, '..');
+const reporters = () => {
+  const result: ReporterDescription[] = process.env.CI ? [
+    ['dot'],
+    ['json', { outputFile: path.join(outputDir, 'report.json') }],
+  ] : [
+    ['html', { open: 'on-failure' }]
+  ];
+  if (process.env.PWTEST_BLOB_REPORT === '1')
+    result.push(['blob', { outputDir: path.join(outputDir, 'blob-report') }]);
+  return result;
+};
 const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & PlaywrightTestOptions & TestModeWorkerOptions> = {
   testDir,
   outputDir,
@@ -58,12 +69,7 @@ const config: Config<CoverageWorkerOptions & PlaywrightWorkerOptions & Playwrigh
   forbidOnly: !!process.env.CI,
   preserveOutput: process.env.CI ? 'failures-only' : 'always',
   retries: process.env.CI ? 3 : 0,
-  reporter: process.env.CI ? [
-    ['dot'],
-    ['json', { outputFile: path.join(outputDir, 'report.json') }],
-  ] : [
-    ['html', { open: 'on-failure' }]
-  ],
+  reporter: reporters(),
   projects: [],
   use: {
     connectOptions: mode === 'service' ? {

tests/playwright-test/playwright.config.ts

@@ -17,10 +17,21 @@
 import { config as loadEnv } from 'dotenv';
 loadEnv({ path: path.join(__dirname, '..', '..', '.env') });
 
-import { defineConfig } from './stable-test-runner';
+import { defineConfig, type ReporterDescription } from './stable-test-runner';
 import * as path from 'path';
 
 const outputDir = path.join(__dirname, '..', '..', 'test-results');
+const reporters = () => {
+  const result: ReporterDescription[] = process.env.CI ? [
+    ['dot'],
+    ['json', { outputFile: path.join(outputDir, 'report.json') }],
+  ] : [
+    ['list']
+  ];
+  if (process.env.PWTEST_BLOB_REPORT === '1')
+    result.push(['blob', { outputDir: path.join(outputDir, 'blob-report') }]);
+  return result;
+};
 export default defineConfig({
   timeout: 30000,
   forbidOnly: !!process.env.CI,
@@ -39,10 +50,5 @@ export default defineConfig({
       testIgnore: [path.join(__dirname, '../fixtures/**')],
     },
   ],
-  reporter: process.env.CI ? [
-    ['dot'],
-    ['json', { outputFile: path.join(outputDir, 'report.json') }],
-  ] : [
-    ['list']
-  ],
+  reporter: reporters(),
 });