diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml index fdd2a5003a..343b0c9932 100644 --- a/.github/workflows/tests_primary.yml +++ b/.github/workflows/tests_primary.yml @@ -55,6 +55,11 @@ jobs: - run: npm run build - run: npx playwright install --with-deps ${{ matrix.browser }} chromium - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test -- --project=${{ matrix.browser }} + env: + PWTEST_BLOB_REPORT: 1 + - name: Upload artifacts to Azure Blob Storage + if: always() && github.event_name != 'pull_request' + run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" - run: node tests/config/checkCoverage.js ${{ matrix.browser }} - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() @@ -65,6 +70,29 @@ jobs: name: ${{ matrix.browser }}-${{ matrix.os }}-test-results path: test-results + merge_test_linux: + if: ${{ always() && github.event_name != 'pull_request' }} + needs: [test_linux] + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 16 + - name: Install dependencies + run: npm ci + - name: Download Blob Reports from Azure Blob Storage + run: | + az storage blob download-batch -d . -s '$web' --pattern 'run-${{ github.run_id }}-${{ github.sha }}-test_linux/*.jsonl' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + - name: Merge into HTML Report + run: | + npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_linux' + - name: Upload HTML Report to Azure Blob Storage + run: | + az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_linux' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_linux/index.html" + test_linux_chromium_tot: name: ${{ matrix.os }} (chromium tip-of-tree) strategy: @@ -103,9 +131,14 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] node-version: [16] + shard: [1/2, 2/2] include: - os: ubuntu-latest node-version: 18 + shard: [1/2] + - os: ubuntu-latest + node-version: 18 + shard: [2/2] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -118,13 +151,43 @@ jobs: DEBUG: pw:install - run: npm run build - run: npx playwright install --with-deps - - run: npm run ttest + - run: npm run ttest -- --shard ${{ matrix.shard }} + env: + PWTEST_BLOB_REPORT: 1 if: matrix.os != 'ubuntu-latest' - - run: xvfb-run npm run ttest + - run: xvfb-run npm run ttest -- --shard ${{ matrix.shard }} + env: + PWTEST_BLOB_REPORT: 1 if: matrix.os == 'ubuntu-latest' - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json if: always() shell: bash + - name: Upload artifacts to Azure Blob Storage + if: always() && github.event_name != 'pull_request' + run: az storage blob upload-batch -s test-results/blob-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + + merge_test_test_runner: + if: ${{ always() && github.event_name != 'pull_request' }} + needs: [test_test_runner] + timeout-minutes: 10 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: 16 + - name: Install dependencies + run: npm ci + - name: Download Blob Reports from Azure Blob Storage + run: | + az storage blob download-batch -d . -s '$web' --pattern 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner/*.jsonl' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + - name: Merge into HTML Report + run: | + npx playwright merge-reports --reporter html --attachments missing 'run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' + - name: Upload HTML Report to Azure Blob Storage + run: | + az storage blob upload-batch -s playwright-report -d '$web/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner' --connection-string "${{ secrets.AZURE_CONNECTION_STRING_FOR_BLOB_REPORT }}" + echo "Report url: https://mspwblobreport.z1.web.core.windows.net/run-${{ github.run_id }}-${{ github.sha }}-test_test_runner/index.html" test_web_components: name: Web Components diff --git a/tests/library/playwright.config.ts b/tests/library/playwright.config.ts index 90ba9e7c9b..3a4a619a4c 100644 --- a/tests/library/playwright.config.ts +++ b/tests/library/playwright.config.ts @@ -17,7 +17,7 @@ import { config as loadEnv } from 'dotenv'; loadEnv({ path: path.join(__dirname, '..', '..', '.env') }); -import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions } from '@playwright/test'; +import type { Config, PlaywrightTestOptions, PlaywrightWorkerOptions, ReporterDescription } from '@playwright/test'; import * as path from 'path'; import type { TestModeWorkerOptions } from '../config/testModeFixtures'; import type { TestModeName } from '../config/testMode'; @@ -42,6 +42,17 @@ const trace = !!process.env.PWTEST_TRACE; const outputDir = path.join(__dirname, '..', '..', 'test-results'); const testDir = path.join(__dirname, '..'); +const reporters = () => { + const result: ReporterDescription[] = process.env.CI ? [ + ['dot'], + ['json', { outputFile: path.join(outputDir, 'report.json') }], + ] : [ + ['html', { open: 'on-failure' }] + ]; + if (process.env.PWTEST_BLOB_REPORT === '1') + result.push(['blob', { outputDir: path.join(outputDir, 'blob-report') }]); + return result; +}; const config: Config = { testDir, outputDir, @@ -58,12 +69,7 @@ const config: Config { + const result: ReporterDescription[] = process.env.CI ? [ + ['dot'], + ['json', { outputFile: path.join(outputDir, 'report.json') }], + ] : [ + ['list'] + ]; + if (process.env.PWTEST_BLOB_REPORT === '1') + result.push(['blob', { outputDir: path.join(outputDir, 'blob-report') }]); + return result; +}; export default defineConfig({ timeout: 30000, forbidOnly: !!process.env.CI, @@ -39,10 +50,5 @@ export default defineConfig({ testIgnore: [path.join(__dirname, '../fixtures/**')], }, ], - reporter: process.env.CI ? [ - ['dot'], - ['json', { outputFile: path.join(outputDir, 'report.json') }], - ] : [ - ['list'] - ], + reporter: reporters(), });