test: migrate more page tests to folio (#6062)

.github/workflows/tests.yml

@@ -38,12 +38,12 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
-      env:
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
       env:
         BROWSER: ${{ matrix.browser }}
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
+      env:
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     # Checking coverage across two test suites is hard. Temporary disabled.
     # - run: node test/checkCoverage.js
@@ -73,12 +73,12 @@ jobs:
     - run: npm ci
     - run: npm run build
     - run: node lib/cli/cli install-deps ${{ matrix.browser }} chromium
-    - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
-      env:
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
       env:
         BROWSER: ${{ matrix.browser }}
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
+      env:
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
@@ -106,14 +106,14 @@ jobs:
     - run: npm ci
     - run: npm run build
     - run: node lib/cli/cli install-deps
-    - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
-      shell: bash
-      env:
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
       shell: bash
       env:
         BROWSER: ${{ matrix.browser }}
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: npm run folio -- ${{ matrix.browser }} --reporter=dot,json
+      shell: bash
+      env:
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
@@ -164,15 +164,15 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
-      if: ${{ always() }}
-      env:
-        HEADFUL: 1
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
       if: ${{ always() }}
       env:
         BROWSER: ${{ matrix.browser }}
+        HEADFUL: 1
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
+      if: ${{ always() }}
+      env:
         HEADFUL: 1
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
@@ -204,13 +204,13 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
-      env:
-        PWMODE: "${{ matrix.mode }}"
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json"
       env:
         BROWSER: "chromium"
+        PWMODE: "${{ matrix.mode }}"
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
+      env:
         PWMODE: "${{ matrix.mode }}"
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
@@ -242,13 +242,13 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
-      env:
-        PWVIDEO: 1
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json -p video"
       env:
         BROWSER: ${{ matrix.browser }}
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- ${{ matrix.browser }} --reporter=dot,json"
+      env:
+        PWVIDEO: 1
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
@@ -259,7 +259,11 @@ jobs:
         path: test-results
 
   test_android:
-    name: Android Emulator
+    name: Android Emulator (shard ${{ matrix.shard }})
+    strategy:
+      fail-fast: false
+      matrix:
+        shard: [1, 2]
     runs-on: macos-10.15
     env:
       FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
@@ -277,7 +281,7 @@ jobs:
     - name: Start Android Emulator
       run: utils/avd_start.sh
     - name: Run tests
-      run: npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts --reporter=dot,json
+      run: npm run build-folio && node tests/folio/cli.js --config=tests/config/android.config.ts --reporter=dot,json --shard=${{ matrix.shard }}/2
     - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
       if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
     - uses: actions/upload-artifact@v1
@@ -311,13 +315,13 @@ jobs:
     # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
-    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
-      env:
-        PW_CHROMIUM_CHANNEL: "chrome"
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npx folio test/ --workers=1 --forbid-only --timeout=60000 --global-timeout=5400000 --retries=3 --reporter=dot,json"
       env:
         BROWSER: "chromium"
+        PW_CHROMIUM_CHANNEL: "chrome"
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run folio -- chromium --reporter=dot,json"
+      env:
         PW_CHROMIUM_CHANNEL: "chrome"
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - uses: actions/upload-artifact@v1
@@ -343,15 +347,15 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run folio -- chromium --reporter=dot,json
-      shell: bash
-      env:
-        PW_CHROMIUM_CHANNEL: "chrome"
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
       shell: bash
       env:
         BROWSER: "chromium"
+        PW_CHROMIUM_CHANNEL: "chrome"
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: npm run folio -- chromium --reporter=dot,json
+      shell: bash
+      env:
         PW_CHROMIUM_CHANNEL: "chrome"
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - uses: actions/upload-artifact@v1
@@ -374,13 +378,13 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run folio -- chromium --reporter=dot,json
-      env:
-        PW_CHROMIUM_CHANNEL: "chrome"
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
       env:
         BROWSER: "chromium"
+        PW_CHROMIUM_CHANNEL: "chrome"
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: npm run folio -- chromium --reporter=dot,json
+      env:
         PW_CHROMIUM_CHANNEL: "chrome"
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - uses: actions/upload-artifact@v1
@@ -406,15 +410,15 @@ jobs:
     - run: npm run build
     # This only created problems, should we move ffmpeg back into npm?
     - run: node lib/cli/cli install ffmpeg
-    - run: npm run folio -- chromium --reporter=dot,json
-      shell: bash
-      env:
-        PW_CHROMIUM_CHANNEL: "msedge"
-        FOLIO_JSON_OUTPUT_NAME: "test-results/report-new.json"
     - run: npx folio test/ --workers=1 --forbid-only --global-timeout=5400000 --retries=3 --reporter=dot,json
       shell: bash
       env:
         BROWSER: "chromium"
+        PW_CHROMIUM_CHANNEL: "msedge"
+        FOLIO_JSON_OUTPUT_NAME: "test-results/report-old.json"
+    - run: npm run folio -- chromium --reporter=dot,json
+      shell: bash
+      env:
         PW_CHROMIUM_CHANNEL: "msedge"
         FOLIO_JSON_OUTPUT_NAME: "test-results/report.json"
     - uses: actions/upload-artifact@v1