devops: make headful a matrix, collect test results (#3027)

.github/workflows/tests.yml

@@ -38,26 +38,25 @@ jobs:
     # Wrap `npm run` in a subshell to redirect STDERR to file.
     # Enable core dumps in the subshell.
     - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && npm run jest -- --testTimeout=30000"
-      if: ${{ always() }}
       env:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "*,-pw:wrapped*"
         DEBUG_FILE: "testrun.log"
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: ${{ matrix.browser }}-linux-jest-report
+        path: jest-report.json  
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: ${{ matrix.browser }}-linux-output
         path: test/output-${{ matrix.browser }}
     - uses: actions/upload-artifact@v1
-      if: failure()
+      if: ${{ always() }}
       with:
         name: ${{ matrix.browser }}-linux-testrun.log
         path: testrun.log
-    - uses: actions/upload-artifact@v1
-      if: failure()
-      with:
-        name: ${{ matrix.browser }}-linux-coredumps
-        path: coredumps
 
   test_mac:
     name: "macOS"
@@ -79,13 +78,18 @@ jobs:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "*,-pw:wrapped*"
         DEBUG_FILE: "testrun.log"
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: ${{ matrix.browser }}-mac-jest-report
+        path: jest-report.json  
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: ${{ matrix.browser }}-mac-output
         path: test/output-${{ matrix.browser }}
     - uses: actions/upload-artifact@v1
-      if: failure()
+      if: ${{ always() }}
       with:
         name: ${{ matrix.browser }}-mac-testrun.log
         path: testrun.log
@@ -111,13 +115,18 @@ jobs:
         BROWSER: ${{ matrix.browser }}
         DEBUG: "*,-pw:wrapped*"
         DEBUG_FILE: "testrun.log"
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: ${{ matrix.browser }}-win-jest-report
+        path: jest-report.json
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: ${{ matrix.browser }}-win-output
         path: test/output-${{ matrix.browser }}
     - uses: actions/upload-artifact@v1
-      if: failure()
+      if: ${{ always() }}
       with:
         name: ${{ matrix.browser }}-win-testrun.log
         path: testrun.log
@@ -142,6 +151,10 @@ jobs:
 
   headful_linux:
     name: "Headful Linux"
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: [chromium, firefox, webkit]
     runs-on: ubuntu-18.04
     steps:
     - uses: actions/checkout@v2
@@ -160,31 +173,19 @@ jobs:
     - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && npm run jest -- --testTimeout=30000"
       if: ${{ always() }}
       env:
-        BROWSER: "chromium"
+        BROWSER: ${{ matrix.browser }}
-        HEADLESS: "false"
-        DEBUG_FILE: "testrun.log"
-    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && npm run jest -- --testTimeout=30000"
-      if: ${{ always() }}
-      env:
-        BROWSER: "firefox"
-        HEADLESS: "false"
-        DEBUG_FILE: "testrun.log"
-    - run: xvfb-run --auto-servernum -- bash -c "ulimit -c unlimited && npm run jest -- --testTimeout=30000"
-      if: ${{ always() }}
-      env:
-        BROWSER: "webkit"
         HEADLESS: "false"
         DEBUG_FILE: "testrun.log"
     - uses: actions/upload-artifact@v1
-      if: failure()
+      if: ${{ always() }}
       with:
-        name: headful-linux-testrun.log
+        name: headful-${{ matrix.browser }}-linux-jest-report
+        path: jest-report.json
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: headful-${{ matrix.browser }}-linux-testrun.log
         path: testrun.log
-    - uses: actions/upload-artifact@v1
-      if: failure()
-      with:
-        name: headful-linux-coredumps
-        path: coredumps
 
   rpc_linux:
     name: "RPC Linux"
@@ -214,18 +215,18 @@ jobs:
         DEBUG: "*,-pw:wrapped*"
         DEBUG_FILE: "testrun.log"
         PWCHANNEL: ${{ matrix.transport }}
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: rpc-${{ matrix.transport }}-${{ matrix.browser }}-linux-jest-report
+        path: jest-report.json
     - uses: actions/upload-artifact@v1
       if: failure()
       with:
         name: rpc-${{ matrix.transport }}-${{ matrix.browser }}-linux-output
         path: test/output-${{ matrix.browser }}
     - uses: actions/upload-artifact@v1
-      if: failure()
+      if: ${{ always() }}
       with:
         name: rpc-${{ matrix.transport }}-${{ matrix.browser }}-linux-testrun.log
         path: testrun.log
-    - uses: actions/upload-artifact@v1
-      if: failure()
-      with:
-        name: rpc-${{ matrix.transport }}-${{ matrix.browser }}-linux-coredumps
-        path: coredumps

.gitignore

@@ -14,3 +14,4 @@ yarn.lock
 /src/generated/*
 lib/
 /types/*
+jest-report.json

jest.config.js

@@ -7,5 +7,9 @@ module.exports = /** @type {import('@jest/types').Config.InitialOptions} */ ({
   testRunner: 'jest-circus/runner',
   testTimeout: 10000,
   globalSetup: './jest/setup.js',
-  globalTeardown: './jest/teardown.js'
+  globalTeardown: './jest/teardown.js',
+  reporters: [
+    'default',
+    './jest/reporter'
+  ]
 });

test/jest/reporter.js

@@ -0,0 +1,23 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const fs = require("fs")
+
+module.exports = function Reporter() {
+  this.onRunComplete = (test, runResults) => {
+    fs.writeFileSync('jest-report.json', JSON.stringify(runResults, undefined, 2));
+  }
+}