test: run page tests on electron bot (#6122)

.github/workflows/tests.yml

@@ -225,8 +225,6 @@ jobs:
       matrix:
         shard: [1, 2]
     runs-on: macos-10.15
-    env:
-      PW_ANDROID_TESTS: 1
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-node@v2
@@ -364,3 +362,30 @@ jobs:
       with:
         name: edge-stable-win-test-results
         path: test-results
+
+  test_electron:
+    name: "Electron Linux"
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-node@v2
+      with:
+        node-version: 10
+    - run: npm ci
+    - run: npm run build
+    - run: node lib/cli/cli install-deps chromium
+    - run: mkdir -p coredumps
+    # Set core dump file name pattern
+    - run: sudo bash -c 'echo "$(pwd -P)/coredumps/core-pid_%p.dump" > /proc/sys/kernel/core_pattern'
+    # XVFB-RUN merges both STDOUT and STDERR, whereas we need only STDERR
+    # Wrap `npm run` in a subshell to redirect STDERR to file.
+    # Enable core dumps in the subshell.
+    - run: xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- bash -c "ulimit -c unlimited && npm run etest"
+    - run: node tests/config/checkCoverage.js electron
+    - run: ./utils/upload_flakiness_dashboard.sh ./test-results/report.json
+      if: always() && github.repository == 'microsoft/playwright' && (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release-'))
+    - uses: actions/upload-artifact@v1
+      if: ${{ always() }}
+      with:
+        name: electron-linux-test-results
+        path: test-results

package.json

@@ -12,7 +12,8 @@
     "ctest": "folio --config=tests/config/default.config.ts --tag=chromium",
     "ftest": "folio --config=tests/config/default.config.ts --tag=firefox",
     "wtest": "folio --config=tests/config/default.config.ts --tag=webkit",
-    "atest": "folio --config=tests/config/android.config.ts",
+    "atest": "cross-env PW_ANDROID_TESTS=1 folio --config=tests/config/android.config.ts",
+    "etest": "cross-env PW_ELECTRON_TESTS=1 folio --config=tests/config/electron.config.ts",
     "test": "folio --config=tests/config/default.config.ts",
     "eslint": "[ \"$CI\" = true ] && eslint --quiet -f codeframe --ext js,ts . || eslint --ext js,ts .",
     "tsc": "tsc -p .",

src/server/electron/electron.ts

@@ -79,7 +79,8 @@ export class ElectronApplication extends SdkObject {
     // Needs to be sync.
     const windowId = ++this._lastWindowId;
     page.on(Page.Events.Close, () => {
-      page.browserWindow.dispose();
+      if (page.browserWindow)
+        page.browserWindow.dispose();
       this._windows.delete(page);
     });
     page._browserWindowId = windowId;

tests/browsercontext-viewport.spec.ts

@@ -20,7 +20,8 @@ import { test as browserTest } from './config/browserTest';
 import { verifyViewport } from './config/utils';
 
 it.beforeEach(async () => {
-  it.skip(!!process.env.PW_ANDROID_TESTS);
+  it.skip(!!process.env.PW_ANDROID_TESTS, 'Default viewport is null');
+  it.skip(!!process.env.PW_ELECTRON_TESTS, 'Default viewport is null');
 });
 
 it('should get the proper default viewport size', async ({page, server}) => {

tests/chromium/chromium.spec.ts

@@ -23,6 +23,7 @@ pageTest.describe('chromium', () => {
   pageTest.beforeEach(async ({ browserName }) => {
     pageTest.skip(browserName !== 'chromium');
     pageTest.skip(!!process.env.PW_ANDROID_TESTS);
+    pageTest.skip(!!process.env.PW_ELECTRON_TESTS);
   });
 
   pageTest('should create a worker from a service worker', async ({page, server}) => {

tests/chromium/js-coverage.spec.ts

@@ -19,6 +19,7 @@ import { test as it, expect } from '../config/pageTest';
 it.describe('JS Coverage', () => {
   it.beforeEach(async ({ browserName }) => {
     it.skip(browserName !== 'chromium');
+    it.fixme(!!process.env.PW_ELECTRON_TESTS);
   });
 
   it('should work', async function({page, server}) {

tests/config/browserEnv.ts

@@ -37,6 +37,7 @@ type TestOptions = {
   mode: 'default' | 'driver' | 'service';
   video?: boolean;
   traceDir?: string;
+  coverageBrowserName?: string;
 };
 
 class DriverMode {
@@ -120,7 +121,7 @@ export class PlaywrightEnv implements Env<PlaywrightTestArgs> {
   }
 
   async beforeAll(workerInfo: WorkerInfo) {
-    this._coverage = installCoverageHooks(this._browserName);
+    this._coverage = installCoverageHooks(this._options.coverageBrowserName || this._browserName);
     require('../../lib/utils/utils').setUnderTest();
     this._playwright = await this._mode.setup(workerInfo);
     this._browserType = this._playwright[this._browserName];

tests/config/checkCoverage.js

@@ -51,9 +51,6 @@ if (browserName !== 'chromium') {
 if (browserName === 'webkit')
   api.delete('browserContext.clearPermissions');
 
-// Android coverage is abysmal.
-api = new Set(Array.from(api).filter(name => !name.toLowerCase().startsWith('android')));
-
 const coverageDir = path.join(__dirname, '..', 'coverage-report');
 
 const coveredMethods = new Set();

tests/config/coverage.js

@@ -64,9 +64,12 @@ function apiForBrowser(browserName) {
   const api = require('../../lib/client/api');
   const otherBrowsers = ['chromium', 'webkit', 'firefox'].filter(name => name.toLowerCase() !== browserName.toLowerCase());
   const filteredKeys = Object.keys(api).filter(apiName => {
+    if (apiName.toLowerCase().startsWith('android'))
+      return browserName === 'android';
     if (apiName.toLowerCase().startsWith('electron'))
-      return browserName === 'chromium';
-    return !otherBrowsers.some(otherName => apiName.toLowerCase().startsWith(otherName));
+      return browserName === 'electron';
+    return browserName !== 'electron' && browserName !== 'android' &&
+        !otherBrowsers.some(otherName => apiName.toLowerCase().startsWith(otherName));
   });
   const filteredAPI = {};
   for (const key of filteredKeys)

tests/config/default.config.ts

@@ -20,11 +20,9 @@ import { test as playwrightTest, slowTest as playwrightSlowTest } from './playwr
 import { test as browserTest, slowTest as browserSlowTest } from './browserTest';
 import { test as contextTest } from './contextTest';
 import { test as pageTest } from './pageTest';
-import { test as electronTest } from './electronTest';
 import { test as cliTest } from './cliTest';
 import { PlaywrightEnv, BrowserEnv, PageEnv, BrowserName } from './browserEnv';
 import { ServerEnv } from './serverEnv';
-import { ElectronEnv } from './electronEnv';
 import { CLIEnv } from './cliEnv';
 
 const config: folio.Config = {
@@ -80,6 +78,4 @@ for (const browserName of browsers) {
   contextTest.runWith(folio.merge(serverEnv, new PageEnv(browserName, options)), { tag: browserName });
   if (mode !== 'service')
     cliTest.runWith(folio.merge(serverEnv, new CLIEnv(browserName, options)), { tag: browserName });
-  if (browserName === 'chromium')
-    electronTest.runWith(folio.merge(serverEnv, new ElectronEnv({ mode })), { tag: browserName });
 }

tests/config/electron.config.ts

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as folio from 'folio';
+import * as path from 'path';
+import { test as electronTest } from './electronTest';
+import { test as pageTest } from './pageTest';
+import { ServerEnv } from './serverEnv';
+import { ElectronEnv, ElectronPageEnv } from './electronEnv';
+
+const config: folio.Config = {
+  testDir: path.join(__dirname, '..'),
+  outputDir: path.join(__dirname, '..', '..', 'test-results'),
+  timeout: 30000,
+  globalTimeout: 5400000,
+};
+if (process.env.CI) {
+  config.workers = 1;
+  config.forbidOnly = true;
+  config.retries = 3;
+}
+folio.setConfig(config);
+
+if (process.env.CI) {
+  folio.setReporters([
+    new folio.reporters.dot(),
+    new folio.reporters.json({ outputFile: path.join(__dirname, '..', '..', 'test-results', 'report.json') }),
+  ]);
+}
+
+const serverEnv = new ServerEnv();
+electronTest.runWith(folio.merge(serverEnv, new ElectronEnv()), { tag: 'electron' });
+pageTest.runWith(folio.merge(serverEnv, new ElectronPageEnv()), { tag: 'electron' });

tests/config/electronEnv.ts

@@ -19,21 +19,30 @@ import { PlaywrightEnv } from './browserEnv';
 import * as path from 'path';
 import { ElectronTestArgs } from './electronTest';
 import { ElectronApplication, Page } from '../../index';
+import { PageTestArgs } from './pageTest';
 
 export class ElectronEnv extends PlaywrightEnv implements Env<ElectronTestArgs> {
   private _electronApp: ElectronApplication | undefined;
   private _windows: Page[] = [];
 
-  constructor(options: { mode: 'default' | 'driver' | 'service' }) {
-    super('chromium', options);
+  constructor() {
+    super('chromium', { mode: 'default', coverageBrowserName: 'electron' });
+    // This env prevents 'Electron Security Policy' console message.
+    process.env['ELECTRON_DISABLE_SECURITY_WARNINGS'] = 'true';
   }
 
   private async _newWindow() {
     const [ window ] = await Promise.all([
       this._electronApp!.waitForEvent('window'),
       this._electronApp!.evaluate(electron => {
-        const window = new electron.BrowserWindow({ width: 800, height: 600 });
-        window.loadURL('data:text/html,<title>Hello World 1</title>');
+        const window = new electron.BrowserWindow({
+          width: 800,
+          height: 600,
+          // Sandboxed windows share process with their window.open() children
+          // and can script them. We use that heavily in our tests.
+          webPreferences: { sandbox: true }
+        });
+        window.loadURL('about:blank');
       })
     ]);
     this._windows.push(window);
@@ -63,3 +72,15 @@ export class ElectronEnv extends PlaywrightEnv implements Env<ElectronTestArgs>
     await super.afterEach(testInfo);
   }
 }
+
+export class ElectronPageEnv extends ElectronEnv implements Env<PageTestArgs> {
+  async beforeEach(testInfo: TestInfo) {
+    const result = await super.beforeEach(testInfo);
+    const page = await result.newWindow();
+    return {
+      ...result,
+      browserVersion: require('electron/package.json').version,
+      page,
+    };
+  }
+}

tests/elementhandle-owner-frame.spec.ts

@@ -74,6 +74,8 @@ it('should work for detached elements', async ({ page, server }) => {
 });
 
 it('should work for adopted elements', async ({ page, server }) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
+
   await page.goto(server.EMPTY_PAGE);
   const [popup] = await Promise.all([
     page.waitForEvent('popup'),

tests/elementhandle-query-selector.spec.ts

@@ -35,6 +35,8 @@ it('should return null for non-existing element', async ({page, server}) => {
 });
 
 it('should work for adopted elements', async ({page,server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
+
   await page.goto(server.EMPTY_PAGE);
   const [popup] = await Promise.all([
     page.waitForEvent('popup'),

tests/emulation-focus.spec.ts

@@ -24,6 +24,8 @@ it('should think that it is focused by default', async ({page}) => {
 });
 
 it('should think that all pages are focused', async ({page}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron');
+
   const page2 = await page.context().newPage();
   expect(await page.evaluate('document.hasFocus()')).toBe(true);
   expect(await page2.evaluate('document.hasFocus()')).toBe(true);
@@ -41,6 +43,8 @@ it('should focus popups by default', async ({page, server}) => {
 });
 
 it('should provide target for keyboard events', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron');
+
   const page2 = await page.context().newPage();
   await Promise.all([
     page.goto(server.PREFIX + '/input/textarea.html'),
@@ -64,6 +68,8 @@ it('should provide target for keyboard events', async ({page, server}) => {
 });
 
 it('should not affect mouse event target page', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron');
+
   const page2 = await page.context().newPage();
   function clickCounter() {
     document.onclick = () => window['clickCount']  = (window['clickCount'] || 0) + 1;
@@ -86,6 +92,8 @@ it('should not affect mouse event target page', async ({page, server}) => {
 });
 
 it('should change document.activeElement', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron');
+
   const page2 = await page.context().newPage();
   await Promise.all([
     page.goto(server.PREFIX + '/input/textarea.html'),
@@ -105,6 +113,7 @@ it('should change document.activeElement', async ({page, server}) => {
 it('should not affect screenshots', async ({page, server, browserName, headful}) => {
   it.skip(browserName === 'firefox' && headful);
   it.skip(!!process.env.PW_ANDROID_TESTS);
+  it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron');
 
   // Firefox headful produces a different image.
   const page2 = await page.context().newPage();

tests/frame-evaluate.spec.ts

@@ -98,6 +98,7 @@ it('should allow cross-frame element handles', async ({ page, server }) => {
 
 it('should not allow cross-frame element handles when frames do not script each other', async ({ page, server }) => {
   it.skip(!!process.env.PW_ANDROID_TESTS);
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
 
   await page.goto(server.EMPTY_PAGE);
   const frame = await attachFrame(page, 'frame1', server.CROSS_PROCESS_PREFIX + '/empty.html');

tests/frame-hierarchy.spec.ts

@@ -155,6 +155,8 @@ it('should report frame from-inside shadow DOM', async ({page, server}) => {
 });
 
 it('should report frame.name()', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
+
   await attachFrame(page, 'theFrameId', server.EMPTY_PAGE);
   await page.evaluate(url => {
     const frame = document.createElement('iframe');
@@ -177,6 +179,8 @@ it('should report frame.parent()', async ({page, server}) => {
 });
 
 it('should report different frame instance when frame re-attaches', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
+
   const frame1 = await attachFrame(page, 'frame1', server.EMPTY_PAGE);
   await page.evaluate(() => {
     window['frame'] = document.querySelector('#frame1');

tests/interception.spec.ts

@@ -113,6 +113,7 @@ it('should intercept network activity from worker', async function({page, server
 
 it('should intercept network activity from worker 2', async function({page, server}) {
   it.skip(!!process.env.PW_ANDROID_TESTS);
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
 
   const url = server.PREFIX + '/worker/worker.js';
   await page.route(url, route => {
@@ -130,6 +131,8 @@ it('should intercept network activity from worker 2', async function({page, serv
 });
 
 it('should work with regular expression passed from a different context', async ({page, server}) => {
+  it.skip(!!process.env.PW_ELECTRON_TESTS);
+
   const ctx = vm.createContext();
   const regexp = vm.runInContext('new RegExp("empty\\.html")', ctx);
   let intercepted = false;

tests/page-basic.spec.ts

@@ -132,6 +132,8 @@ it('should fail with error upon disconnect', async ({page}) => {
 });
 
 it('page.url should work', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
+
   expect(page.url()).toBe('about:blank');
   await page.goto(server.EMPTY_PAGE);
   expect(page.url()).toBe(server.EMPTY_PAGE);
@@ -182,6 +184,7 @@ it('page.frame should respect url', async function({page, server}) {
 
 it('should have sane user agent', async ({page, isChromium, isFirefox}) => {
   it.skip(!!process.env.PW_ANDROID_TESTS);
+  it.skip(!!process.env.PW_ELECTRON_TESTS);
 
   const userAgent = await page.evaluate(() => navigator.userAgent);
   const [

tests/page-dialog.spec.ts

@@ -28,6 +28,8 @@ it('should fire', async ({page, server}) => {
 });
 
 it('should allow accepting prompts', async ({page}) => {
+  it.skip(!!process.env.PW_ELECTRON_TESTS, 'prompt() is not a thing in electron');
+
   page.on('dialog', dialog => {
     expect(dialog.type()).toBe('prompt');
     expect(dialog.defaultValue()).toBe('yes.');
@@ -39,6 +41,8 @@ it('should allow accepting prompts', async ({page}) => {
 });
 
 it('should dismiss the prompt', async ({page}) => {
+  it.skip(!!process.env.PW_ELECTRON_TESTS, 'prompt() is not a thing in electron');
+
   page.on('dialog', dialog => {
     dialog.dismiss();
   });
@@ -101,6 +105,8 @@ it('should handle multiple confirms', async ({page}) => {
 });
 
 it('should auto-dismiss the prompt without listeners', async ({page}) => {
+  it.skip(!!process.env.PW_ELECTRON_TESTS, 'prompt() is not a thing in electron');
+
   const result = await page.evaluate(() => prompt('question?'));
   expect(result).toBe(null);
 });

tests/page-event-crash.spec.ts

@@ -71,6 +71,9 @@ it.describe('', () => {
   });
 
   it('should be able to close context when page crashes', async args => {
+    it.skip(!!process.env.PW_ANDROID_TESTS);
+    it.skip(!!process.env.PW_ELECTRON_TESTS);
+
     const { page } = args;
     await page.setContent(`<div>This page should crash</div>`);
     crash(args);

tests/page-event-pageerror.spec.ts

@@ -70,6 +70,7 @@ it('should handle object', async ({page, isChromium, isFirefox}) => {
 
 it('should handle window', async ({page, isChromium, isFirefox}) => {
   it.fixme(isFirefox);
+  it.skip(!!process.env.PW_ELECTRON_TESTS);
 
   // Firefox just does not report this error.
   const [error] = await Promise.all([

tests/page-expose-function.spec.ts

@@ -242,6 +242,7 @@ it('should work with internal bindings', async ({page, toImpl, server, mode, bro
   it.skip(mode !== 'default');
   it.skip(browserName !== 'chromium');
   it.skip(!!process.env.PW_ANDROID_TESTS);
+  it.skip(!!process.env.PW_ELECTRON_TESTS);
 
   const implPage: import('../src/server/page').Page = toImpl(page);
   let foo;

tests/page-goto.spec.ts

@@ -490,6 +490,8 @@ it('should fail when canceled by another navigation', async ({page, server}) =>
 });
 
 it('should work with lazy loading iframes', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
+
   await page.goto(server.PREFIX + '/frames/lazy-frame.html');
   expect(page.frames().length).toBe(2);
 });

tests/page-route.spec.ts

@@ -125,6 +125,7 @@ it('should contain referer header', async ({page, server}) => {
 
 it('should properly return navigation response when URL has cookies', async ({page, server}) => {
   it.skip(!!process.env.PW_ANDROID_TESTS);
+  it.fixme(!!process.env.PW_ELECTRON_TESTS, 'error: Browser context management is not supported.');
 
   // Setup cookie.
   await page.goto(server.EMPTY_PAGE);
@@ -535,7 +536,7 @@ it('should support cors with GET', async ({page, server}) => {
 });
 
 it('should support cors with POST', async ({page, server, isChromium, browserChannel}) => {
-  it.fail(isChromium && !browserChannel && !process.env.PW_ANDROID_TESTS, 'https://github.com/microsoft/playwright/issues/6016');
+  it.fail(isChromium && !browserChannel && !process.env.PW_ANDROID_TESTS && !process.env.PW_ELECTRON_TESTS, 'https://github.com/microsoft/playwright/issues/6016');
 
   await page.goto(server.EMPTY_PAGE);
   await page.route('**/cars', async route => {
@@ -559,7 +560,7 @@ it('should support cors with POST', async ({page, server, isChromium, browserCha
 });
 
 it('should support cors with credentials', async ({page, server, isChromium, browserChannel}) => {
-  it.fail(isChromium && !browserChannel && !process.env.PW_ANDROID_TESTS, 'https://github.com/microsoft/playwright/issues/6016');
+  it.fail(isChromium && !browserChannel && !process.env.PW_ANDROID_TESTS && !process.env.PW_ELECTRON_TESTS, 'https://github.com/microsoft/playwright/issues/6016');
 
   await page.goto(server.EMPTY_PAGE);
   await page.route('**/cars', async route => {
@@ -619,7 +620,7 @@ it('should reject cors with disallowed credentials', async ({page, server}) => {
 });
 
 it('should support cors for different methods', async ({page, server, isChromium, browserChannel}) => {
-  it.fail(isChromium && !browserChannel && !process.env.PW_ANDROID_TESTS, 'https://github.com/microsoft/playwright/issues/6016');
+  it.fail(isChromium && !browserChannel && !process.env.PW_ANDROID_TESTS && !process.env.PW_ELECTRON_TESTS, 'https://github.com/microsoft/playwright/issues/6016');
 
   await page.goto(server.EMPTY_PAGE);
   await page.route('**/cars', async (route, request) => {

tests/page-screenshot.spec.ts

@@ -146,6 +146,8 @@ it.describe('page screenshot', () => {
   });
 
   it('should render white background on jpeg file', async ({page, server}) => {
+    it.fixme(!!process.env.PW_ELECTRON_TESTS, 'omitBackground with jpeg does not work');
+
     await page.setViewportSize({ width: 100, height: 100 });
     await page.goto(server.EMPTY_PAGE);
     const screenshot = await page.screenshot({omitBackground: true, type: 'jpeg'});
@@ -223,6 +225,8 @@ it.describe('page screenshot', () => {
   });
 
   it('path option should detect jpeg', async ({page, server}, testInfo) => {
+    it.fixme(!!process.env.PW_ELECTRON_TESTS, 'omitBackground with jpeg does not work');
+
     await page.setViewportSize({ width: 100, height: 100 });
     await page.goto(server.EMPTY_PAGE);
     const outputPath = testInfo.outputPath('screenshot.jpg');

tests/page-wait-for-load-state.spec.ts

@@ -132,6 +132,8 @@ it('should work with clicking target=_blank', async ({page, server}) => {
 });
 
 it('should wait for load state of newPage', async ({page, server}) => {
+  it.fixme(!!process.env.PW_ELECTRON_TESTS, 'BrowserContext.newPage does not work in Electron');
+
   const [newPage] = await Promise.all([
     page.context().waitForEvent('page'),
     page.context().newPage(),

tests/selectors-misc.spec.ts

@@ -30,6 +30,7 @@ it('should work for open shadow roots', async ({page, server}) => {
 it('should click on links in shadow dom', async ({page, server, browserName, browserChannel}) => {
   it.fixme(browserName === 'chromium' && !!browserChannel, 'Enable when crrev.com/864024 get to the stable channel');
   it.fixme(!!process.env.PW_ANDROID_TESTS);
+  it.fixme(!!process.env.PW_ELECTRON_TESTS);
 
   await page.goto(server.PREFIX + '/shadow-dom-link.html');
   expect(await page.evaluate(() => (window as any).clickCount)).toBe(0);