test: gardening (#18623)

- Separate worker for some Android tests.
- Use png comparator for some screenshots tests instead of buffer
equality.
- Skip drag&drop tests on Android.
- Various timeout fixes.

tests/android/android.spec.ts

@@ -0,0 +1,58 @@
+/**
+ * Copyright 2020 Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import net from 'net';
+import { androidTest as test, expect } from './androidTest';
+
+// Force a separate worker to avoid messing up with `androidDevice` fixture.
+test.use({ launchOptions: {} });
+
+test('androidDevice.close', async function({ playwright }) {
+  const devices = await playwright._android.devices();
+  expect(devices.length).toBe(1);
+  const device = devices[0];
+  const events = [];
+  device.on('close', () => events.push('close'));
+  await device.close();
+  await device.close();
+  expect(events).toEqual(['close']);
+});
+
+test('should be able to use a custom port', async function({ playwright }) {
+  const proxyPort = 5038;
+  let countOfIncomingConnections = 0;
+  let countOfConnections = 0;
+  const server = net.createServer(socket => {
+    ++countOfIncomingConnections;
+    ++countOfConnections;
+    socket.on('close', () => countOfConnections--);
+    const client = net.connect(5037);
+    socket.pipe(client).pipe(socket);
+  });
+  await new Promise<void>(resolve => server.listen(proxyPort, resolve));
+
+  const devices = await playwright._android.devices({ port: proxyPort });
+  expect(countOfIncomingConnections).toBeGreaterThanOrEqual(1);
+  expect(devices).toHaveLength(1);
+  const device = devices[0];
+  const value = await device.shell('echo foobar');
+  expect(value.toString()).toBe('foobar\n');
+  await device.close();
+
+  await new Promise(resolve => server.close(resolve));
+  expect(countOfIncomingConnections).toBeGreaterThanOrEqual(1);
+  expect(countOfConnections).toBe(0);
+});

tests/android/browser.spec.ts

@@ -14,7 +14,6 @@
  * limitations under the License.
  */
 
-import net from 'net';
 import { androidTest as test, expect } from './androidTest';
 
 test.afterAll(async ({ androidDevice }) => {
@@ -62,32 +61,6 @@ test('should be able to send CDP messages', async ({ androidDevice }) => {
   await context.close();
 });
 
-test('should be able to use a custom port', async function({ playwright }) {
-  const proxyPort = 5038;
-  let countOfIncomingConnections = 0;
-  let countOfConnections = 0;
-  const server = net.createServer(socket => {
-    ++countOfIncomingConnections;
-    ++countOfConnections;
-    socket.on('close', () => countOfConnections--);
-    const client = net.connect(5037);
-    socket.pipe(client).pipe(socket);
-  });
-  await new Promise<void>(resolve => server.listen(proxyPort, resolve));
-
-  const devices = await playwright._android.devices({ port: proxyPort });
-  expect(countOfIncomingConnections).toBeGreaterThanOrEqual(1);
-  expect(devices).toHaveLength(1);
-  const device = devices[0];
-  const value = await device.shell('echo foobar');
-  expect(value.toString()).toBe('foobar\n');
-  await device.close();
-
-  await new Promise(resolve => server.close(resolve));
-  expect(countOfIncomingConnections).toBeGreaterThanOrEqual(1);
-  expect(countOfConnections).toBe(0);
-});
-
 test('should be able to pass context options', async ({ androidDevice, httpsServer }) => {
   const context = await androidDevice.launchBrowser({
     colorScheme: 'dark',

tests/android/device.spec.ts

@@ -92,14 +92,3 @@ test('androidDevice.options.omitDriverInstall', async function({ playwright }) {
 
   expect(fillStatus).toBe('success');
 });
-
-test('androidDevice.close', async function({ playwright }) {
-  const devices = await playwright._android.devices();
-  expect(devices.length).toBe(1);
-  const device = devices[0];
-  const events = [];
-  device.on('close', () => events.push('close'));
-  await device.close();
-  await device.close();
-  expect(events).toEqual(['close']);
-});

tests/android/launch-server.spec.ts

@@ -17,6 +17,9 @@
 import ws from 'ws';
 import { androidTest as test, expect } from './androidTest';
 
+// Force a separate worker to avoid messing up with `androidDevice` fixture.
+test.use({ launchOptions: {} });
+
 test('android.launchServer should connect to a device', async ({ playwright }) => {
   const browserServer = await playwright._android.launchServer();
   const device = await playwright._android.connect(browserServer.wsEndpoint());

tests/library/tracing.spec.ts

@@ -195,7 +195,7 @@ test('should not include trace resources from the provious chunks', async ({ con
   await page.setContent('<button>Click</button>');
   await page.click('"Click"');
   // Give it enough time for both screenshots to get into the trace.
-  await new Promise(f => setTimeout(f, 1000));
+  await new Promise(f => setTimeout(f, 3000));
   await context.tracing.stopChunk({ path: testInfo.outputPath('trace1.zip') });
 
   await context.tracing.startChunk();

tests/page/page-drag.spec.ts

@@ -20,6 +20,7 @@ import { attachFrame } from '../config/utils';
 
 it.describe('Drag and drop', () => {
   it.skip(({ browserName, browserMajorVersion }) => browserName === 'chromium' && browserMajorVersion < 91);
+  it.skip(({ isAndroid }) => isAndroid, 'No drag&drop on Android.');
 
   it('should work @smoke', async ({ page, server }) => {
     await page.goto(server.PREFIX + '/drag-n-drop.html');

tests/page/page-screenshot.spec.ts

@@ -20,6 +20,7 @@ import { verifyViewport, attachFrame } from '../config/utils';
 import type { Route } from 'playwright-core';
 import path from 'path';
 import fs from 'fs';
+import { comparePNGs } from '../config/comparator';
 
 it.describe('page screenshot', () => {
   it.skip(({ browserName, headless }) => browserName === 'firefox' && !headless, 'Firefox headed produces a different image.');
@@ -613,7 +614,7 @@ it.describe('page screenshot animations', () => {
     const buffer2 = await page.screenshot({
       animations: 'disabled',
     });
-    expect(buffer1.equals(buffer2)).toBe(true);
+    expect(comparePNGs(buffer1, buffer2)).toBe(null);
   });
 
   it('should resume infinite animations', async ({ page, server }) => {
@@ -624,7 +625,7 @@ it.describe('page screenshot animations', () => {
     const buffer1 = await page.screenshot();
     await rafraf(page);
     const buffer2 = await page.screenshot();
-    expect(buffer1.equals(buffer2)).toBe(false);
+    expect(comparePNGs(buffer1, buffer2, { threshold: 0.2, maxDiffPixels: 50 })).not.toBe(null);
   });
 
   it('should not capture infinite web animations', async ({ page, server }) => {
@@ -644,7 +645,7 @@ it.describe('page screenshot animations', () => {
     const buffer1 = await page.screenshot();
     await rafraf(page);
     const buffer2 = await page.screenshot();
-    expect(buffer1.equals(buffer2)).toBe(false);
+    expect(comparePNGs(buffer1, buffer2, { threshold: 0.2, maxDiffPixels: 50 })).not.toBe(null);
   });
 
   it('should fire transitionend for finite transitions', async ({ page, server }) => {

tests/page/page-wait-for-navigation.spec.ts

@@ -156,6 +156,8 @@ it('should work with DOM history.back()/history.forward()', async ({ page, serve
 });
 
 it('should work when subframe issues window.stop()', async ({ browserName, page, server }) => {
+  it.fixme(browserName === 'webkit', 'WebKit issues load event in some cases, but not always');
+
   server.setRoute('/frames/style.css', (req, res) => {});
   let done = false;
   page.goto(server.PREFIX + '/frames/one-frame.html').then(() => done = true).catch(() => {});
@@ -165,8 +167,7 @@ it('should work when subframe issues window.stop()', async ({ browserName, page,
       fulfill();
   }));
   await frame.evaluate(() => window.stop());
-  await page.waitForTimeout(2000);  // give it some time to erroneously resolve
-  expect(done).toBe(browserName !== 'webkit');  // Chromium and Firefox issue load event in this case.
+  expect(done).toBe(true);
 });
 
 it('should work with url match', async ({ page, server }) => {