feat(test-runner): implement expect.poll (#12815)

This patch implements `expect.poll()` method that polls given
predicate until a given synchronous predicate completes.

Usage:

```js
// wait until page gets 3 frames.
await expect.poll(() => page.frames().length, {
  timeout: 1000,
  message: 'custom error message',
}).toBe(3);
```

Fixes #10235

docs/src/test-assertions-js.md

@@ -96,6 +96,25 @@ The same works with soft assertions:
 expect.soft(value, 'my soft assertion').toBe(56);
 ```
 
+## Polling
+
+You can convert any synchronous `expect` to an asynchronous polling one using `expect.poll`.
+
+The following method will poll given function until it returns HTTP status 200:
+
+```js
+expect.poll(async () => {
+  const response = await page.request.get('https://api.example.com');
+  return response.status();
+}, {
+  // Custom error message
+  message: 'make sure API eventually succeeds', // custom error message
+  // Poll for 10 seconds; defaults to 5 seconds. Pass 0 to disable timeout.
+  timeout: 10000,
+}).toBe(200);
+```
+
+
 ## API reference
 See the following pages for Playwright-specific assertions:
 - [APIResponseAssertions] assertions for [APIResponse]

packages/playwright-test/src/expect.ts

@@ -15,6 +15,7 @@
  */
 
 import expectLibrary from 'expect';
+import { raceAgainstTimeout } from 'playwright-core/lib/utils/async';
 import path from 'path';
 import {
   INVERTED_COLOR,
@@ -47,7 +48,8 @@ import { toMatchSnapshot, toHaveScreenshot, getSnapshotName } from './matchers/t
 import type { Expect, TestError } from './types';
 import matchers from 'expect/build/matchers';
 import { currentTestInfo } from './globals';
-import { serializeError, captureStackTrace } from './util';
+import { serializeError, captureStackTrace, currentExpectTimeout } from './util';
+import { monotonicTime } from 'playwright-core/lib/utils/utils';
 
 // #region
 // Mirrored from https://github.com/facebook/jest/blob/f13abff8df9a0e1148baf3584bcde6d1b479edc7/packages/expect/src/print.ts
@@ -89,21 +91,25 @@ export const printReceivedStringContainExpectedResult = (
 
 // #endregion
 
-function createExpect(actual: unknown, message: string|undefined, isSoft: boolean) {
-  if (message !== undefined && typeof message !== 'string')
-    throw new Error('expect(actual, optionalErrorMessage): optional error message must be a string.');
-  return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(message || '', isSoft));
+type ExpectMessageOrOptions = undefined | string | { message?: string, timeout?: number };
+
+function createExpect(actual: unknown, messageOrOptions: ExpectMessageOrOptions, isSoft: boolean, isPoll: boolean) {
+  return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(messageOrOptions, isSoft, isPoll));
 }
 
 export const expect: Expect = new Proxy(expectLibrary as any, {
-  apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, message: string|undefined]) {
-    const [actual, message] = argumentsList;
-    return createExpect(actual, message, false /* isSoft */);
+  apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, messageOrOptions: ExpectMessageOrOptions]) {
+    const [actual, messageOrOptions] = argumentsList;
+    return createExpect(actual, messageOrOptions, false /* isSoft */, false /* isPoll */);
   }
 });
 
-expect.soft = (actual: unknown, message: string|undefined) => {
-  return createExpect(actual, message, true /* isSoft */);
+expect.soft = (actual: unknown, messageOrOptions: ExpectMessageOrOptions) => {
+  return createExpect(actual, messageOrOptions, true /* isSoft */, false /* isPoll */);
+};
+
+expect.poll = (actual: unknown, messageOrOptions: ExpectMessageOrOptions) => {
+  return createExpect(actual, messageOrOptions, false /* isSoft */, true /* isPoll */);
 };
 
 expectLibrary.setState({ expand: false });
@@ -133,19 +139,25 @@ const customMatchers = {
 };
 
 type ExpectMetaInfo = {
-  message: string;
+  message?: string;
   isSoft: boolean;
+  isPoll: boolean;
+  pollTimeout?: number;
 };
 
 let expectCallMetaInfo: undefined|ExpectMetaInfo = undefined;
 
 class ExpectMetaInfoProxyHandler {
-  private _message: string;
-  private _isSoft: boolean;
+  private _info: ExpectMetaInfo;
 
-  constructor(message: string, isSoft: boolean) {
-    this._message = message;
-    this._isSoft = isSoft;
+  constructor(messageOrOptions: ExpectMessageOrOptions, isSoft: boolean, isPoll: boolean) {
+    this._info = { isSoft, isPoll };
+    if (typeof messageOrOptions === 'string') {
+      this._info.message = messageOrOptions;
+    } else {
+      this._info.message = messageOrOptions?.message;
+      this._info.pollTimeout = messageOrOptions?.timeout;
+    }
   }
 
   get(target: any, prop: any, receiver: any): any {
@@ -157,15 +169,17 @@ class ExpectMetaInfoProxyHandler {
       if (!testInfo)
         return value.call(target, ...args);
       const handleError = (e: Error) => {
-        if (this._isSoft)
+        if (this._info.isSoft)
           testInfo._failWithError(serializeError(e), false /* isHardError */);
         else
           throw e;
       };
       try {
         expectCallMetaInfo = {
-          message: this._message,
-          isSoft: this._isSoft,
+          message: this._info.message,
+          isSoft: this._info.isSoft,
+          isPoll: this._info.isPoll,
+          pollTimeout: this._info.pollTimeout,
         };
         let result = value.call(target, ...args);
         if ((result instanceof Promise))
@@ -180,6 +194,36 @@ class ExpectMetaInfoProxyHandler {
   }
 }
 
+async function pollMatcher(matcher: any, timeout: number, thisArg: any, generator: () => any, ...args: any[]) {
+  let result: { pass: boolean, message: () => string } | undefined = undefined;
+  const startTime = monotonicTime();
+  const pollIntervals = [100, 250, 500];
+  while (true) {
+    const elapsed = monotonicTime() - startTime;
+    if (timeout !== 0 && elapsed > timeout)
+      break;
+    const received = timeout !== 0 ? await raceAgainstTimeout(generator, timeout - elapsed) : await generator();
+    if (received.timedOut)
+      break;
+    result = matcher.call(thisArg, received.result, ...args);
+    const success = result!.pass !== thisArg.isNot;
+    if (success)
+      return result;
+    await new Promise(x => setTimeout(x, pollIntervals.shift() ?? 1000));
+  }
+  const timeoutMessage = `Timeout ${timeout}ms exceeded while waiting on the predicate`;
+  const message = result ? [
+    result.message(),
+    '',
+    `Call Log:`,
+    `- ${timeoutMessage}`,
+  ].join('\n') : timeoutMessage;
+  return {
+    pass: thisArg.isNot,
+    message: () => message,
+  };
+}
+
 function wrap(matcherName: string, matcher: any) {
   return function(this: any, ...args: any[]) {
     const testInfo = currentTestInfo();
@@ -196,10 +240,12 @@ function wrap(matcherName: string, matcher: any) {
     const frame = stackTrace.frames[0];
     const customMessage = expectCallMetaInfo?.message ?? '';
     const isSoft = expectCallMetaInfo?.isSoft ?? false;
+    const isPoll = expectCallMetaInfo?.isPoll ?? false;
+    const pollTimeout = expectCallMetaInfo?.pollTimeout;
     const step = testInfo._addStep({
       location: frame && frame.file ? { file: path.resolve(process.cwd(), frame.file), line: frame.line || 0, column: frame.column || 0 } : undefined,
       category: 'expect',
-      title: customMessage || `expect${isSoft ? '.soft' : ''}${this.isNot ? '.not' : ''}.${matcherName}${titleSuffix}`,
+      title: customMessage || `expect${isPoll ? '.poll' : ''}${isSoft ? '.soft' : ''}${this.isNot ? '.not' : ''}.${matcherName}${titleSuffix}`,
       canHaveChildren: true,
       forceNoParent: false
     });
@@ -240,7 +286,19 @@ function wrap(matcherName: string, matcher: any) {
     };
 
     try {
-      const result = matcher.call(this, ...args);
+      let result;
+      const [receivedOrGenerator, ...otherArgs] = args;
+      if (isPoll) {
+        if (typeof receivedOrGenerator !== 'function')
+          throw new Error('`expect.poll()` accepts only function as a first argument');
+        if ((customMatchers as any)[matcherName] || matcherName === 'resolves' || matcherName === 'rejects')
+          throw new Error(`\`expect.poll()\` does not support "${matcherName}" matcher.`);
+        result = pollMatcher(matcher, currentExpectTimeout({ timeout: pollTimeout }), this, receivedOrGenerator, ...otherArgs);
+      } else {
+        if (typeof receivedOrGenerator === 'function')
+          throw new Error('Cannot accept function as a first argument; did you mean to use `expect.poll()`?');
+        result = matcher.call(this, ...args);
+      }
       if (result instanceof Promise)
         return result.then(reportStepEnd).catch(reportStepError);
       return reportStepEnd(result);

packages/playwright-test/types/testExpect.d.ts

@@ -28,8 +28,9 @@ type MakeMatchers<T, ReturnValue = T> = PlaywrightTest.Matchers<ReturnValue> &
   ExtraMatchers<T, APIResponse, APIResponseMatchers>
 
 export declare type Expect = {
-  <T = unknown>(actual: T, message?: string): MakeMatchers<T>;
-  soft: <T = unknown>(actual: T, message?: string) => MakeMatchers<T>;
+  <T = unknown>(actual: T, messageOrOptions?: string | { message?: string }): MakeMatchers<T>;
+  soft: <T = unknown>(actual: T, messageOrOptions?: string | { message?: string }) => MakeMatchers<T>;
+  poll: <T = unknown>(actual: () => T | Promise<T>, messageOrOptions?: string | { message?: string, timeout?: number }) => Omit<PlaywrightTest.Matchers<T>, 'rejects' | 'resolves'>;
 
   extend(arg0: any): void;
   getState(): expect.MatcherState;

tests/playwright-test/expect-poll.spec.ts

@@ -0,0 +1,146 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAnsi } from './playwright-test-fixtures';
+
+test('should poll predicate', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should poll sync predicate', async () => {
+        let i = 0;
+        await test.expect.poll(() => ++i).toBe(3);
+      });
+      test('should poll async predicate', async () => {
+        let i = 0;
+        await test.expect.poll(async () => {
+          await new Promise(x => setTimeout(x, 50));
+          return ++i;
+        }).toBe(3);
+      });
+      test('should poll predicate that returns a promise', async () => {
+        let i = 0;
+        await test.expect.poll(() => Promise.resolve(++i)).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(3);
+});
+
+test('should compile', async ({ runTSC }) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should poll sync predicate', () => {
+        let i = 0;
+        test.expect.poll(() => ++i).toBe(3);
+        test.expect.poll(() => ++i, 'message').toBe(3);
+        test.expect.poll(() => ++i, { message: 'message' }).toBe(3);
+        test.expect.poll(() => ++i, { timeout: 100 }).toBe(3);
+        test.expect.poll(() => ++i, { message: 'message', timeout: 100 }).toBe(3);
+        test.expect.poll(async () => {
+          await new Promise(x => setTimeout(x, 50));
+          return ++i;
+        }).toBe(3);
+        test.expect.poll(() => Promise.resolve(++i)).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should respect timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async () => {
+        await test.expect.poll(() => false, { timeout: 100 }).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Timeout 100ms exceeded while waiting on the predicate');
+  expect(stripAnsi(result.output)).toContain(`
+  7 |         await test.expect.poll(() => false, { timeout: 100 }).
+  `.trim());
+});
+
+test('should fail when passed in non-function', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async () => {
+        await test.expect.poll(false).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Error: `expect.poll()` accepts only function as a first argument');
+});
+
+test('should fail when used with web-first assertion', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async ({ page }) => {
+        await test.expect.poll(() => page.locator('body')).toHaveText('foo');
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Error: `expect.poll()` does not support "toHaveText" matcher');
+});
+
+test('should time out when running infinite predicate', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async ({ page }) => {
+        await test.expect.poll(() => new Promise(x => {}), { timeout: 100 }).toBe(42);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Timeout 100ms exceeded');
+});
+
+test('should show error that is thrown from predicate', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async ({ page }) => {
+        await test.expect.poll(() => { throw new Error('foo bar baz'); }, { timeout: 100 }).toBe(42);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('foo bar baz');
+});
+
+test('should support .not predicate', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async ({ page }) => {
+        let i = 0;
+        await test.expect.poll(() => ++i).not.toBeLessThan(3);
+        expect(i).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});

tests/playwright-test/expect-soft.spec.ts

@@ -23,6 +23,7 @@ test('soft expects should compile', async ({ runTSC }) => {
       test('should work', () => {
         test.expect.soft(1+1).toBe(3);
         test.expect.soft(1+1, 'custom error message').toBe(3);
+        test.expect.soft(1+1, { message: 'custom error message' }).toBe(3);
       });
     `
   });
@@ -43,6 +44,19 @@ test('soft expects should work', async ({ runInlineTest }) => {
   expect(stripAnsi(result.output)).toContain('woof-woof');
 });
 
+test('should fail when passed in function', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should work', () => {
+        test.expect.soft(() => 1+1).toBe(2);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Cannot accept function as a first argument; did you mean to use `expect.poll()`?');
+});
+
 test('should report a mixture of soft and non-soft errors', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'a.spec.ts': `
@@ -51,7 +65,7 @@ test('should report a mixture of soft and non-soft errors', async ({ runInlineTe
         test.expect.soft(1+1, 'one plus one').toBe(3);
         test.expect.soft(2*2, 'two times two').toBe(5);
         test.expect(3/3, 'three div three').toBe(7);
-        test.expect.soft(6-4, 'six minus four').toBe(3);
+        test.expect.soft(6-4, { message: 'six minus four' }).toBe(3);
       });
     `
   });

tests/playwright-test/expect.spec.ts

@@ -67,20 +67,6 @@ test('should not expand huge arrays', async ({ runInlineTest }) => {
   expect(result.output.length).toBeLessThan(100000);
 });
 
-test('should fail when passed `null` instead of message', async ({ runInlineTest }) => {
-  const result = await runInlineTest({
-    'expect-test.spec.ts': `
-      const { test } = pwt;
-      test('custom expect message', () => {
-        test.expect(1+1, null).toEqual(3);
-      });
-    `
-  });
-  expect(result.exitCode).toBe(1);
-  expect(result.passed).toBe(0);
-  expect(stripAnsi(result.output)).toContain(`optional error message must be a string.`);
-});
-
 test('should include custom error message', async ({ runInlineTest }) => {
   const result = await runInlineTest({
     'expect-test.spec.ts': `
@@ -105,7 +91,7 @@ test('should include custom error message with web-first assertions', async ({ r
     'expect-test.spec.ts': `
       const { test } = pwt;
       test('custom expect message', async ({page}) => {
-        await expect(page.locator('x-foo'), 'x-foo must be visible').toBeVisible({timeout: 1});
+        await expect(page.locator('x-foo'), { message: 'x-foo must be visible' }).toBeVisible({timeout: 1});
       });
     `
   });