feat(api): introduce expect().toPass (#19463)

docs/src/test-assertions-js.md

@@ -150,3 +150,27 @@ await expect.poll(async () => {
   timeout: 60_000
 }).toBe(200);
 ```
+
+## Retrying
+
+You can retry blocks of code until they are passing successfully.
+
+```js
+await expect(async () => {
+  const response = await page.request.get('https://api.example.com');
+  expect(response.status()).toBe(200);
+}).toPass();
+```
+
+You can also specify custom timeout for retry intervals:
+
+```js
+await expect(async () => {
+  const response = await page.request.get('https://api.example.com');
+  expect(response.status()).toBe(200);
+}).toPass({
+  // Probe, wait 1s, probe, wait 2s, probe, wait 10s, probe, wait 10s, probe, .... Defaults to [100, 250, 500, 1000].
+  intervals: [1_000, 2_000, 10_000],
+  timeout: 60_000
+});
+```

packages/playwright-test/src/expect.ts

@@ -38,6 +38,7 @@ import {
   toHaveURL,
   toHaveValue,
   toHaveValues,
+  toPass
 } from './matchers/matchers';
 import { toMatchSnapshot, toHaveScreenshot } from './matchers/toMatchSnapshot';
 import type { Expect } from './types';
@@ -103,7 +104,7 @@ function createExpect(actual: unknown, messageOrOptions: ExpectMessageOrOptions,
   return new Proxy(expectLibrary(actual), new ExpectMetaInfoProxyHandler(messageOrOptions, isSoft, isPoll, generator));
 }
 
-export const expect: Expect = new Proxy(expectLibrary as any, {
+export const expect: Expect = new Proxy(expectLibrary, {
   apply: function(target: any, thisArg: any, argumentsList: [actual: unknown, messageOrOptions: ExpectMessageOrOptions]) {
     const [actual, messageOrOptions] = argumentsList;
     return createExpect(actual, messageOrOptions, false /* isSoft */, false /* isPoll */);
@@ -145,6 +146,7 @@ const customMatchers = {
   toHaveValues,
   toMatchSnapshot,
   toHaveScreenshot,
+  toPass,
 };
 
 type Generator = () => any;

packages/playwright-test/src/matchers/matchers.ts

@@ -25,6 +25,8 @@ import { toEqual } from './toEqual';
 import { toExpectedTextValues, toMatchText } from './toMatchText';
 import type { ParsedStackTrace } from 'playwright-core/lib/utils/stackTrace';
 import { isTextualMimeType } from 'playwright-core/lib/utils/mimeType';
+import { monotonicTime } from 'playwright-core/lib/utils';
+import { raceAgainstTimeout } from 'playwright-core/lib/utils/timeoutRunner';
 
 interface LocatorEx extends Locator {
   _expect(customStackTrace: ParsedStackTrace, expression: string, options: Omit<FrameExpectOptions, 'expectedValue'> & { expectedValue?: any }): Promise<{ matches: boolean, received?: any, log?: string[], timedOut?: boolean }>;
@@ -309,3 +311,52 @@ export async function toBeOK(
   const pass = response.ok();
   return { message, pass };
 }
+
+export async function toPass(
+  this: ReturnType<Expect['getState']>,
+  callback: () => void,
+  options: {
+    intervals?: number[];
+    timeout?: number,
+  } = {},
+) {
+  let matcherError: Error | undefined;
+  const startTime = monotonicTime();
+  const pollIntervals = options.intervals || [100, 250, 500, 1000];
+  const lastPollInterval = pollIntervals[pollIntervals.length - 1] || 1000;
+  const timeout = options.timeout !== undefined ? options.timeout : 0;
+  const isNot = this.isNot;
+
+  while (true) {
+    const elapsed = monotonicTime() - startTime;
+    if (timeout !== 0 && elapsed > timeout)
+      break;
+    try {
+      const wrappedCallback = () => Promise.resolve().then(callback);
+      const received = timeout !== 0 ? await raceAgainstTimeout(wrappedCallback, timeout - elapsed)
+        : await wrappedCallback().then(() => ({ timedOut: false }));
+      if (received.timedOut)
+        break;
+      // The check passed, exit sucessfully.
+      if (isNot)
+        matcherError = new Error('Expected to fail, but passed');
+      else
+        return { message: () => '', pass: true };
+    } catch (e) {
+      if (isNot)
+        return { message: () => '', pass: false };
+      matcherError = e;
+    }
+    await new Promise(x => setTimeout(x, pollIntervals!.shift() ?? lastPollInterval));
+  }
+
+  const timeoutMessage = `Timeout ${timeout}ms exceeded while waiting on the predicate`;
+  const message = () => matcherError ? [
+    matcherError.message,
+    '',
+    `Call Log:`,
+    `- ${timeoutMessage}`,
+  ].join('\n') : timeoutMessage;
+
+  return { message, pass: false };
+}

packages/playwright-test/types/test.d.ts

@@ -3287,14 +3287,20 @@ type MakeMatchers<R, T> = BaseMatchers<R, T> & {
      */
     resolves: MakeMatchers<Promise<R>, Awaited<T>>;
     /**
-    * Unwraps the reason of a rejected promise so any other matcher can be chained.
+     * Unwraps the reason of a rejected promise so any other matcher can be chained.
-    * If the promise is fulfilled the assertion fails.
+     * If the promise is fulfilled the assertion fails.
-    */
+     */
     rejects: MakeMatchers<Promise<R>, Awaited<T>>;
   } & SnapshotAssertions &
   ExtraMatchers<T, Page, PageAssertions> &
   ExtraMatchers<T, Locator, LocatorAssertions> &
-  ExtraMatchers<T, APIResponse, APIResponseAssertions>;
+  ExtraMatchers<T, APIResponse, APIResponseAssertions> &
+  ExtraMatchers<T, Function, {
+    /**
+     * Retries the callback until it passes.
+     */
+    toPass(options?: { timeout?: number, intervals?: number[] }): Promise<void>;
+  }>;
 
 type BaseExpect = {
   // Removed following methods because they rely on a test-runner integration from Jest which we don't support:

tests/playwright-test/expect-to-pass.spec.ts

@@ -0,0 +1,169 @@
+/**
+ * Copyright Microsoft Corporation. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, stripAnsi } from './playwright-test-fixtures';
+
+test('should retry predicate', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should toPass sync predicate', async () => {
+        let i = 0;
+        await test.expect(() => {
+          expect(++i).toBe(3);
+        }).toPass();
+        expect(i).toBe(3);
+      });
+      test('should toPass async predicate', async () => {
+        let i = 0;
+        await test.expect(async () => {
+          await new Promise(x => setTimeout(x, 50));
+          expect(++i).toBe(3);
+        }).toPass();
+        expect(i).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(2);
+});
+
+test('should respect timeout', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async () => {
+        await test.expect(() => {
+          expect(1).toBe(2);
+        }).toPass({ timeout: 100 });
+      });
+    `
+  });
+  expect(result.exitCode).toBe(1);
+  expect(stripAnsi(result.output)).toContain('Timeout 100ms exceeded while waiting on the predicate');
+  expect(stripAnsi(result.output)).toContain('Received: 1');
+  expect(stripAnsi(result.output)).toContain(`
+  7 |         await test.expect(() => {
+  `.trim());
+});
+
+test('should not fail when used with web-first assertion', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async ({ page }) => {
+        let i = 0;
+        await test.expect(async () => {
+          if (++i < 3)
+            await expect(page.locator('body')).toHaveText('foo', { timeout: 1 });
+        }).toPass();
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should support .not predicate', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should pass', async ({ page }) => {
+        let i = 0;
+        await test.expect(() => {
+          expect(++i).toBeLessThan(3);
+        }).not.toPass();
+        expect(i).toBe(3);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+  expect(result.passed).toBe(1);
+});
+
+test('should respect interval', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail', async () => {
+        let probes = 0;
+        await test.expect(() => {
+          ++probes
+          expect(1).toBe(2);
+        }).toPass({ timeout: 1000, intervals: [600] }).catch(() => {});
+        // Probe at 0s, at 0.6s.
+        expect(probes).toBe(2);
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should compile', async ({ runTSC }) => {
+  const result = await runTSC({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should poll sync predicate', async ({ page }) => {
+        let i = 0;
+        test.expect(() => ++i).toPass();
+        test.expect(() => ++i, 'message').toPass();
+        test.expect(() => ++i, { message: 'message' }).toPass();
+        test.expect(() => ++i).toPass({ timeout: 100 });
+        test.expect(() => ++i, { message: 'message' }).toPass({ timeout: 100 });
+        test.expect(async () => {
+          await new Promise(x => setTimeout(x, 50));
+          return ++i;
+        }).toPass();
+        test.expect(() => Promise.resolve(++i)).toPass();
+      });
+    `
+  });
+  expect(result.exitCode).toBe(0);
+});
+
+test('should use custom message', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should fail with custom message', async () => {
+        await test.expect(() => {
+          expect(1).toBe(3);
+        }, 'Custom message').toPass({ timeout: 1 });
+      });
+    `
+  });
+  expect(stripAnsi(result.output)).toContain('Error: Custom message');
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+});
+
+test('should work with soft', async ({ runInlineTest }) => {
+  const result = await runInlineTest({
+    'a.spec.ts': `
+      const { test } = pwt;
+      test('should respect soft', async () => {
+        await test.expect.soft(() => {
+          expect(1).toBe(3);
+        }).toPass({ timeout: 1 });
+        expect.soft(2).toBe(3);
+      });
+    `
+  });
+  expect(stripAnsi(result.output)).toContain('Received: 1');
+  expect(stripAnsi(result.output)).toContain('Received: 2');
+  expect(result.exitCode).toBe(1);
+  expect(result.failed).toBe(1);
+});

utils/generate_types/overrides-test.d.ts

@@ -293,14 +293,20 @@ type MakeMatchers<R, T> = BaseMatchers<R, T> & {
      */
     resolves: MakeMatchers<Promise<R>, Awaited<T>>;
     /**
-    * Unwraps the reason of a rejected promise so any other matcher can be chained.
+     * Unwraps the reason of a rejected promise so any other matcher can be chained.
-    * If the promise is fulfilled the assertion fails.
+     * If the promise is fulfilled the assertion fails.
-    */
+     */
     rejects: MakeMatchers<Promise<R>, Awaited<T>>;
   } & SnapshotAssertions &
   ExtraMatchers<T, Page, PageAssertions> &
   ExtraMatchers<T, Locator, LocatorAssertions> &
-  ExtraMatchers<T, APIResponse, APIResponseAssertions>;
+  ExtraMatchers<T, APIResponse, APIResponseAssertions> &
+  ExtraMatchers<T, Function, {
+    /**
+     * Retries the callback until it passes.
+     */
+    toPass(options?: { timeout?: number, intervals?: number[] }): Promise<void>;
+  }>;
 
 type BaseExpect = {
   // Removed following methods because they rely on a test-runner integration from Jest which we don't support: