diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ab13ff42ec..201b9ac4ac 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -189,7 +189,7 @@ jobs: if: failure() with: name: ce-playwright-trace - path: test-apps/e2e/**/test-results/**/trace.zip + path: test-apps/e2e/test-results/**/trace.zip retention-days: 1 e2e_ee: @@ -231,7 +231,7 @@ jobs: if: failure() with: name: ee-playwright-trace - path: test-apps/e2e/**/test-results/**/trace.zip + path: test-apps/e2e/test-results/**/trace.zip retention-days: 1 api_ce_pg: diff --git a/docs/docs/guides/e2e/00-setup.md b/docs/docs/guides/e2e/00-setup.md index ca77a6d7db..36e11e7862 100644 --- a/docs/docs/guides/e2e/00-setup.md +++ b/docs/docs/guides/e2e/00-setup.md @@ -29,6 +29,46 @@ This will spawn by default a Strapi instance per testing domain (e.g. content-ma If you need to clean the test-apps folder because they are not working as expected, you can run `yarn test:e2e clean` which will clean said directory. +### Running specific tests + +To run only one domain, meaning a top-level directory in e2e/tests such as "admin" or "content-manager", use the `--domains` option. + +```shell +yarn test:e2e --domains admin +yarn test:e2e --domain admin +``` + +To run a specific file, you can pass arguments and options to playwright using `--` between the test:e2e options and the playwright options, such as: + +```shell +# to run just the login.spec.ts file in the admin domain +yarn test:e2e --domains admin -- login.spec.ts +``` + +### Concurrency / parallellization + +By default, every domain is run with its own test app in parallel with the other domains. The tests within a domain are run in series, one at a time. + +If you need an easier way to view the output, or have problems running multiple apps at once on your system, you can use the `-c` option + +```shell +# only run one domain at a time +yarn test:e2e -c 1 +``` + +### Env Variables to Control Test Config + +Some helpers have been added to allow you to modify the playwright configuration on your own system without touching the playwright config file used by the test runner. + +| env var | Description | Default | +| ---------------------------- | -------------------------------------------- | ------------------ | +| PLAYWRIGHT_WEBSERVER_TIMEOUT | timeout for starting the Strapi server | 16000 (160s) | +| PLAYWRIGHT_ACTION_TIMEOUT | playwright action timeout (ie, click()) | 15000 (15s) | +| PLAYWRIGHT_EXPECT_TIMEOUT | playwright expect waitFor timeout | 10000 (10s) | +| PLAYWRIGHT_TIMEOUT | playwright timeout, for each individual test | 30000 (30s) | +| PLAYWRIGHT_OUTPUT_DIR | playwright output dir, such as trace files | '../test-results/' | +| PLAYWRIGHT_VIDEO | set 'true' to save videos on failed tests | false | + ## Strapi Templates The test-app you create uses a [template](https://docs.strapi.io/developer-docs/latest/setup-deployment-guides/installation/templates.html) found at `e2e/app-template` in this folder we can store our premade content schemas & any customisations we may need such as other plugins / custom fields / endpoints etc. diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000000..b9f6b59e50 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,3 @@ +## End-to-end Playwright Tests + +See contributor docs in docs/docs/guides/e2e for more info diff --git a/playwright.base.config.js b/playwright.base.config.js index fa0a6f6099..4a79bb3168 100644 --- a/playwright.base.config.js +++ b/playwright.base.config.js @@ -1,5 +1,6 @@ // @ts-check const { devices } = require('@playwright/test'); +const { parseType } = require('@strapi/utils'); const getEnvNum = (envVar, defaultValue) => { if (envVar !== undefined && envVar !== null) { @@ -8,6 +9,22 @@ const getEnvNum = (envVar, defaultValue) => { return defaultValue; }; +const getEnvString = (envVar, defaultValue) => { + if (envVar?.trim().length) { + return envVar; + } + + return defaultValue; +}; + +const getEnvBool = (envVar, defaultValue) => { + if (!envVar || envVar === '') { + return defaultValue; + } + + return parseType({ type: 'boolean', value: envVar.toLowerCase() }); +}; + /** * @typedef ConfigOptions * @type {{ port: number; testDir: string; appDir: string }} @@ -28,7 +45,7 @@ const createConfig = ({ port, testDir, appDir }) => ({ * Maximum time expect() should wait for the condition to be met. * For example in `await expect(locator).toHaveText();` */ - timeout: getEnvNum(process.env.PLAYWRIGHT_EXPECT_TIMEOUT, 30 * 1000), + timeout: getEnvNum(process.env.PLAYWRIGHT_EXPECT_TIMEOUT, 10 * 1000), }, /* Run tests in files in parallel */ fullyParallel: false, @@ -46,13 +63,22 @@ const createConfig = ({ port, testDir, appDir }) => ({ baseURL: `http://127.0.0.1:${port}`, /* Default time each action such as `click()` can take to 20s */ - actionTimeout: getEnvNum(process.env.PLAYWRIGHT_ACTION_TIMEOUT, 20 * 1000), + actionTimeout: getEnvNum(process.env.PLAYWRIGHT_ACTION_TIMEOUT, 15 * 1000), /* Collect trace when a test failed on the CI. See https://playwright.dev/docs/trace-viewer Until https://github.com/strapi/strapi/issues/18196 is fixed we can't enable this locally, because the Strapi server restarts every time a new file (trace) is created. */ - trace: process.env.CI ? 'retain-on-failure' : 'off', + trace: 'retain-on-failure', + video: getEnvBool(process.env.PLAYWRIGHT_VIDEO, false) + ? { + mode: 'retain-on-failure', // 'retain-on-failure' to save videos only for failed tests + size: { + width: 1280, + height: 720, + }, + } + : 'off', }, /* Configure projects for major browsers */ @@ -80,7 +106,7 @@ const createConfig = ({ port, testDir, appDir }) => ({ ], /* Folder for test artifacts such as screenshots, videos, traces, etc. */ - outputDir: 'test-results/', + outputDir: getEnvString(process.env.PLAYWRIGHT_OUTPUT_DIR, '../test-results/'), // in the test-apps/e2e dir, to avoid writing files to the running Strapi project dir /* Run your local dev server before starting the tests */ webServer: {