test: improve e2e playwright config

.github/workflows/tests.yml

@@ -189,7 +189,7 @@ jobs:
         if: failure()
         with:
           name: ce-playwright-trace
-          path: test-apps/e2e/**/test-results/**/trace.zip
+          path: test-apps/e2e/test-results/**/trace.zip
           retention-days: 1
 
   e2e_ee:
@@ -231,7 +231,7 @@ jobs:
         if: failure()
         with:
           name: ee-playwright-trace
-          path: test-apps/e2e/**/test-results/**/trace.zip
+          path: test-apps/e2e/test-results/**/trace.zip
           retention-days: 1
 
   api_ce_pg:

docs/docs/guides/e2e/00-setup.md

@@ -29,6 +29,46 @@ This will spawn by default a Strapi instance per testing domain (e.g. content-ma
 
 If you need to clean the test-apps folder because they are not working as expected, you can run `yarn test:e2e clean` which will clean said directory.
 
+### Running specific tests
+
+To run only one domain, meaning a top-level directory in e2e/tests such as "admin" or "content-manager", use the `--domains` option.
+
+```shell
+yarn test:e2e --domains admin
+yarn test:e2e --domain admin
+```
+
+To run a specific file, you can pass arguments and options to playwright using `--` between the test:e2e options and the playwright options, such as:
+
+```shell
+# to run just the login.spec.ts file in the admin domain
+yarn test:e2e --domains admin -- login.spec.ts
+```
+
+### Concurrency / parallellization
+
+By default, every domain is run with its own test app in parallel with the other domains. The tests within a domain are run in series, one at a time.
+
+If you need an easier way to view the output, or have problems running multiple apps at once on your system, you can use the `-c` option
+
+```shell
+# only run one domain at a time
+yarn test:e2e -c 1
+```
+
+### Env Variables to Control Test Config
+
+Some helpers have been added to allow you to modify the playwright configuration on your own system without touching the playwright config file used by the test runner.
+
+| env var                      | Description                                  | Default            |
+| ---------------------------- | -------------------------------------------- | ------------------ |
+| PLAYWRIGHT_WEBSERVER_TIMEOUT | timeout for starting the Strapi server       | 16000 (160s)       |
+| PLAYWRIGHT_ACTION_TIMEOUT    | playwright action timeout (ie, click())      | 15000 (15s)        |
+| PLAYWRIGHT_EXPECT_TIMEOUT    | playwright expect waitFor timeout            | 10000 (10s)        |
+| PLAYWRIGHT_TIMEOUT           | playwright timeout, for each individual test | 30000 (30s)        |
+| PLAYWRIGHT_OUTPUT_DIR        | playwright output dir, such as trace files   | '../test-results/' |
+| PLAYWRIGHT_VIDEO             | set 'true' to save videos on failed tests    | false              |
+
 ## Strapi Templates
 
 The test-app you create uses a [template](https://docs.strapi.io/developer-docs/latest/setup-deployment-guides/installation/templates.html) found at `e2e/app-template` in this folder we can store our premade content schemas & any customisations we may need such as other plugins / custom fields / endpoints etc.

e2e/README.md

@@ -0,0 +1,3 @@
+## End-to-end Playwright Tests
+
+See contributor docs in docs/docs/guides/e2e for more info

playwright.base.config.js

@@ -1,5 +1,6 @@
 // @ts-check
 const { devices } = require('@playwright/test');
+const { parseType } = require('@strapi/utils');
 
 const getEnvNum = (envVar, defaultValue) => {
   if (envVar !== undefined && envVar !== null) {
@@ -8,6 +9,22 @@ const getEnvNum = (envVar, defaultValue) => {
   return defaultValue;
 };
 
+const getEnvString = (envVar, defaultValue) => {
+  if (envVar?.trim().length) {
+    return envVar;
+  }
+
+  return defaultValue;
+};
+
+const getEnvBool = (envVar, defaultValue) => {
+  if (!envVar || envVar === '') {
+    return defaultValue;
+  }
+
+  return parseType({ type: 'boolean', value: envVar.toLowerCase() });
+};
+
 /**
  * @typedef ConfigOptions
  * @type {{ port: number; testDir: string; appDir: string }}
@@ -28,7 +45,7 @@ const createConfig = ({ port, testDir, appDir }) => ({
      * Maximum time expect() should wait for the condition to be met.
      * For example in `await expect(locator).toHaveText();`
      */
-    timeout: getEnvNum(process.env.PLAYWRIGHT_EXPECT_TIMEOUT, 30 * 1000),
+    timeout: getEnvNum(process.env.PLAYWRIGHT_EXPECT_TIMEOUT, 10 * 1000),
   },
   /* Run tests in files in parallel */
   fullyParallel: false,
@@ -46,13 +63,22 @@ const createConfig = ({ port, testDir, appDir }) => ({
     baseURL: `http://127.0.0.1:${port}`,
 
     /* Default time each action such as `click()` can take to 20s */
-    actionTimeout: getEnvNum(process.env.PLAYWRIGHT_ACTION_TIMEOUT, 20 * 1000),
+    actionTimeout: getEnvNum(process.env.PLAYWRIGHT_ACTION_TIMEOUT, 15 * 1000),
 
     /* Collect trace when a test failed on the CI. See https://playwright.dev/docs/trace-viewer
        Until https://github.com/strapi/strapi/issues/18196 is fixed we can't enable this locally,
        because the Strapi server restarts every time a new file (trace) is created.
     */
-    trace: process.env.CI ? 'retain-on-failure' : 'off',
+    trace: 'retain-on-failure',
+    video: getEnvBool(process.env.PLAYWRIGHT_VIDEO, false)
+      ? {
+          mode: 'retain-on-failure', // 'retain-on-failure' to save videos only for failed tests
+          size: {
+            width: 1280,
+            height: 720,
+          },
+        }
+      : 'off',
   },
 
   /* Configure projects for major browsers */
@@ -80,7 +106,7 @@ const createConfig = ({ port, testDir, appDir }) => ({
   ],
 
   /* Folder for test artifacts such as screenshots, videos, traces, etc. */
-  outputDir: 'test-results/',
+  outputDir: getEnvString(process.env.PLAYWRIGHT_OUTPUT_DIR, '../test-results/'), // in the test-apps/e2e dir, to avoid writing files to the running Strapi project dir
 
   /* Run your local dev server before starting the tests */
   webServer: {