fix: flaky tests (#22829)

docs/docs/guides/e2e/01-app-template.md

@@ -15,13 +15,25 @@ Here you can read about what content schemas the test instance has & the API cus
 
 ## Update the app template
 
+:::info
+The app template should be realistic and structured in a way an actual user might create an app using Strapi
+:::
+
 To update the app template:
 
-- run the tests to create a Strapi app based on the existing template at `test-apps/e2e/test-app-<number>`.
-- Move into this folder and run `yarn develop`.
-- Login using the credentials found in `e2e/constants.js`.
-- Make any changes you need (i.e. create a content-type).
-- Replace the existing template in `e2e/app-template` with the current folder content (only keep the files you need).
+- Run `yarn test:e2e clean` to remove existing test apps
+- Run `yarn test:e2e -c=1 -- --ui` to generate a test app (don't run any tests)
+- Follow the instructions to [import the existing data set](./02-data-transfer.md#importing-an-existing-data-packet)
+- With the test app server running on 1337 you can now login to the app
+- Make changes in the content-type builder
+- Copy the generated files in the test app to the app template
+
+Once the app template is updated:
+
+- Run `yarn test:e2e clean` to remove existing test apps
+- Run `yarn test:e2e -c=1 -- --ui` to generate a new test app using the updated template (don't run any tests)
+- Follow the instructions to [import the existing data set](./02-data-transfer.md#importing-an-existing-data-packet)
+- Follow the instructions to [export the updated data set](./02-data-transfer.md#exporting-an-updated-data-packet)
 
 ## Content Schemas
 
@@ -190,6 +202,42 @@ This collection type is internationalized.
 }
 ```
 
+### Match
+
+```json
+{
+  // ...
+  "attributes": {
+    "date": {
+      "type": "date"
+    },
+    "kit_man": {
+      "type": "string"
+    },
+    "opponent": {
+      "type": "string",
+      "required": true,
+      "regex": "^(?!.*richmond).*"
+    },
+    "lineup": {
+      "type": "component",
+      "repeatable": true,
+      "component": "match.player"
+    },
+    "most_valuable_player": {
+      "type": "component",
+      "repeatable": false,
+      "component": "match.player"
+    },
+    "sections": {
+      "type": "dynamiczone",
+      "components": ["match.player", "product.variations"]
+    }
+  }
+  // ...
+}
+```
+
 ### Shop (Single Type)
 
 This single type is internationalized.

packages/core/content-releases/admin/src/components/tests/ReleaseActionMenu.test.tsx

@@ -15,11 +15,11 @@ describe('ReleaseActionMenu', () => {
       </ReleaseActionMenu.Root>
     );
 
-    const menuTrigger = await screen.findByRole('button', { name: 'Release action options' });
-    expect(menuTrigger).toBeInTheDocument();
+    await user.click(await screen.findByRole('button', { name: 'Release action options' }));
 
-    await user.click(menuTrigger);
-    expect(screen.getByRole('menuitem', { name: 'Remove from release' })).toBeInTheDocument();
-    expect(screen.getByRole('menuitem', { name: 'Edit entry' })).toBeInTheDocument();
+    expect(
+      await screen.findByRole('menuitem', { name: 'Remove from release' })
+    ).toBeInTheDocument();
+    expect(await screen.findByRole('menuitem', { name: 'Edit entry' })).toBeInTheDocument();
   });
 });

tests/e2e/app-template/src/api/match/content-types/match/schema.json

@@ -0,0 +1,39 @@
+{
+  "kind": "collectionType",
+  "collectionName": "matches",
+  "info": {
+    "singularName": "match",
+    "pluralName": "matches",
+    "displayName": "Match"
+  },
+  "options": {
+    "draftAndPublish": true
+  },
+  "attributes": {
+    "date": {
+      "type": "date"
+    },
+    "kit_man": {
+      "type": "string"
+    },
+    "opponent": {
+      "type": "string",
+      "required": true,
+      "regex": "^(?!.*richmond).*"
+    },
+    "lineup": {
+      "type": "component",
+      "repeatable": true,
+      "component": "match.player"
+    },
+    "most_valuable_player": {
+      "type": "component",
+      "repeatable": false,
+      "component": "match.player"
+    },
+    "sections": {
+      "type": "dynamiczone",
+      "components": ["match.player", "product.variations"]
+    }
+  }
+}

tests/e2e/app-template/src/components/match/player.json

@@ -0,0 +1,19 @@
+{
+  "collectionName": "components_match_player",
+  "info": {
+    "displayName": "Player",
+    "icon": "moon"
+  },
+  "options": {},
+  "attributes": {
+    "full_name": {
+      "type": "string",
+      "required": true,
+      "regex": "^(?!.*nate).*"
+    },
+    "position": {
+      "type": "string",
+      "required": false
+    }
+  }
+}

tests/e2e/constants.ts

@@ -12,6 +12,7 @@ export const ALLOWED_CONTENT_TYPES = [
   'api::homepage.homepage',
   'api::product.product',
   'api::shop.shop',
+  'api::match.match',
   'api::upcoming-match.upcoming-match',
   'api::unique.unique',
   'plugin::content-manager.history-version',

tests/e2e/tests/content-manager/create-content.spec.ts

@@ -1,120 +1,26 @@
 import { test, expect } from '@playwright/test';
-import { addAttributesToContentType } from '../../utils/content-types';
-import { sharedSetup } from '../../utils/setup';
 import { clickAndWait, dragElementAbove, findAndClose, isElementBefore } from '../../utils/shared';
 import { createContent, FieldValue, verifyFields } from '../../utils/content-creation';
-import { resetFiles } from '../../utils/file-reset';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
+import { login } from '../../utils/login';
 
 test.describe('Adding content', () => {
   test.beforeEach(async ({ page }) => {
-    await sharedSetup('ctb-edit-st', page, {
-      login: true,
-      skipTour: true,
-      resetFiles: true,
-      importData: 'with-admin.tar',
-      afterSetup: async ({ page }) => {
-        // TODO: to save the time from a server restart, add these components directly inside the with-admin dataset and remove this
-        await addAttributesToContentType(page, 'Article', [
-          {
-            type: 'text',
-            name: 'testtext',
-            advanced: { required: true, regexp: '^(?!.*fail).*' },
-          },
-          {
-            type: 'component',
-            name: 'testrepeatablecomp',
-            component: {
-              options: {
-                repeatable: true,
-                name: 'testrepeatablecomp2',
-                icon: 'moon',
-                categorySelect: 'product',
-                attributes: [
-                  {
-                    type: 'text',
-                    name: 'testrepeatablecomp2text',
-                    advanced: { required: true, regexp: '^(?!.*fail).*' },
-                  },
-                ],
-              },
-            },
-          },
-          {
-            type: 'component',
-            name: 'testsinglecomp',
-            component: {
-              options: {
-                repeatable: false,
-                name: 'testsinglecomp2',
-                icon: 'moon',
-                categorySelect: 'product',
-                attributes: [
-                  {
-                    type: 'text',
-                    name: 'testsinglecomp2text',
-                    advanced: { required: true, regexp: '^(?!.*fail).*' },
-                  },
-                ],
-              },
-            },
-          },
-          {
-            type: 'dz',
-            name: 'testdz',
-            dz: {
-              components: [
-                // New repeatable component with existing category
-                {
-                  type: 'component',
-                  name: 'testnewcomponentexistingcategory',
-                  component: {
-                    options: {
-                      name: 'testnewcomponentrepeatable',
-                      icon: 'moon',
-                      categorySelect: 'product',
-                      attributes: [
-                        {
-                          type: 'text',
-                          name: 'testnewcomponentexistingcategorytext',
-                          advanced: { required: true, regexp: '^(?!.*fail).*' },
-                        },
-                      ],
-                    },
-                  },
-                },
-                // Existing component with existing category
-                {
-                  type: 'component',
-                  name: 'testexistingcomponentexistingcategory',
-                  component: {
-                    useExisting: 'variations',
-                    options: {
-                      name: 'testvariations',
-                      icon: 'globe',
-                    },
-                  },
-                },
-              ],
-            },
-          },
-        ]);
-      },
-    });
+    await resetDatabaseAndImportDataFromPath('with-admin.tar');
+    await page.goto('/admin');
+    await login({ page });
 
-    await clickAndWait(page, page.getByRole('link', { name: 'Content-Type Builder' }));
-  });
-
-  test.afterAll(async () => {
-    await resetFiles();
+    // Navigate to Content Manager
+    await clickAndWait(page, page.getByRole('link', { name: 'Content Manager' }));
   });
 
   test('I want to be able to save and publish content', async ({ page }) => {
     await createContent(
       page,
-      'Article',
+      'Match',
       [
         {
-          name: 'testtext',
+          name: 'opponent*',
           type: 'text',
           value: 'testname',
         },
@@ -126,40 +32,40 @@ test.describe('Adding content', () => {
   test('I want to set component order when creating content', async ({ page }) => {
     const fields = [
       {
-        name: 'testtext',
+        name: 'opponent*',
         type: 'text',
         value: 'testname',
       },
       {
-        name: 'testdz',
+        name: 'sections',
         type: 'dz',
         value: [
           {
-            category: 'product',
-            name: 'testnewcomponentexistingcategory',
+            category: 'match',
+            name: 'player',
             fields: [
               {
                 type: 'text',
-                name: 'testnewcomponentexistingcategorytext',
-                value: 'First component text value',
+                name: 'full_name*',
+                value: 'Roy Kent',
               },
             ],
           },
           {
             category: 'product',
             name: 'variations',
-            fields: [{ type: 'text', name: 'name', value: 'Second component text value' }],
+            fields: [{ type: 'text', name: 'name', value: 'Roy Kent Shirt Jersey' }],
           },
         ],
       },
     ] satisfies FieldValue[];
 
-    await createContent(page, 'Article', fields, { save: false, publish: false, verify: false });
+    await createContent(page, 'Match', fields, { save: false, publish: false, verify: false });
 
     await page.waitForLoadState('networkidle');
 
     const source = page.locator('li:has-text("variations")');
-    const target = page.locator('li:has-text("testnewcomponentexistingcategory")');
+    const target = page.locator('li:has-text("player")');
     await dragElementAbove(page, {
       source,
       target,
@@ -180,14 +86,15 @@ test.describe('Adding content', () => {
     {
       description: 'empty required text field (basic)',
       fields: [
-        { name: 'testtext', type: 'text', value: '' },
-        { name: 'title', type: 'text', value: 'at least one field requires text' },
+        { name: 'opponent*', type: 'text', value: '' },
+        { name: 'kit_man', type: 'text', value: 'Roy Kent' },
       ],
       expectedError: 'This value is required',
     },
     {
       description: 'invalid regexp text field (basic)',
-      fields: [{ name: 'testtext', type: 'text', value: 'fail-regexp' }],
+      // Regex tests that richmond is not the opponent*
+      fields: [{ name: 'opponent*', type: 'text', value: 'richmond' }],
       expectedError: 'The value does not match the regex',
     },
 
@@ -195,36 +102,37 @@ test.describe('Adding content', () => {
     {
       description: 'empty required text field (single component)',
       fields: [
-        { name: 'testtext', type: 'text', value: 'fill required text' },
         {
-          name: 'testsinglecomp',
+          name: 'most_valuable_player',
           type: 'component',
           value: [
             {
-              category: 'product',
-              name: 'testsinglecomp',
-              fields: [{ name: 'testsinglecomp2text', type: 'text', value: '' }],
+              category: 'match',
+              name: 'player',
+              fields: [{ name: 'name', type: 'text', value: '' }],
             },
           ],
         },
+        { name: 'opponent*', type: 'text', value: 'West Ham' },
       ],
       expectedError: 'This value is required',
     },
     {
       description: 'invalid regexp text field (single component)',
       fields: [
-        { name: 'testtext', type: 'text', value: 'fill required text' },
         {
-          name: 'testcomponent',
+          name: 'most_valuable_player',
           type: 'component',
           value: [
             {
-              category: 'product',
-              name: 'testsinglecomp',
-              fields: [{ name: 'testsinglecomp2text', type: 'text', value: 'fail regexp' }],
+              category: 'match',
+              name: 'player',
+              // Regex tests that nate is not the name
+              fields: [{ name: 'full_name*', type: 'text', value: 'nate' }],
             },
           ],
         },
+        { name: 'opponent*', type: 'text', value: 'West Ham' },
       ],
       expectedError: 'The value does not match the regex',
     },
@@ -233,36 +141,37 @@ test.describe('Adding content', () => {
     {
       description: 'empty required text field (repeatable component)',
       fields: [
-        { name: 'testtext', type: 'text', value: 'fill required text' },
         {
-          name: 'testrepeatablecomp',
+          name: 'lineup',
           type: 'component_repeatable',
           value: [
             {
-              category: 'product',
-              name: 'testrepeatablecomp',
-              fields: [{ name: 'testrepeatablecomp2text', type: 'text', value: '' }],
+              category: 'match',
+              name: 'player',
+              fields: [{ name: 'full_name*', type: 'text', value: '' }],
             },
           ],
         },
+        { name: 'opponent*', type: 'text', value: 'West Ham' },
       ],
       expectedError: 'This value is required',
     },
     {
       description: 'invalid regexp text field (repeatable component)',
       fields: [
-        { name: 'testtext', type: 'text', value: 'fill required text' },
         {
-          name: 'testrepeatablecomp',
+          name: 'lineup',
           type: 'component_repeatable',
           value: [
             {
-              category: 'product',
-              name: 'testrepeatablecomp',
-              fields: [{ name: 'testrepeatablecomp2text', type: 'text', value: 'fail regexp' }],
+              category: 'match',
+              name: 'player',
+              // Regex tests that nate is not the name
+              fields: [{ name: 'full_name*', type: 'text', value: 'nate' }],
             },
           ],
         },
+        { name: 'opponent*', type: 'text', value: 'West Ham' },
       ],
       expectedError: 'The value does not match the regex',
     },
@@ -271,48 +180,48 @@ test.describe('Adding content', () => {
     {
       description: 'empty required text field (dz component)',
       fields: [
-        { name: 'testtext', type: 'text', value: 'fill required text' },
         {
-          name: 'testdz',
+          name: 'sections',
           type: 'dz',
           value: [
             {
-              category: 'product',
-              name: 'newcomponentexistingcategory',
+              category: 'match',
+              name: 'player',
               fields: [
                 {
                   type: 'text',
-                  name: 'testnewcomponentexistingcategorytext',
+                  name: 'full_name*',
                   value: '',
                 },
               ],
             },
           ],
         },
+        { name: 'opponent*', type: 'text', value: 'West Ham' },
       ],
       expectedError: 'This value is required',
     },
     {
       description: 'invalid regexp text field (dz component)',
       fields: [
-        { name: 'testtext', type: 'text', value: 'fill required text' },
         {
-          name: 'testdz',
+          name: 'sections',
           type: 'dz',
           value: [
             {
-              category: 'product',
-              name: 'newcomponentexistingcategory',
+              category: 'match',
+              name: 'player',
               fields: [
                 {
                   type: 'text',
-                  name: 'testnewcomponentexistingcategorytext',
-                  value: 'fail regexp',
+                  name: 'full_name*',
+                  value: 'nate',
                 },
               ],
             },
           ],
         },
+        { name: 'opponent*', type: 'text', value: 'West Ham' },
       ],
       expectedError: 'The value does not match the regex',
     },
@@ -320,7 +229,7 @@ test.describe('Adding content', () => {
 
   for (const { description, fields, expectedError } of testCases) {
     test(`when I publish ${description} I see an error`, async ({ page }) => {
-      await createContent(page, 'Article', fields, { save: false, publish: true, verify: false });
+      await createContent(page, 'Match', fields, { save: false, publish: true, verify: false });
       expect(page.getByText(expectedError)).toBeVisible();
     });
   }

tests/e2e/tests/content-manager/date.spec.ts

@@ -1,9 +1,8 @@
 import { test, expect } from '@playwright/test';
-import { sharedSetup } from '../../utils/setup';
-import { addAttributesToContentType } from '../../utils/content-types';
 import { clickAndWait } from '../../utils/shared';
 import { createContent } from '../../utils/content-creation';
-import { resetFiles } from '../../utils/file-reset';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
+import { login } from '../../utils/login';
 
 // Helper to get date in MM/DD/YYYY format consistently
 function toMMDDYYYY(date: Date) {
@@ -15,40 +14,21 @@ function toMMDDYYYY(date: Date) {
 
 test.describe('Date field tests', () => {
   test.beforeEach(async ({ page }) => {
-    await sharedSetup('ctb-edit-st', page, {
-      login: true,
-      skipTour: true,
-      resetFiles: true,
-      importData: 'with-admin.tar',
-      afterSetup: async ({ page }) => {
-        // Adds a date attribute to the 'Article' content type
-        await addAttributesToContentType(page, 'Article', [
-          {
-            type: 'date',
-            name: 'date',
-            date: {
-              format: 'date', // set to "date" so we only deal with the date (no time)
-            },
-          },
-        ]);
-      },
-    });
+    await resetDatabaseAndImportDataFromPath('with-admin.tar');
+    await page.goto('/admin');
+    await login({ page });
 
     // Navigate to Content Manager
     await clickAndWait(page, page.getByRole('link', { name: 'Content Manager' }));
   });
 
-  test.afterAll(async () => {
-    await resetFiles();
-  });
-
   test('should select the current date from the UI datepicker', async ({ page }) => {
     const today = new Date();
     const zeroPadded = toMMDDYYYY(today);
 
     await createContent(
       page,
-      'Article',
+      'Match',
       [
         {
           type: 'date_date',
@@ -76,7 +56,7 @@ test.describe('Date field tests', () => {
 
     await createContent(
       page,
-      'Article',
+      'Match',
       [
         {
           type: 'date_date',
@@ -103,7 +83,7 @@ test.describe('Date field tests', () => {
 
     await createContent(
       page,
-      'Article',
+      'Match',
       [
         {
           type: 'date_date',

tests/e2e/utils/content-creation.ts

@@ -139,7 +139,7 @@ export const fillField = async (page: Page, field: FieldValue): Promise<void> =>
 
     // all other cases can be handled as text fills
     default:
-      await page.getByLabel(name).fill(String(value));
+      await page.getByLabel(name).last().fill(String(value));
       break;
   }
 };
@@ -184,7 +184,7 @@ export const verifyFields = async (page: Page, fields: FieldValue[]): Promise<vo
         break;
       // TODO: component fields should actually check that they are in the same component
       default:
-        const fieldValue = await page.getByLabel(name).inputValue();
+        const fieldValue = await page.getByLabel(name, { exact: true }).inputValue();
         expect(fieldValue).toBe(String(value)); // Verify text/numeric input values
         break;
     }
@@ -211,16 +211,18 @@ export const createContent = async (
 ): Promise<void> => {
   await navToHeader(page, ['Content Manager', contentType], contentType);
 
-  await clickAndWait(page, page.getByRole('link', { name: 'Create new entry' }));
+  await clickAndWait(page, page.getByRole('link', { name: 'Create new entry' }).last());
 
   await fillFields(page, fields);
 
   if (options.save) {
+    await expect(page.getByRole('button', { name: 'Save' })).toBeEnabled();
     await clickAndWait(page, page.getByRole('button', { name: 'Save' }));
     await findAndClose(page, 'Saved Document', { required: options.verify });
   }
 
   if (options.publish) {
+    await expect(page.getByRole('button', { name: 'Publish' })).toBeEnabled();
     await clickAndWait(page, page.getByRole('button', { name: 'Publish' }));
     await findAndClose(page, 'Published Document', { required: options.verify });
   }