test: add e2e tests for CTB restarts from file changes

docs/docs/guides/e2e/00-setup.md

@@ -34,15 +34,27 @@ If you need to clean the test-apps folder because they are not working as expect
 To run only one domain, meaning a top-level directory in e2e/tests such as "admin" or "content-manager", use the `--domains` option.
 
 ```shell
-yarn test:e2e --domains admin
+yarn test:e2e --domains=admin
-yarn test:e2e --domain admin
+npm run test:e2e --domains=admin
 ```
 
 To run a specific file, you can pass arguments and options to playwright using `--` between the test:e2e options and the playwright options, such as:
 
 ```shell
 # to run just the login.spec.ts file in the admin domain
-yarn test:e2e --domains admin -- login.spec.ts
+yarn test:e2e --domains=admin -- login.spec.ts
+npm run test:e2e --domains=admin -- login.spec.ts
+```
+
+Note that you must still include a domain, otherwise playwright will attempt to run every domain filtering by that filename, and any domains that do not contain that filename will fail with "no tests found"
+
+### Running specific browsers
+
+To run only a specific browser (to speed up test development, for example) you can pass `--project` to playwright with the value(s) `chromium`, `firefox`, or `webkit`
+
+```shell
+yarn test:e2e --domains=admin -- login.spec.ts --project=chromium
+npm run test:e2e --domains=admin -- login.spec.ts --project=chromium
 ```
 
 ### Concurrency / parallellization

packages/core/data-transfer/src/strapi/remote/handlers/utils.ts

@@ -23,9 +23,14 @@ export const transformUpgradeHeader = (header = '') => {
 
 let timeouts: Record<string, number> | undefined;
 
+const hasHttpServer = () => {
+  // during server restarts, strapi may not have ever been defined at all, so we have to check it first
+  return typeof strapi !== 'undefined' && !!strapi?.server?.httpServer;
+};
+
 // temporarily disable server timeouts while transfer is running
 const disableTimeouts = () => {
-  if (!strapi?.server?.httpServer) {
+  if (!hasHttpServer()) {
     return;
   }
 
@@ -45,7 +50,7 @@ const disableTimeouts = () => {
   strapi.log.info('[Data transfer] Disabling http timeouts');
 };
 const resetTimeouts = () => {
-  if (!strapi?.server?.httpServer || !timeouts) {
+  if (!hasHttpServer() || !timeouts) {
     return;
   }
 

playwright.base.config.js

@@ -64,11 +64,6 @@ const createConfig = ({ port, testDir, appDir }) => ({
 
     /* Default time each action such as `click()` can take to 20s */
     actionTimeout: getEnvNum(process.env.PLAYWRIGHT_ACTION_TIMEOUT, 20 * 1000),
-
-    /* Collect trace when a test failed on the CI. See https://playwright.dev/docs/trace-viewer
-       Until https://github.com/strapi/strapi/issues/18196 is fixed we can't enable this locally,
-       because the Strapi server restarts every time a new file (trace) is created.
-    */
     trace: 'retain-on-failure',
     video: getEnvBool(process.env.PLAYWRIGHT_VIDEO, false)
       ? {
@@ -105,8 +100,10 @@ const createConfig = ({ port, testDir, appDir }) => ({
     },
   ],
 
-  /* Folder for test artifacts such as screenshots, videos, traces, etc. */
+  /* Folder for test artifacts such as screenshots, videos, traces, etc.
-  outputDir: getEnvString(process.env.PLAYWRIGHT_OUTPUT_DIR, '../test-results/'), // in the test-apps/e2e dir, to avoid writing files to the running Strapi project dir
+   * Must be outside the project itself or develop mode will restart when files are written
+   * */
+  outputDir: getEnvString(process.env.PLAYWRIGHT_OUTPUT_DIR, '../test-results/'),
 
   /* Run your local dev server before starting the tests */
   webServer: {

tests/e2e/scripts/dts-export.js

@@ -1,95 +1,4 @@
-const {
+const { exportData } = require('../utils/dts-export');
-  file: {
-    providers: { createLocalFileDestinationProvider },
-  },
-  strapi: {
-    providers: { createLocalStrapiSourceProvider },
-  },
-  engine: { createTransferEngine },
-} = require('@strapi/data-transfer');
-const strapiFactory = require('@strapi/strapi');
-const { ALLOWED_CONTENT_TYPES } = require('../constants');
-
-/**
- * Export the data from a strapi project.
- * This script should be run as `node <path-to>/dts-export.js [exportFilePath]` from the
- * root directory of a strapi project e.g. `/examples/kitchensink`. Remember to import
- * the `with-admin` tar file into the project first because the tests rely on the data.
- */
-const exportData = async () => {
-  let args = process.argv.slice(2);
-
-  if (args.length !== 1) {
-    console.error('Please provide the export file name as a parameter.');
-    process.exit(1);
-  }
-
-  const strapi = await createStrapiInstance();
-
-  const source = createSourceProvider(strapi);
-  const destination = createDestinationProvider(args[0]);
-
-  const engine = createTransferEngine(source, destination, {
-    versionStrategy: 'ignore', // for an export to file, versionStrategy will always be skipped
-    schemaStrategy: 'ignore', // for an export to file, schemaStrategy will always be skipped
-    only: ['content', 'files'],
-    transforms: {
-      links: [
-        {
-          filter(link) {
-            return (
-              ALLOWED_CONTENT_TYPES.includes(link.left.type) &&
-              ALLOWED_CONTENT_TYPES.includes(link.right.type)
-            );
-          },
-        },
-      ],
-      entities: [
-        {
-          filter(entity) {
-            return ALLOWED_CONTENT_TYPES.includes(entity.type);
-          },
-        },
-      ],
-    },
-  });
-
-  engine.diagnostics.onDiagnostic(console.log);
-
-  try {
-    const results = await engine.transfer();
-
-    console.log(JSON.stringify(results.engine, null, 2));
-  } catch {
-    console.error('Export process failed.');
-    process.exit(1);
-  }
-
-  process.exit(0);
-};
-
-const createSourceProvider = (strapi) =>
-  createLocalStrapiSourceProvider({
-    async getStrapi() {
-      return strapi;
-    },
-  });
-
-const createDestinationProvider = (filePath) =>
-  createLocalFileDestinationProvider({
-    file: { path: filePath },
-    encryption: { enabled: false },
-    compression: { enabled: false },
-  });
-
-const createStrapiInstance = async (logLevel = 'error') => {
-  const appContext = await strapiFactory.compile();
-  const app = strapiFactory(appContext);
-
-  app.log.level = logLevel;
-  const loadedApp = await app.load();
-
-  return loadedApp;
-};
 
+// TODO: make an actual yargs command and pass common options to exportData so it's easier to build the test data
 exportData();

tests/e2e/tests/admin/login.spec.js

@@ -1,6 +1,6 @@
 import { test, expect } from '@playwright/test';
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
-import { toggleRateLimiting } from '../../scripts/rate-limit';
+import { toggleRateLimiting } from '../../utils/rate-limit';
 import { ADMIN_EMAIL_ADDRESS, ADMIN_PASSWORD } from '../../constants';
 import { login } from '../../utils/login';
 

tests/e2e/tests/admin/logout.spec.js

@@ -1,6 +1,6 @@
 import { test, expect } from '@playwright/test';
 // eslint-disable-next-line import/extensions
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
 import { login } from '../../utils/login';
 
 test.describe('Log Out', () => {

tests/e2e/tests/admin/signup.spec.js

@@ -1,6 +1,6 @@
 import { test, expect } from '@playwright/test';
 
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
 import { ADMIN_EMAIL_ADDRESS, ADMIN_PASSWORD } from '../../constants';
 
 /**

tests/e2e/tests/admin/transfer/tokens.spec.ts

@@ -1,6 +1,6 @@
 import { test, expect } from '@playwright/test';
 import { login } from '../../../utils/login';
-import { resetDatabaseAndImportDataFromPath } from '../../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../../utils/dts-import';
 import { navToHeader } from '../../../utils/shared';
 
 const createTransferToken = async (page, tokenName, duration, type) => {

tests/e2e/tests/content-manager/editview.spec.ts

@@ -1,6 +1,6 @@
 import { test, expect } from '@playwright/test';
 import { login } from '../../utils/login';
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
 import { findAndClose } from '../../utils/shared';
 
 test.describe('Edit View', () => {

tests/e2e/tests/content-manager/listview.spec.ts

@@ -3,7 +3,7 @@
  */
 import { test, expect } from '@playwright/test';
 import { login } from '../../utils/login';
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
 
 test.describe('List View', () => {
   test.beforeEach(async ({ page }) => {

tests/e2e/tests/content-releases/release-details-page.spec.ts

@@ -1,6 +1,6 @@
 import { test, expect, type Page } from '@playwright/test';
 import { describeOnCondition } from '../../utils/shared';
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
 import { login } from '../../utils/login';
 
 const edition = process.env.STRAPI_DISABLE_EE === 'true' ? 'CE' : 'EE';

tests/e2e/tests/content-releases/releases-page.spec.ts

@@ -1,6 +1,6 @@
 import { test, expect } from '@playwright/test';
 import { describeOnCondition } from '../../utils/shared';
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
 import { login } from '../../utils/login';
 
 const edition = process.env.STRAPI_DISABLE_EE === 'true' ? 'CE' : 'EE';

tests/e2e/tests/content-type-builder/collection-type/create-collection-type.spec.ts

@@ -0,0 +1,56 @@
+import { test, expect } from '@playwright/test';
+import { login } from '../../../utils/login';
+import { resetDatabaseAndImportDataFromPath } from '../../../utils/dts-import';
+import { waitForRestart } from '../../../utils/restart';
+import { resetFiles } from '../../../utils/file-reset';
+
+test.describe('Create collection type', () => {
+  test.beforeEach(async ({ page }) => {
+    await resetFiles();
+    await resetDatabaseAndImportDataFromPath('with-admin.tar');
+    await page.goto('/admin');
+    await login({ page });
+
+    await page.getByRole('link', { name: 'Content-Type Builder' }).click();
+
+    // close the tutorial modal if it's visible
+    const modal = page.getByRole('button', { name: 'Close' });
+    if (modal.isVisible()) {
+      await modal.click();
+      await expect(modal).not.toBeVisible();
+    }
+  });
+
+  // TODO: each test should have a beforeAll that does this, maybe combine all the setup into one util to simplify it
+  // to keep other suites that don't modify files from needing to reset files, clean up after ourselves at the end
+  test.afterAll(async () => {
+    await resetFiles();
+  });
+
+  test('Can create a collection type', async ({ page }) => {
+    await page.getByRole('button', { name: 'Create new collection type' }).click();
+
+    await expect(page.getByRole('heading', { name: 'Create a collection type' })).toBeVisible();
+
+    const displayName = page.getByLabel('Display name');
+    await displayName.fill('Secret Document');
+
+    const singularId = page.getByLabel('API ID (Singular)');
+    await expect(singularId).toHaveValue('secret-document');
+
+    const pluralId = page.getByLabel('API ID (Plural)');
+    await expect(pluralId).toHaveValue('secret-documents');
+
+    await page.getByRole('button', { name: 'Continue' }).click();
+
+    await expect(page.getByText('Select a field for your collection type')).toBeVisible();
+    await page.getByText('Small or long text').click();
+    await page.getByLabel('Name', { exact: true }).fill('myattribute');
+    await page.getByRole('button', { name: 'Finish' }).click();
+    await page.getByRole('button', { name: 'Save' }).click();
+
+    await waitForRestart(page);
+
+    await expect(page.getByRole('heading', { name: 'Secret Document' })).toBeVisible();
+  });
+});

tests/e2e/tests/content-type-builder/ctb-edit-view.spec.ts

@@ -1,7 +1,7 @@
 import { test, expect } from '@playwright/test';
 import { login } from '../../utils/login';
 import { navToHeader } from '../../utils/shared';
-import { resetDatabaseAndImportDataFromPath } from '../../scripts/dts-import';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
 
 test.describe('Edit View CTB', () => {
   test.beforeEach(async ({ page }) => {

tests/e2e/tests/content-type-builder/tutorial.spec.ts

@@ -0,0 +1,26 @@
+import { test, expect } from '@playwright/test';
+import { login } from '../../utils/login';
+import { resetDatabaseAndImportDataFromPath } from '../../utils/dts-import';
+
+test.describe('Tutorial', () => {
+  test.beforeEach(async ({ page }) => {
+    await resetDatabaseAndImportDataFromPath('with-admin.tar');
+    await page.goto('/admin');
+    await login({ page });
+  });
+
+  test('Shows tutorial on first content type', async ({ page }) => {
+    await page.getByRole('link', { name: 'Content-type Builder' }).click();
+
+    const modalHeader = page.getByRole('heading', { name: '🧠 Create a first Collection' });
+    expect(modalHeader).toBeVisible();
+    await modalHeader.click();
+
+    const closeButton = page.getByRole('button', { name: 'Close' });
+    expect(closeButton).toBeVisible();
+    await closeButton.click();
+
+    await expect(closeButton).not.toBeVisible();
+    await expect(modalHeader).not.toBeVisible();
+  });
+});

tests/e2e/utils/dts-export.js

@@ -0,0 +1,93 @@
+const {
+  file: {
+    providers: { createLocalFileDestinationProvider },
+  },
+  strapi: {
+    providers: { createLocalStrapiSourceProvider },
+  },
+  engine: { createTransferEngine },
+} = require('@strapi/data-transfer');
+const strapiFactory = require('@strapi/strapi');
+const { ALLOWED_CONTENT_TYPES } = require('../constants');
+
+/**
+ * Export the data from a strapi project.
+ * This script should be run as `node <path-to>/dts-export.js [exportFilePath]` from the
+ * root directory of a strapi project e.g. `/examples/kitchensink`. Remember to import
+ * the `with-admin` tar file into the project first because the tests rely on the data.
+ */
+export const exportData = async () => {
+  let args = process.argv.slice(2);
+
+  if (args.length !== 1) {
+    console.error('Please provide the export file name as a parameter.');
+    process.exit(1);
+  }
+
+  const strapi = await createStrapiInstance();
+
+  const source = createSourceProvider(strapi);
+  const destination = createDestinationProvider(args[0]);
+
+  const engine = createTransferEngine(source, destination, {
+    versionStrategy: 'ignore', // for an export to file, versionStrategy will always be skipped
+    schemaStrategy: 'ignore', // for an export to file, schemaStrategy will always be skipped
+    only: ['content', 'files'],
+    transforms: {
+      links: [
+        {
+          filter(link) {
+            return (
+              ALLOWED_CONTENT_TYPES.includes(link.left.type) &&
+              ALLOWED_CONTENT_TYPES.includes(link.right.type)
+            );
+          },
+        },
+      ],
+      entities: [
+        {
+          filter(entity) {
+            return ALLOWED_CONTENT_TYPES.includes(entity.type);
+          },
+        },
+      ],
+    },
+  });
+
+  engine.diagnostics.onDiagnostic(console.log);
+
+  try {
+    const results = await engine.transfer();
+
+    console.log(JSON.stringify(results.engine, null, 2));
+  } catch {
+    console.error('Export process failed.');
+    process.exit(1);
+  }
+
+  process.exit(0);
+};
+
+const createSourceProvider = (strapi) =>
+  createLocalStrapiSourceProvider({
+    async getStrapi() {
+      return strapi;
+    },
+  });
+
+const createDestinationProvider = (filePath) =>
+  createLocalFileDestinationProvider({
+    file: { path: filePath },
+    encryption: { enabled: false },
+    compression: { enabled: false },
+  });
+
+const createStrapiInstance = async (logLevel = 'error') => {
+  const appContext = await strapiFactory.compile();
+  const app = strapiFactory(appContext);
+
+  app.log.level = logLevel;
+  const loadedApp = await app.load();
+
+  return loadedApp;
+};

tests/e2e/utils/file-reset.js

@@ -0,0 +1,54 @@
+import execa from 'execa';
+
+const gitUser = ['-c', 'user.name=Strapi CLI', '-c', 'user.email=test@strapi.io'];
+
+function delay(seconds) {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+export const pollHealthCheck = async (interval = 1000, timeout = 30000) => {
+  const url = `http://127.0.0.1:${process.env.PORT ?? 1337}/_health`;
+  console.log(`Starting to poll: ${url}`);
+
+  let elapsed = 0;
+
+  while (elapsed < timeout) {
+    try {
+      const response = await fetch(url, { method: 'HEAD' });
+      if (response.ok) {
+        console.log('The service is up and running!');
+        return; // Exit if the service is up
+      }
+      // If the response is not okay, throw an error to catch it below
+      throw new Error('Service not ready');
+    } catch (error) {
+      console.log('Waiting for the service to come up...');
+      // Wait for the specified interval before trying again
+      await new Promise((resolve) => setTimeout(resolve, interval));
+      elapsed += interval; // Update the elapsed time
+    }
+  }
+
+  // If we've exited the loop because of the timeout
+  console.error('Timeout reached, service did not become available in time.');
+};
+
+export const resetFiles = async () => {
+  if (process.env.TEST_APP_PATH) {
+    console.log('Restoring filesystem');
+    await execa('git', [...gitUser, 'reset', '--hard'], {
+      stdio: 'inherit',
+      cwd: process.env.TEST_APP_PATH,
+    });
+    const dryRun = await execa('git', [...gitUser, 'clean', '-fd'], {
+      stdio: 'inherit',
+      cwd: process.env.TEST_APP_PATH,
+    });
+  }
+
+  // wait for server to restart after modifying files
+  console.log('Waiting for Strapi to restart...');
+  // TODO: this is both a waste of time and flaky. We need to find a way to access playwright server output and watch for the "up" log to appear
+  await delay(3); // give it time to detect file changes and begin its restart
+  await pollHealthCheck(); // give it time to come back up
+};

tests/e2e/utils/restart.ts

@@ -0,0 +1,50 @@
+import { Page, expect } from '@playwright/test';
+
+// Function to check modal visibility
+const isModalVisible = async (page: Page) => {
+  return page.isVisible('text="Waiting for restart..."');
+};
+
+/**
+ * Wait for a restart modal to appear, but instead of failing if it doesn't, attempt to
+ * refresh the page and see if it comes back up
+ */
+export const waitForRestart = async (page, timeout = 60000) => {
+  const initialWaitForModal = 5000; // Time to wait for the modal to initially appear
+  let elapsedTime = 0;
+  const checkInterval = 1000; // Check every 1 second
+  const reloadTimeout = 15000; // 15 seconds before trying to reload
+
+  // Initially wait for the modal to become visible
+  try {
+    await page.waitForSelector('text="Waiting for restart..."', {
+      state: 'visible',
+      timeout: initialWaitForModal,
+    });
+  } catch (error) {
+    console.log('The modal did not become visible within the initial wait period.');
+    throw error; // Or handle this scenario as appropriate
+  }
+
+  // Now wait until the modal is not visible or until the reloadTimeout
+  let modalVisible = await isModalVisible(page);
+  while (modalVisible && elapsedTime < reloadTimeout) {
+    await new Promise((r) => setTimeout(r, checkInterval));
+    elapsedTime += checkInterval;
+    modalVisible = await isModalVisible(page);
+  }
+
+  // If modal is still visible after reloadTimeout, reload the page and wait again
+  if (modalVisible) {
+    console.log("Restart overlay didn't disappear after 15 seconds. Reloading page...");
+    await page.reload({ waitUntil: 'domcontentloaded' });
+    // Optionally, wait again for the modal to disappear after reloading
+  }
+
+  // Final check to ensure the modal has disappeared
+  if (await isModalVisible(page)) {
+    throw new Error('Restart overlay did not disappear after waiting and reloading.');
+  }
+
+  console.log('Restart overlay has disappeared, proceeding with the test.');
+};

tests/scripts/run-e2e-tests.js

@@ -59,10 +59,10 @@ const setupTestEnvironment = async (generatedAppPath) => {
 yargs
   .parserConfiguration({
     /**
-     * This lets us pass any other arguments to playwright
+     * When unknown options is false, using -- to separate playwright args from test:e2e args works
-     * e.g. the name of a specific test or the project we want to run
+     * When it is true, the script gets confused about additional arguments, with or without using -- to separate commands
      */
-    'unknown-options-as-args': true,
+    'unknown-options-as-args': false,
   })
   .command({
     command: '*',
@@ -214,6 +214,28 @@ module.exports = config
 
               await fs.writeFile(pathToPlaywrightConfig, configFileTemplate);
 
+              // Store the filesystem state with git so it can be reset between tests
+              // TODO: if we have a large test test suite, it might be worth it to run a `strapi start` and then shutdown here to generate documentation and types only once and save unneccessary server restarts from those files being cleared every time
+              console.log('Initializing git');
+
+              const gitUser = ['-c', 'user.name=Strapi CLI', '-c', 'user.email=test@strapi.io'];
+
+              await execa('git', [...gitUser, 'init'], {
+                stdio: 'inherit',
+                cwd: testAppPath,
+              });
+
+              // we need to use -A to track even hidden files like .env; remember we're only using git as a file state manager
+              await execa('git', [...gitUser, 'add', '-A', '.'], {
+                stdio: 'inherit',
+                cwd: testAppPath,
+              });
+
+              await execa('git', [...gitUser, 'commit', '-m', 'initial commit'], {
+                stdio: 'inherit',
+                cwd: testAppPath,
+              });
+
               console.log(`Running ${chalk.blue(domain)} e2e tests`);
 
               await execa(
@@ -225,6 +247,7 @@ module.exports = config
                   env: {
                     PORT: port,
                     HOST: '127.0.0.1',
+                    TEST_APP_PATH: testAppPath,
                     STRAPI_DISABLE_EE: !process.env.STRAPI_LICENSE,
                   },
                 }