feat(database): add internal migrations

docs/docs/docs/01-core/database/03-migrations.md

@@ -0,0 +1,7 @@
+---
+title: Migrations
+description: Conceptual guide to migrations in Strapi
+tags:
+  - database
+  - migration
+---

packages/core/database/src/migrations/__tests__/common.test.ts

@@ -0,0 +1,47 @@
+import { wrapTransaction } from '../common';
+import { Database } from '../..';
+
+describe('wrapTransaction', () => {
+  let db: Database;
+  let fn: jest.Mock;
+  const trx: jest.Mock = jest.fn();
+
+  beforeEach(() => {
+    db = {
+      connection: {
+        transaction: jest.fn().mockImplementation((cb) => cb(trx)),
+      },
+    } as any;
+
+    fn = jest.fn().mockResolvedValue(undefined);
+  });
+
+  it('should wrap the function in a transaction', async () => {
+    const wrappedFn = wrapTransaction(db)(fn);
+    await wrappedFn();
+
+    expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function));
+    expect(fn).toHaveBeenCalledWith(trx, db);
+  });
+
+  it('should return the result of the wrapped function', async () => {
+    const result = {};
+    fn.mockResolvedValueOnce(result);
+
+    const wrappedFn = wrapTransaction(db)(fn);
+    const res = await wrappedFn();
+
+    expect(res).toBe(result);
+  });
+
+  it('should rollback the transaction if the wrapped function throws an error', async () => {
+    const error = new Error('Test error');
+    fn.mockRejectedValueOnce(error);
+
+    const wrappedFn = wrapTransaction(db)(fn);
+
+    await expect(wrappedFn()).rejects.toThrow(error);
+    expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function));
+    expect(fn).toHaveBeenCalledWith(trx, db);
+  });
+});

packages/core/database/src/migrations/__tests__/storage.test.ts

@@ -0,0 +1,92 @@
+import { createStorage } from '../storage';
+
+import { Database } from '../..';
+
+describe('createStorage', () => {
+  let db: Database;
+  let tableName: string;
+  let storage: ReturnType<typeof createStorage>;
+
+  beforeEach(() => {
+    db = {
+      getSchemaConnection: jest.fn().mockReturnValue({
+        hasTable: jest.fn().mockResolvedValue(false),
+        createTable: jest.fn().mockResolvedValue(undefined),
+      }),
+      getConnection: jest.fn().mockReturnValue({
+        insert: jest.fn().mockReturnValue({
+          into: jest.fn().mockResolvedValue(undefined),
+        }),
+        del: jest.fn().mockReturnValue({
+          where: jest.fn().mockResolvedValue(undefined),
+        }),
+        select: jest.fn().mockReturnValue({
+          from: jest.fn().mockReturnValue({
+            orderBy: jest.fn().mockResolvedValue([]),
+          }),
+        }),
+      }),
+    } as any;
+
+    tableName = 'migrations';
+    storage = createStorage({ db, tableName });
+  });
+
+  describe('logMigration', () => {
+    it('should insert a new migration log', async () => {
+      const name = '20220101120000_create_users_table';
+      const time = new Date();
+
+      await storage.logMigration({ name });
+
+      expect(db.getConnection().insert).toHaveBeenCalledWith({ name, time });
+      expect(db.getConnection().insert({}).into).toHaveBeenCalledWith(tableName);
+    });
+  });
+
+  describe('unlogMigration', () => {
+    it('should delete a migration log', async () => {
+      const name = '20220101120000_create_users_table';
+
+      await storage.unlogMigration({ name });
+
+      expect(db.getConnection().del).toHaveBeenCalled();
+      expect(db.getConnection().del().where).toHaveBeenCalledWith({ name });
+    });
+  });
+
+  describe('executed', () => {
+    it('should create migration table if it does not exist', async () => {
+      await storage.executed();
+
+      expect(db.getSchemaConnection().hasTable).toHaveBeenCalledWith(tableName);
+      expect(db.getSchemaConnection().createTable).toHaveBeenCalledWith(
+        tableName,
+        expect.any(Function)
+      );
+    });
+
+    it('should return an empty array if no migration has been executed', async () => {
+      const result = await storage.executed();
+
+      expect(result).toEqual([]);
+    });
+
+    it('should return an array of executed migration names', async () => {
+      const logs = [
+        { name: '20220101120000_create_users_table' },
+        { name: '20220101130000_create_posts_table' },
+      ];
+
+      (db.getSchemaConnection().hasTable as jest.Mock).mockResolvedValue(true);
+      (db.getConnection().select().from('').orderBy as jest.Mock).mockResolvedValue(logs);
+
+      const result = await storage.executed();
+
+      expect(result).toEqual([
+        '20220101120000_create_users_table',
+        '20220101130000_create_posts_table',
+      ]);
+    });
+  });
+});

packages/core/database/src/migrations/common.ts

@@ -0,0 +1,26 @@
+import type { Resolver } from 'umzug';
+import type { Knex } from 'knex';
+
+import type { Database } from '..';
+
+export interface MigrationProvider {
+  shouldRun(): Promise<boolean>;
+  up(): Promise<void>;
+  down(): Promise<void>;
+}
+
+export type Context = { db: Database };
+
+export type MigrationResolver = Resolver<Context>;
+
+export type MigrationFn = (knex: Knex, db: Database) => Promise<void>;
+
+export type Migration = {
+  name: string;
+  up: MigrationFn;
+  down: MigrationFn;
+};
+
+export const wrapTransaction = (db: Database) => (fn: MigrationFn) => () => {
+  return db.connection.transaction((trx) => Promise.resolve(fn(trx, db)));
+};

packages/core/database/src/migrations/index.ts

@@ -1,93 +1,35 @@
-import path from 'node:path';
+import { createUserMigrationProvider } from './users';
-import fse from 'fs-extra';
+import { createInternalMigrationProvider } from './internal';
-import { Umzug } from 'umzug';
-
-import type { Resolver } from 'umzug';
-import type { Knex } from 'knex';
-
-import { createStorage } from './storage';
 
+import type { MigrationProvider } from './common';
 import type { Database } from '..';
 
-export interface MigrationProvider {
+export type { MigrationProvider };
-  shouldRun(): Promise<boolean>;
-  up(): Promise<void>;
-  down(): Promise<void>;
-}
 
-type MigrationResolver = Resolver<{ db: Database }>;
-
-const wrapTransaction = (db: Database) => (fn: (knex: Knex) => unknown) => () => {
-  return db.connection.transaction((trx) => Promise.resolve(fn(trx)));
-};
-
-// TODO: check multiple commands in one sql statement
-const migrationResolver: MigrationResolver = ({ name, path, context }) => {
-  const { db } = context;
-
-  if (!path) {
-    throw new Error(`Migration ${name} has no path`);
-  }
-
-  // if sql file run with knex raw
-  if (path.match(/\.sql$/)) {
-    const sql = fse.readFileSync(path, 'utf8');
-
-    return {
-      name,
-      up: wrapTransaction(db)((knex) => knex.raw(sql)),
-      async down() {
-        throw new Error('Down migration is not supported for sql files');
-      },
-    };
-  }
-
-  // NOTE: we can add some ts register if we want to handle ts migration files at some point
-  // eslint-disable-next-line @typescript-eslint/no-var-requires
-  const migration = require(path);
-  return {
-    name,
-    up: wrapTransaction(db)(migration.up),
-    down: wrapTransaction(db)(migration.down),
-  };
-};
-
-const createUmzugProvider = (db: Database) => {
-  const migrationDir = path.join(strapi.dirs.app.root, 'database/migrations');
-
-  fse.ensureDirSync(migrationDir);
-
-  return new Umzug({
-    storage: createStorage({ db, tableName: 'strapi_migrations' }),
-    logger: console,
-    context: { db },
-    migrations: {
-      glob: ['*.{js,sql}', { cwd: migrationDir }],
-      resolve: migrationResolver,
-    },
-  });
-};
-
-// NOTE: when needed => add internal migrations for core & plugins. How do we overlap them with users migrations ?
-
-/**
- * Creates migrations provider
- * @type {import('.').createMigrationsProvider}
- */
 export const createMigrationsProvider = (db: Database): MigrationProvider => {
-  const migrations = createUmzugProvider(db);
+  const providers = [createUserMigrationProvider(db), createInternalMigrationProvider(db)];
 
   return {
     async shouldRun() {
-      const pending = await migrations.pending();
+      const shouldRunResponses = await Promise.all(
+        providers.map((provider) => provider.shouldRun())
+      );
 
-      return pending.length > 0 && db.config?.settings?.runMigrations === true;
+      return shouldRunResponses.some((shouldRun) => shouldRun);
     },
     async up() {
-      await migrations.up();
+      for (const provider of providers) {
+        if (await provider.shouldRun()) {
+          await provider.up();
+        }
+      }
     },
     async down() {
-      await migrations.down();
+      for (const provider of providers) {
+        if (await provider.shouldRun()) {
+          await provider.down();
+        }
+      }
     },
   };
 };

packages/core/database/src/migrations/internal-migrations/index.ts

@@ -0,0 +1,12 @@
+import type { Migration } from '../common';
+
+/**
+ * List of all the internal migrations. The array order will be the order in which they are executed.
+ *
+ * {
+ *   name: 'some-name',
+ *   async up(knex: Knex, db: Database) {},
+ *   async down(knex: Knex, db: Database) {},
+ * },
+ */
+export const internalMigrations: Migration[] = [];

packages/core/database/src/migrations/internal.ts

@@ -0,0 +1,38 @@
+import { Umzug } from 'umzug';
+
+import { wrapTransaction } from './common';
+import { internalMigrations } from './internal-migrations';
+import { createStorage } from './storage';
+
+import type { MigrationProvider } from './common';
+import type { Database } from '..';
+
+export const createInternalMigrationProvider = (db: Database): MigrationProvider => {
+  const context = { db };
+
+  const umzugProvider = new Umzug({
+    storage: createStorage({ db, tableName: 'strapi_migrations_internal' }),
+    logger: console,
+    context,
+    migrations: internalMigrations.map((migration) => {
+      return {
+        name: migration.name,
+        up: wrapTransaction(context.db)(migration.up),
+        down: wrapTransaction(context.db)(migration.down),
+      };
+    }),
+  });
+
+  return {
+    async shouldRun() {
+      const pendingMigrations = await umzugProvider.pending();
+      return pendingMigrations.length > 0;
+    },
+    async up() {
+      await umzugProvider.up();
+    },
+    async down() {
+      await umzugProvider.down();
+    },
+  };
+};

packages/core/database/src/migrations/storage.ts

@@ -2,11 +2,11 @@ import type { Database } from '..';
 
 export interface Options {
   db: Database;
-  tableName?: string;
+  tableName: string;
 }
 
 export const createStorage = (opts: Options) => {
-  const { db, tableName = 'strapi_migrations' } = opts;
+  const { db, tableName } = opts;
 
   const hasMigrationTable = () => db.getSchemaConnection().hasTable(tableName);
 

packages/core/database/src/migrations/users.ts

@@ -0,0 +1,70 @@
+import path from 'node:path';
+import fse from 'fs-extra';
+import { Umzug } from 'umzug';
+
+import { createStorage } from './storage';
+import { wrapTransaction } from './common';
+import type { MigrationProvider, MigrationResolver } from './common';
+import type { Database } from '..';
+
+// TODO: check multiple commands in one sql statement
+const migrationResolver: MigrationResolver = ({ name, path, context }) => {
+  const { db } = context;
+
+  if (!path) {
+    throw new Error(`Migration ${name} has no path`);
+  }
+
+  // if sql file run with knex raw
+  if (path.match(/\.sql$/)) {
+    const sql = fse.readFileSync(path, 'utf8');
+
+    return {
+      name,
+      up: wrapTransaction(db)((knex) => knex.raw(sql)),
+      async down() {
+        throw new Error('Down migration is not supported for sql files');
+      },
+    };
+  }
+
+  // NOTE: we can add some ts register if we want to handle ts migration files at some point
+  // eslint-disable-next-line @typescript-eslint/no-var-requires
+  const migration = require(path);
+  return {
+    name,
+    up: wrapTransaction(db)(migration.up),
+    down: wrapTransaction(db)(migration.down),
+  };
+};
+
+export const createUserMigrationProvider = (db: Database): MigrationProvider => {
+  const dir = path.join(strapi.dirs.app.root, 'database/migrations');
+
+  fse.ensureDirSync(dir);
+
+  const context = { db };
+
+  const umzugProvider = new Umzug({
+    storage: createStorage({ db, tableName: 'strapi_migrations' }),
+    logger: console,
+    context,
+    migrations: {
+      glob: ['*.{js,sql}', { cwd: dir }],
+      resolve: migrationResolver,
+    },
+  });
+
+  return {
+    async shouldRun() {
+      const pendingMigrations = await umzugProvider.pending();
+      return pendingMigrations.length > 0 && db.config?.settings?.runMigrations === true;
+    },
+    async up() {
+      await umzugProvider.up();
+    },
+    async down() {
+      await umzugProvider.down();
+    },
+  };
+};

packages/core/database/src/schema/diff.ts

@@ -16,7 +16,12 @@ import type {
 } from './types';
 import type { Database } from '..';
 
-const RESERVED_TABLE_NAMES = ['strapi_migrations', 'strapi_database_schema'];
+// TODO: get that list dynamically instead
+const RESERVED_TABLE_NAMES = [
+  'strapi_migrations',
+  'strapi_migrations_internal',
+  'strapi_database_schema',
+];
 
 const statuses = {
   CHANGED: 'CHANGED',