diff --git a/docs/docs/docs/01-core/database/03-migrations.md b/docs/docs/docs/01-core/database/03-migrations.md new file mode 100644 index 0000000000..bf92ee75cd --- /dev/null +++ b/docs/docs/docs/01-core/database/03-migrations.md @@ -0,0 +1,7 @@ +--- +title: Migrations +description: Conceptual guide to migrations in Strapi +tags: + - database + - migration +--- diff --git a/packages/core/database/src/migrations/__tests__/common.test.ts b/packages/core/database/src/migrations/__tests__/common.test.ts new file mode 100644 index 0000000000..71843140fa --- /dev/null +++ b/packages/core/database/src/migrations/__tests__/common.test.ts @@ -0,0 +1,47 @@ +import { wrapTransaction } from '../common'; +import { Database } from '../..'; + +describe('wrapTransaction', () => { + let db: Database; + let fn: jest.Mock; + const trx: jest.Mock = jest.fn(); + + beforeEach(() => { + db = { + connection: { + transaction: jest.fn().mockImplementation((cb) => cb(trx)), + }, + } as any; + + fn = jest.fn().mockResolvedValue(undefined); + }); + + it('should wrap the function in a transaction', async () => { + const wrappedFn = wrapTransaction(db)(fn); + await wrappedFn(); + + expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function)); + expect(fn).toHaveBeenCalledWith(trx, db); + }); + + it('should return the result of the wrapped function', async () => { + const result = {}; + fn.mockResolvedValueOnce(result); + + const wrappedFn = wrapTransaction(db)(fn); + const res = await wrappedFn(); + + expect(res).toBe(result); + }); + + it('should rollback the transaction if the wrapped function throws an error', async () => { + const error = new Error('Test error'); + fn.mockRejectedValueOnce(error); + + const wrappedFn = wrapTransaction(db)(fn); + + await expect(wrappedFn()).rejects.toThrow(error); + expect(db.connection.transaction).toHaveBeenCalledWith(expect.any(Function)); + expect(fn).toHaveBeenCalledWith(trx, db); + }); +}); diff --git a/packages/core/database/src/migrations/__tests__/storage.test.ts b/packages/core/database/src/migrations/__tests__/storage.test.ts new file mode 100644 index 0000000000..215a58b999 --- /dev/null +++ b/packages/core/database/src/migrations/__tests__/storage.test.ts @@ -0,0 +1,92 @@ +import { createStorage } from '../storage'; + +import { Database } from '../..'; + +describe('createStorage', () => { + let db: Database; + let tableName: string; + let storage: ReturnType; + + beforeEach(() => { + db = { + getSchemaConnection: jest.fn().mockReturnValue({ + hasTable: jest.fn().mockResolvedValue(false), + createTable: jest.fn().mockResolvedValue(undefined), + }), + getConnection: jest.fn().mockReturnValue({ + insert: jest.fn().mockReturnValue({ + into: jest.fn().mockResolvedValue(undefined), + }), + del: jest.fn().mockReturnValue({ + where: jest.fn().mockResolvedValue(undefined), + }), + select: jest.fn().mockReturnValue({ + from: jest.fn().mockReturnValue({ + orderBy: jest.fn().mockResolvedValue([]), + }), + }), + }), + } as any; + + tableName = 'migrations'; + storage = createStorage({ db, tableName }); + }); + + describe('logMigration', () => { + it('should insert a new migration log', async () => { + const name = '20220101120000_create_users_table'; + const time = new Date(); + + await storage.logMigration({ name }); + + expect(db.getConnection().insert).toHaveBeenCalledWith({ name, time }); + expect(db.getConnection().insert({}).into).toHaveBeenCalledWith(tableName); + }); + }); + + describe('unlogMigration', () => { + it('should delete a migration log', async () => { + const name = '20220101120000_create_users_table'; + + await storage.unlogMigration({ name }); + + expect(db.getConnection().del).toHaveBeenCalled(); + expect(db.getConnection().del().where).toHaveBeenCalledWith({ name }); + }); + }); + + describe('executed', () => { + it('should create migration table if it does not exist', async () => { + await storage.executed(); + + expect(db.getSchemaConnection().hasTable).toHaveBeenCalledWith(tableName); + expect(db.getSchemaConnection().createTable).toHaveBeenCalledWith( + tableName, + expect.any(Function) + ); + }); + + it('should return an empty array if no migration has been executed', async () => { + const result = await storage.executed(); + + expect(result).toEqual([]); + }); + + it('should return an array of executed migration names', async () => { + const logs = [ + { name: '20220101120000_create_users_table' }, + { name: '20220101130000_create_posts_table' }, + ]; + + (db.getSchemaConnection().hasTable as jest.Mock).mockResolvedValue(true); + (db.getConnection().select().from('').orderBy as jest.Mock).mockResolvedValue(logs); + + const result = await storage.executed(); + + expect(result).toEqual([ + '20220101120000_create_users_table', + '20220101130000_create_posts_table', + ]); + }); + }); +}); diff --git a/packages/core/database/src/migrations/common.ts b/packages/core/database/src/migrations/common.ts new file mode 100644 index 0000000000..ffe7257a94 --- /dev/null +++ b/packages/core/database/src/migrations/common.ts @@ -0,0 +1,26 @@ +import type { Resolver } from 'umzug'; +import type { Knex } from 'knex'; + +import type { Database } from '..'; + +export interface MigrationProvider { + shouldRun(): Promise; + up(): Promise; + down(): Promise; +} + +export type Context = { db: Database }; + +export type MigrationResolver = Resolver; + +export type MigrationFn = (knex: Knex, db: Database) => Promise; + +export type Migration = { + name: string; + up: MigrationFn; + down: MigrationFn; +}; + +export const wrapTransaction = (db: Database) => (fn: MigrationFn) => () => { + return db.connection.transaction((trx) => Promise.resolve(fn(trx, db))); +}; diff --git a/packages/core/database/src/migrations/index.ts b/packages/core/database/src/migrations/index.ts index bf10e108e1..a15c8d50bb 100644 --- a/packages/core/database/src/migrations/index.ts +++ b/packages/core/database/src/migrations/index.ts @@ -1,93 +1,35 @@ -import path from 'node:path'; -import fse from 'fs-extra'; -import { Umzug } from 'umzug'; - -import type { Resolver } from 'umzug'; -import type { Knex } from 'knex'; - -import { createStorage } from './storage'; +import { createUserMigrationProvider } from './users'; +import { createInternalMigrationProvider } from './internal'; +import type { MigrationProvider } from './common'; import type { Database } from '..'; -export interface MigrationProvider { - shouldRun(): Promise; - up(): Promise; - down(): Promise; -} +export type { MigrationProvider }; -type MigrationResolver = Resolver<{ db: Database }>; - -const wrapTransaction = (db: Database) => (fn: (knex: Knex) => unknown) => () => { - return db.connection.transaction((trx) => Promise.resolve(fn(trx))); -}; - -// TODO: check multiple commands in one sql statement -const migrationResolver: MigrationResolver = ({ name, path, context }) => { - const { db } = context; - - if (!path) { - throw new Error(`Migration ${name} has no path`); - } - - // if sql file run with knex raw - if (path.match(/\.sql$/)) { - const sql = fse.readFileSync(path, 'utf8'); - - return { - name, - up: wrapTransaction(db)((knex) => knex.raw(sql)), - async down() { - throw new Error('Down migration is not supported for sql files'); - }, - }; - } - - // NOTE: we can add some ts register if we want to handle ts migration files at some point - // eslint-disable-next-line @typescript-eslint/no-var-requires - const migration = require(path); - return { - name, - up: wrapTransaction(db)(migration.up), - down: wrapTransaction(db)(migration.down), - }; -}; - -const createUmzugProvider = (db: Database) => { - const migrationDir = path.join(strapi.dirs.app.root, 'database/migrations'); - - fse.ensureDirSync(migrationDir); - - return new Umzug({ - storage: createStorage({ db, tableName: 'strapi_migrations' }), - logger: console, - context: { db }, - migrations: { - glob: ['*.{js,sql}', { cwd: migrationDir }], - resolve: migrationResolver, - }, - }); -}; - -// NOTE: when needed => add internal migrations for core & plugins. How do we overlap them with users migrations ? - -/** - * Creates migrations provider - * @type {import('.').createMigrationsProvider} - */ export const createMigrationsProvider = (db: Database): MigrationProvider => { - const migrations = createUmzugProvider(db); + const providers = [createUserMigrationProvider(db), createInternalMigrationProvider(db)]; return { async shouldRun() { - const pending = await migrations.pending(); + const shouldRunResponses = await Promise.all( + providers.map((provider) => provider.shouldRun()) + ); - return pending.length > 0 && db.config?.settings?.runMigrations === true; + return shouldRunResponses.some((shouldRun) => shouldRun); }, async up() { - await migrations.up(); + for (const provider of providers) { + if (await provider.shouldRun()) { + await provider.up(); + } + } }, async down() { - await migrations.down(); + for (const provider of providers) { + if (await provider.shouldRun()) { + await provider.down(); + } + } }, }; }; diff --git a/packages/core/database/src/migrations/internal-migrations/index.ts b/packages/core/database/src/migrations/internal-migrations/index.ts new file mode 100644 index 0000000000..6dc83101cf --- /dev/null +++ b/packages/core/database/src/migrations/internal-migrations/index.ts @@ -0,0 +1,12 @@ +import type { Migration } from '../common'; + +/** + * List of all the internal migrations. The array order will be the order in which they are executed. + * + * { + * name: 'some-name', + * async up(knex: Knex, db: Database) {}, + * async down(knex: Knex, db: Database) {}, + * }, + */ +export const internalMigrations: Migration[] = []; diff --git a/packages/core/database/src/migrations/internal.ts b/packages/core/database/src/migrations/internal.ts new file mode 100644 index 0000000000..6a5379f047 --- /dev/null +++ b/packages/core/database/src/migrations/internal.ts @@ -0,0 +1,38 @@ +import { Umzug } from 'umzug'; + +import { wrapTransaction } from './common'; +import { internalMigrations } from './internal-migrations'; +import { createStorage } from './storage'; + +import type { MigrationProvider } from './common'; +import type { Database } from '..'; + +export const createInternalMigrationProvider = (db: Database): MigrationProvider => { + const context = { db }; + + const umzugProvider = new Umzug({ + storage: createStorage({ db, tableName: 'strapi_migrations_internal' }), + logger: console, + context, + migrations: internalMigrations.map((migration) => { + return { + name: migration.name, + up: wrapTransaction(context.db)(migration.up), + down: wrapTransaction(context.db)(migration.down), + }; + }), + }); + + return { + async shouldRun() { + const pendingMigrations = await umzugProvider.pending(); + return pendingMigrations.length > 0; + }, + async up() { + await umzugProvider.up(); + }, + async down() { + await umzugProvider.down(); + }, + }; +}; diff --git a/packages/core/database/src/migrations/storage.ts b/packages/core/database/src/migrations/storage.ts index c648a28583..ee57da79f1 100644 --- a/packages/core/database/src/migrations/storage.ts +++ b/packages/core/database/src/migrations/storage.ts @@ -2,11 +2,11 @@ import type { Database } from '..'; export interface Options { db: Database; - tableName?: string; + tableName: string; } export const createStorage = (opts: Options) => { - const { db, tableName = 'strapi_migrations' } = opts; + const { db, tableName } = opts; const hasMigrationTable = () => db.getSchemaConnection().hasTable(tableName); diff --git a/packages/core/database/src/migrations/users.ts b/packages/core/database/src/migrations/users.ts new file mode 100644 index 0000000000..36d60b4054 --- /dev/null +++ b/packages/core/database/src/migrations/users.ts @@ -0,0 +1,70 @@ +import path from 'node:path'; +import fse from 'fs-extra'; +import { Umzug } from 'umzug'; + +import { createStorage } from './storage'; +import { wrapTransaction } from './common'; +import type { MigrationProvider, MigrationResolver } from './common'; +import type { Database } from '..'; + +// TODO: check multiple commands in one sql statement +const migrationResolver: MigrationResolver = ({ name, path, context }) => { + const { db } = context; + + if (!path) { + throw new Error(`Migration ${name} has no path`); + } + + // if sql file run with knex raw + if (path.match(/\.sql$/)) { + const sql = fse.readFileSync(path, 'utf8'); + + return { + name, + up: wrapTransaction(db)((knex) => knex.raw(sql)), + async down() { + throw new Error('Down migration is not supported for sql files'); + }, + }; + } + + // NOTE: we can add some ts register if we want to handle ts migration files at some point + // eslint-disable-next-line @typescript-eslint/no-var-requires + const migration = require(path); + return { + name, + up: wrapTransaction(db)(migration.up), + down: wrapTransaction(db)(migration.down), + }; +}; + +export const createUserMigrationProvider = (db: Database): MigrationProvider => { + const dir = path.join(strapi.dirs.app.root, 'database/migrations'); + + fse.ensureDirSync(dir); + + const context = { db }; + + const umzugProvider = new Umzug({ + storage: createStorage({ db, tableName: 'strapi_migrations' }), + logger: console, + context, + migrations: { + glob: ['*.{js,sql}', { cwd: dir }], + resolve: migrationResolver, + }, + }); + + return { + async shouldRun() { + const pendingMigrations = await umzugProvider.pending(); + return pendingMigrations.length > 0 && db.config?.settings?.runMigrations === true; + }, + async up() { + await umzugProvider.up(); + }, + async down() { + await umzugProvider.down(); + }, + }; +}; diff --git a/packages/core/database/src/schema/diff.ts b/packages/core/database/src/schema/diff.ts index 2db9f7f4a1..0374e09ff6 100644 --- a/packages/core/database/src/schema/diff.ts +++ b/packages/core/database/src/schema/diff.ts @@ -16,7 +16,12 @@ import type { } from './types'; import type { Database } from '..'; -const RESERVED_TABLE_NAMES = ['strapi_migrations', 'strapi_database_schema']; +// TODO: get that list dynamically instead +const RESERVED_TABLE_NAMES = [ + 'strapi_migrations', + 'strapi_migrations_internal', + 'strapi_database_schema', +]; const statuses = { CHANGED: 'CHANGED',