chore: add document-id migration (#19514)

* chore: add document-id migration

* fix: use database identifiers utils

* fix: leave rename identifiers migration name as it was before

* chore: revert database config

* feat: simplify migration document ids query

* update with subquery

---------

Co-authored-by: Marc-Roig <marc12info@gmail.com>
Co-authored-by: Marc Roig <marc.roig.campos@strapi.io>

packages/core/database/package.json

@@ -39,6 +39,7 @@
     "watch": "pack-up watch"
   },
   "dependencies": {
+    "@paralleldrive/cuid2": "2.2.2",
     "@strapi/utils": "5.0.0-beta.1",
     "date-fns": "2.30.0",
     "debug": "4.3.4",

packages/core/database/src/migrations/internal-migrations/5.0.0-02-document-id.ts

@@ -0,0 +1,178 @@
+import { createId } from '@paralleldrive/cuid2';
+import type { Knex } from 'knex';
+
+import type { Migration } from '../common';
+import type { Database } from '../..';
+import type { Meta } from '../../metadata';
+import { identifiers } from '../../utils/identifiers';
+
+interface Params {
+  joinColumn: string;
+  inverseJoinColumn: string;
+  tableName: string;
+  joinTableName: string;
+}
+
+const QUERIES = {
+  async postgres(knex: Knex, params: Params) {
+    const res = await knex.raw(
+      `
+    SELECT :tableName:.id as id, string_agg(DISTINCT :inverseJoinColumn:::character varying, ',') as other_ids
+    FROM :tableName:
+    LEFT JOIN :joinTableName: ON :tableName:.id = :joinTableName:.:joinColumn:
+    WHERE document_id IS NULL
+    GROUP BY :tableName:.id, :joinColumn:
+    LIMIT 1;
+  `,
+      params
+    );
+
+    return res.rows;
+  },
+  async mysql(knex: Knex, params: Params) {
+    const [res] = await knex.raw(
+      `
+    SELECT :tableName:.id as id, group_concat(DISTINCT :inverseJoinColumn:) as other_ids
+    FROM :tableName:
+    LEFT JOIN :joinTableName: ON :tableName:.id = :joinTableName:.:joinColumn:
+    WHERE document_id IS NULL
+    GROUP BY :tableName:.id, :joinColumn:
+    LIMIT 1;
+  `,
+      params
+    );
+
+    return res;
+  },
+  async sqlite(knex: Knex, params: Params) {
+    return knex.raw(
+      `
+    SELECT :tableName:.id as id, group_concat(DISTINCT :inverseJoinColumn:) as other_ids
+    FROM :tableName:
+    LEFT JOIN :joinTableName: ON :tableName:.id = :joinTableName:.:joinColumn:
+    WHERE document_id IS NULL
+    GROUP BY :joinColumn:
+    LIMIT 1;
+    `,
+      params
+    );
+  },
+};
+
+const getNextIdsToCreateDocumentId = async (
+  db: Database,
+  knex: Knex,
+  {
+    joinColumn,
+    inverseJoinColumn,
+    tableName,
+    joinTableName,
+  }: {
+    joinColumn: string;
+    inverseJoinColumn: string;
+    tableName: string;
+    joinTableName: string;
+  }
+): Promise<number[]> => {
+  const res = await QUERIES[db.dialect.client as keyof typeof QUERIES](knex, {
+    joinColumn,
+    inverseJoinColumn,
+    tableName,
+    joinTableName,
+  });
+
+  if (res.length > 0) {
+    const row = res[0];
+    const otherIds = row.other_ids
+      ? row.other_ids.split(',').map((v: string) => parseInt(v, 10))
+      : [];
+
+    return [row.id, ...otherIds];
+  }
+
+  return [];
+};
+
+// Migrate document ids for tables that have localizations
+const migrateDocumentIdsWithLocalizations = async (db: Database, knex: Knex, meta: Meta) => {
+  const singularName = meta.singularName.toLowerCase();
+  const joinColumn = identifiers.getJoinColumnAttributeIdName(singularName);
+  const inverseJoinColumn = identifiers.getInverseJoinColumnAttributeIdName(singularName);
+
+  let ids: number[];
+
+  do {
+    ids = await getNextIdsToCreateDocumentId(db, knex, {
+      joinColumn,
+      inverseJoinColumn,
+      tableName: meta.tableName,
+      joinTableName: identifiers.getJoinTableName(meta.tableName, `localizations`),
+    });
+
+    if (ids.length > 0) {
+      await knex(meta.tableName).update({ document_id: createId() }).whereIn('id', ids);
+    }
+  } while (ids.length > 0);
+};
+
+// Migrate document ids for tables that don't have localizations
+const migrationDocumentIds = async (db: Database, knex: Knex, meta: Meta) => {
+  let run = true;
+
+  do {
+    const updatedRows = await knex(meta.tableName)
+      .update({ document_id: createId() })
+      .whereIn('id', (builder) => {
+        return builder.whereNull('document_id').select('id').limit(1);
+      });
+
+    if (updatedRows <= 0) {
+      run = false;
+    }
+  } while (run);
+};
+
+const createDocumentIdColumn = async (knex: Knex, tableName: string) => {
+  await knex.schema.alterTable(tableName, (table) => {
+    table.string('document_id');
+  });
+};
+
+const hasLocalizationsJoinTable = async (knex: Knex, tableName: string) => {
+  const joinTableName = identifiers.getJoinTableName(tableName, 'localizations');
+  return knex.schema.hasTable(joinTableName);
+};
+
+export const createdDocumentId: Migration = {
+  name: '5.0.0-02-created-document-id',
+  async up(knex, db) {
+    // do sth
+    for (const meta of db.metadata.values()) {
+      const hasTable = await knex.schema.hasTable(meta.tableName);
+
+      if (!hasTable) {
+        continue;
+      }
+
+      if ('documentId' in meta.attributes) {
+        // add column if doesn't exist
+        const hasDocumentIdColumn = await knex.schema.hasColumn(meta.tableName, 'document_id');
+
+        if (hasDocumentIdColumn) {
+          continue;
+        }
+
+        await createDocumentIdColumn(knex, meta.tableName);
+
+        if (await hasLocalizationsJoinTable(knex, meta.tableName)) {
+          await migrateDocumentIdsWithLocalizations(db, knex, meta);
+        } else {
+          await migrationDocumentIds(db, knex, meta);
+        }
+      }
+    }
+  },
+  async down() {
+    throw new Error('not implemented');
+  },
+};

packages/core/database/src/migrations/internal-migrations/index.ts

@@ -1,5 +1,6 @@
 import type { Migration } from '../common';
-import { renameIdentifiersLongerThanMaxLength } from './5.0.0-convert-identifiers-long-than-max-length';
+import { createdDocumentId } from './5.0.0-02-document-id';
+import { renameIdentifiersLongerThanMaxLength } from './5.0.0-01-convert-identifiers-long-than-max-length';
 
 /**
  * List of all the internal migrations. The array order will be the order in which they are executed.
@@ -10,4 +11,7 @@ import { renameIdentifiersLongerThanMaxLength } from './5.0.0-convert-identifier
  *   async down(knex: Knex, db: Database) {},
  * },
  */
-export const internalMigrations: Migration[] = [renameIdentifiersLongerThanMaxLength];
+export const internalMigrations: Migration[] = [
+  renameIdentifiersLongerThanMaxLength,
+  createdDocumentId,
+];

yarn.lock

@@ -7299,6 +7299,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "@strapi/database@workspace:packages/core/database"
   dependencies:
+    "@paralleldrive/cuid2": "npm:2.2.2"
     "@strapi/pack-up": "npm:5.0.0-beta.1"
     "@strapi/utils": "npm:5.0.0-beta.1"
     date-fns: "npm:2.30.0"