handle compo, dz and media migration when disabling i18n on a field

packages/strapi-connector-bookshelf/lib/queries.js

@@ -89,12 +89,12 @@ module.exports = function createQueryBuilder({ model, strapi }) {
   /**
    * Count entries based on filters
    */
-  function count(params = {}) {
+  function count(params = {}, { transacting } = {}) {
     const filters = pickCountFilters(convertRestQueryParams(params));
 
     return model
       .query(buildQuery({ model, filters }))
-      .count()
+      .count({ transacting })
       .then(Number);
   }
 

packages/strapi-database/lib/database-manager.js

@@ -38,10 +38,10 @@ class DatabaseManager {
 
     validateModelSchemas({ strapi: this.strapi, manager: this });
 
-    await this.connectors.initialize();
-
     this.initializeModelsMap();
 
+    await this.connectors.initialize();
+
     return this;
   }
 
@@ -73,12 +73,7 @@ class DatabaseManager {
       throw new Error(`argument entity is required`);
     }
 
-    const normalizedName = entity.toLowerCase();
+    const model = this.getModel(entity, plugin);
-
-    // get by uid or name / plugin
-    const model = this.models.has(entity)
-      ? this.models.get(entity)
-      : this.getModel(normalizedName, plugin);
 
     if (!model) {
       throw new Error(`The model ${entity} can't be found.`);
@@ -101,11 +96,8 @@ class DatabaseManager {
     return query;
   }
 
-  getModel(name, plugin) {
+  getModelFromStrapi(name, plugin) {
     const key = _.toLower(name);
-
-    if (this.models.has(key)) return this.models.get(key);
-
     if (plugin === 'admin') {
       return _.get(strapi.admin, ['models', key]);
     }
@@ -117,6 +109,17 @@ class DatabaseManager {
     return _.get(strapi, ['models', key]) || _.get(strapi, ['components', key]);
   }
 
+  getModel(name, plugin) {
+    const key = _.toLower(name);
+
+    if (this.models.has(key)) {
+      const { modelName, plugin: pluginName } = this.models.get(key);
+      return this.getModelFromStrapi(modelName, pluginName);
+    } else {
+      return this.getModelFromStrapi(key, plugin);
+    }
+  }
+
   getModelByAssoc(assoc) {
     return this.getModel(assoc.collection || assoc.model, assoc.plugin);
   }

packages/strapi-database/lib/queries/paginated-queries.js

@@ -4,11 +4,10 @@ const _ = require('lodash');
 
 const createPaginatedQuery = ({ fetch, count }) => async (queryParams, ...args) => {
   const params = _.omit(queryParams, ['page', 'pageSize']);
-  const pagination = await getPaginationInfos(queryParams, count);
+  const pagination = await getPaginationInfos(queryParams, count, ...args);
 
   Object.assign(params, paginationToQueryParams(pagination));
-
+  const results = await fetch(params, undefined, ...args);
-  const results = await fetch(params, ...args);
 
   return { results, pagination };
 };
@@ -18,11 +17,10 @@ const createSearchPageQuery = ({ search, countSearch }) =>
 
 const createFindPageQuery = ({ find, count }) => createPaginatedQuery({ fetch: find, count });
 
-const getPaginationInfos = async (queryParams, count) => {
+const getPaginationInfos = async (queryParams, count, ...args) => {
   const { page, pageSize, ...params } = withDefaultPagination(queryParams);
 
-  const total = await count(params);
+  const total = await count(params, ...args);
-
   return {
     page,
     pageSize,

packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js

@@ -44,8 +44,8 @@ describe('i18n - migration utils', () => {
       ],
     ];
 
-    test.each(testData)('%j', (entriesToProcess, attributesToMigrate, expectedResult) => {
+    test.each(testData)('%j', (entriesToProcess, attrsToMigrate, expectedResult) => {
-      const result = getUpdatesInfo({ entriesToProcess, attributesToMigrate });
+      const result = getUpdatesInfo({ entriesToProcess, attrsToMigrate });
 
       expect(result).toEqual(expectedResult);
     });

packages/strapi-plugin-i18n/config/functions/migrations/field/index.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const { difference, intersection } = require('lodash/fp');
+const { difference, keys, intersection, isEmpty } = require('lodash/fp');
 const { getService } = require('../../../../utils');
 const migrateForMongoose = require('./migrate-for-mongoose');
 const migrateForBookshelf = require('./migrate-for-bookshelf');
@@ -16,16 +16,16 @@ const after = async ({ model, definition, previousDefinition, ORM }) => {
   const localizedAttributes = ctService.getLocalizedAttributes(definition);
   const prevLocalizedAttributes = ctService.getLocalizedAttributes(previousDefinition);
   const attributesDisabled = difference(prevLocalizedAttributes, localizedAttributes);
-  const attributesToMigrate = intersection(Object.keys(definition.attributes), attributesDisabled);
+  const attrsToMigrate = intersection(keys(definition.attributes), attributesDisabled);
 
-  if (attributesToMigrate.length === 0) {
+  if (isEmpty(attrsToMigrate)) {
     return;
   }
 
   if (model.orm === 'bookshelf') {
-    await migrateForBookshelf({ ORM, model, attributesToMigrate });
+    await migrateForBookshelf({ ORM, model, attrsToMigrate });
   } else if (model.orm === 'mongoose') {
-    await migrateForMongoose({ model, attributesToMigrate });
+    await migrateForMongoose({ model, attrsToMigrate });
   }
 };
 

packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js

@@ -1,14 +1,11 @@
 'use strict';
 
-const { singular } = require('pluralize');
+const { migrate } = require('./migrate');
-const { has, omit, pick, orderBy } = require('lodash/fp');
+const { areScalarAttrsOnly } = require('./utils');
-const { shouldBeProcessed, getUpdatesInfo } = require('./utils');
-
-const BATCH_SIZE = 1000;
 
 const TMP_TABLE_NAME = '__tmp__i18n_field_migration';
 
-const batchInsertInTmpTable = async (updatesInfo, trx) => {
+const batchInsertInTmpTable = async ({ updatesInfo }, { transacting: trx }) => {
   const tmpEntries = [];
   updatesInfo.forEach(({ entriesIdsToUpdate, attributesValues }) => {
     entriesIdsToUpdate.forEach(id => {
@@ -18,36 +15,26 @@ const batchInsertInTmpTable = async (updatesInfo, trx) => {
   await trx.batchInsert(TMP_TABLE_NAME, tmpEntries, 100);
 };
 
-const batchUpdate = async (updatesInfo, trx, model) => {
+const updateFromTmpTable = async ({ model, attrsToMigrate }, { transacting: trx }) => {
-  const promises = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) =>
-    trx
-      .from(model.collectionName)
-      .update(attributesValues)
-      .whereIn('id', entriesIdsToUpdate)
-  );
-  await Promise.all(promises);
-};
-
-const updateFromTmpTable = async ({ model, trx, attributesToMigrate }) => {
   const collectionName = model.collectionName;
   if (model.client === 'pg') {
-    const substitutes = attributesToMigrate.map(() => '?? = ??.??').join(',');
+    const substitutes = attrsToMigrate.map(() => '?? = ??.??').join(',');
     const bindings = [collectionName];
-    attributesToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr));
+    attrsToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr));
     bindings.push(TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME);
 
     await trx.raw(`UPDATE ?? SET ${substitutes} FROM ?? WHERE ??.id = ??.id;`, bindings);
   } else if (model.client === 'mysql') {
-    const substitutes = attributesToMigrate.map(() => '??.?? = ??.??').join(',');
+    const substitutes = attrsToMigrate.map(() => '??.?? = ??.??').join(',');
     const bindings = [collectionName, TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME];
-    attributesToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr));
+    attrsToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr));
 
     await trx.raw(`UPDATE ?? JOIN ?? ON ??.id = ??.id SET ${substitutes};`, bindings);
   }
 };
 
-const createTmpTable = async ({ ORM, attributesToMigrate, model }) => {
+const createTmpTable = async ({ ORM, attrsToMigrate, model }) => {
-  const columnsToCopy = ['id', ...attributesToMigrate];
+  const columnsToCopy = ['id', ...attrsToMigrate];
   await deleteTmpTable({ ORM });
   await ORM.knex.raw(`CREATE TABLE ?? AS ??`, [
     TMP_TABLE_NAME,
@@ -60,110 +47,21 @@ const createTmpTable = async ({ ORM, attributesToMigrate, model }) => {
 
 const deleteTmpTable = ({ ORM }) => ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME);
 
-const getSortedLocales = async trx => {
+const migrateForBookshelf = async ({ ORM, model, attrsToMigrate }) => {
-  let defaultLocale;
+  const onlyScalarAttrs = areScalarAttrsOnly({ model, attributes: attrsToMigrate });
-  try {
-    const defaultLocaleRow = await trx('core_store')
-      .select('value')
-      .where({ key: 'plugin_i18n_default_locale' })
-      .first();
-    defaultLocale = JSON.parse(defaultLocaleRow.value);
-  } catch (e) {
-    throw new Error("Could not migrate because the default locale doesn't exist");
-  }
 
-  let locales;
+  // optimize migration for pg and mysql when there are only scalar attributes to migrate
-  try {
+  if (onlyScalarAttrs && ['pg', 'mysql'].includes(model.client)) {
-    locales = await trx(strapi.plugins.i18n.models.locale.collectionName).select('code');
-  } catch (e) {
-    throw new Error('Could not migrate because no locale exist');
-  }
-
-  locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale));
-  return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first
-};
-
-const processEntriesWith = async (processFn, { trx, model, attributesToMigrate }) => {
-  const locales = await getSortedLocales(trx);
-  const localizationAssoc = model.associations.find(a => a.alias === 'localizations');
-  const localizationTableName = localizationAssoc.tableCollectionName;
-
-  const locsAttr = model.attributes.localizations;
-  const foreignKey = `${singular(model.collectionName)}_${model.primaryKey}`;
-  const relatedKey = `${locsAttr.attribute}_${locsAttr.column}`;
-  const processedLocaleCodes = [];
-  for (const locale of locales) {
-    let offset = 0;
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      let batch = await trx
-        .select([
-          ...attributesToMigrate.map(attr => `model.${attr}`),
-          'model.id as __strapi_rootId',
-          'relModel.id as id',
-          'relModel.locale',
-        ])
-        .join(
-          `${localizationTableName} as loc`,
-          `model.${model.primaryKey}`,
-          '=',
-          `loc.${foreignKey}`
-        )
-        .join(
-          `${model.collectionName} as relModel`,
-          `loc.${relatedKey}`,
-          '=',
-          `relModel.${model.primaryKey}`
-        )
-        .from(
-          trx
-            .select('*')
-            .from(`${model.collectionName} as subModel`)
-            .orderBy(`subModel.${model.primaryKey}`)
-            .where(`subModel.locale`, locale.code)
-            .offset(offset)
-            .limit(BATCH_SIZE)
-            .as('model')
-        );
-      let entries = batch.reduce((entries, entry) => {
-        if (has(entry.__strapi_rootId, entries)) {
-          entries[entry.__strapi_rootId].localizations.push(pick(['id', 'locale'], entry));
-        } else {
-          entries[entry.__strapi_rootId] = omit(['id', 'locale', '__strapi_rootId'], entry);
-          entries[entry.__strapi_rootId].localizations = [pick(['id', 'locale'], entry)];
-        }
-
-        return entries;
-      }, {});
-      entries = Object.values(entries);
-
-      offset += BATCH_SIZE;
-
-      const entriesToProcess = entries.filter(shouldBeProcessed(processedLocaleCodes));
-      const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate });
-
-      await processFn(updatesInfo, trx, model);
-
-      if (entries.length < BATCH_SIZE) {
-        break;
-      }
-    }
-    processedLocaleCodes.push(locale.code);
-  }
-};
-
-const migrateForBookshelf = async ({ ORM, model, attributesToMigrate }) => {
-  if (['pg', 'mysql'].includes(model.client)) {
     // create table outside of the transaction because mysql doesn't accept the creation inside
-    await createTmpTable({ ORM, attributesToMigrate, model });
+    await createTmpTable({ ORM, attrsToMigrate, model });
-    await ORM.knex.transaction(async trx => {
+    await ORM.knex.transaction(async transacting => {
-      await processEntriesWith(batchInsertInTmpTable, { ORM, trx, model, attributesToMigrate });
+      await migrate({ model, attrsToMigrate }, { migrateFn: batchInsertInTmpTable, transacting });
-      await updateFromTmpTable({ model, trx, attributesToMigrate });
+      await updateFromTmpTable({ model, attrsToMigrate }, { transacting });
     });
     await deleteTmpTable({ ORM });
   } else {
-    await ORM.knex.transaction(async trx => {
+    await ORM.knex.transaction(async transacting => {
-      await processEntriesWith(batchUpdate, { ORM, trx, model, attributesToMigrate });
+      await migrate({ model, attrsToMigrate }, { transacting });
     });
   }
 };

packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js

@@ -1,68 +1,23 @@
 'use strict';
 
-const { orderBy } = require('lodash/fp');
+const { migrate } = require('./migrate');
-const { shouldBeProcessed, getUpdatesInfo } = require('./utils');
+const { areScalarAttrsOnly } = require('./utils');
 
-const BATCH_SIZE = 1000;
+const batchUpdate = async ({ updatesInfo, model }) => {
-
-const getSortedLocales = async () => {
-  let defaultLocale;
-  try {
-    const defaultLocaleRow = await strapi.models['core_store'].findOne({
-      key: 'plugin_i18n_default_locale',
-    });
-    defaultLocale = JSON.parse(defaultLocaleRow.value);
-  } catch (e) {
-    throw new Error("Could not migrate because the default locale doesn't exist");
-  }
-
-  let locales;
-  try {
-    strapi.models;
-    locales = await strapi.plugins.i18n.models.locale.find();
-  } catch (e) {
-    throw new Error('Could not migrate because no locale exist');
-  }
-
-  locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale));
-  return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first
-};
-
-const migrateForMongoose = async ({ model, attributesToMigrate }) => {
-  const locales = await getSortedLocales();
-  const processedLocaleCodes = [];
-  for (const locale of locales) {
-    let lastId;
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      const findParams = { locale: locale.code };
-      if (lastId) {
-        findParams._id = { $gt: lastId };
-      }
-
-      const batch = await model
-        .find(findParams, [...attributesToMigrate, 'locale', 'localizations'])
-        .populate('localizations', 'locale id')
-        .sort({ _id: 1 })
-        .limit(BATCH_SIZE);
-
-      if (batch.length > 0) {
-        lastId = batch[batch.length - 1]._id;
-      }
-      const entriesToProcess = batch.filter(shouldBeProcessed(processedLocaleCodes));
-
-      const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate });
   const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({
     updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues },
   }));
 
   await model.bulkWrite(updates);
+};
 
-      if (batch.length < BATCH_SIZE) {
+const migrateForMongoose = async ({ model, attrsToMigrate }) => {
-        break;
+  const onlyScalarAttrs = areScalarAttrsOnly({ model, attributes: attrsToMigrate });
-      }
+
-    }
+  if (onlyScalarAttrs) {
-    processedLocaleCodes.push(locale.code);
+    await migrate({ model, attrsToMigrate }, { migrateFn: batchUpdate });
+  } else {
+    await migrate({ model, attrsToMigrate });
   }
 };
 

packages/strapi-plugin-i18n/config/functions/migrations/field/migrate.js

@@ -0,0 +1,56 @@
+'use strict';
+
+const { pick, prop } = require('lodash/fp');
+const { getService } = require('../../../../utils');
+const { shouldBeProcessed, getUpdatesInfo, getSortedLocales } = require('./utils');
+
+const BATCH_SIZE = 1000;
+
+const migrateBatch = async (entries, { model, attrsToMigrate }, { transacting }) => {
+  const { copyNonLocalizedAttributes } = getService('content-types');
+
+  const updatePromises = entries.map(entity => {
+    const updateValues = pick(attrsToMigrate, copyNonLocalizedAttributes(model, entity));
+    const entriesIdsToUpdate = entity.localizations.map(prop('id'));
+    console.log('updateValues', JSON.stringify(updateValues, null, 2));
+    return Promise.all(
+      entriesIdsToUpdate.map(id =>
+        strapi.query(model.uid).update({ id }, updateValues, { transacting })
+      )
+    );
+  });
+
+  await Promise.all(updatePromises);
+};
+
+const migrate = async ({ model, attrsToMigrate }, { migrateFn, transacting } = {}) => {
+  const locales = await getSortedLocales({ transacting });
+  const processedLocaleCodes = [];
+  for (const locale of locales) {
+    let page = 1;
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const { results } = await strapi
+        .query(model.uid)
+        .findPage({ locale, page, pageSize: BATCH_SIZE }, { transacting });
+      const entriesToProcess = results.filter(shouldBeProcessed(processedLocaleCodes));
+
+      if (migrateFn) {
+        const updatesInfo = getUpdatesInfo({ entriesToProcess, attrsToMigrate });
+        await migrateFn({ updatesInfo, model }, { transacting });
+      } else {
+        await migrateBatch(entriesToProcess, { model, attrsToMigrate }, { transacting });
+      }
+
+      if (results.length < BATCH_SIZE) {
+        break;
+      }
+      page += 1;
+    }
+    processedLocaleCodes.push(locale);
+  }
+};
+
+module.exports = {
+  migrate,
+};

packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js

@@ -1,6 +1,8 @@
 'use strict';
 
-const { pick, prop, intersection } = require('lodash/fp');
+const { isScalarAttribute } = require('strapi-utils').contentTypes;
+const { pick, prop, map, intersection, isEmpty, orderBy, pipe, every } = require('lodash/fp');
+const { getService } = require('../../../../utils');
 
 const shouldBeProcessed = processedLocaleCodes => entry => {
   return (
@@ -9,17 +11,48 @@ const shouldBeProcessed = processedLocaleCodes => entry => {
   );
 };
 
-const getUpdatesInfo = ({ entriesToProcess, attributesToMigrate }) => {
+const getUpdatesInfo = ({ entriesToProcess, attrsToMigrate }) => {
   const updates = [];
   for (const entry of entriesToProcess) {
-    const attributesValues = pick(attributesToMigrate, entry);
+    const attributesValues = pick(attrsToMigrate, entry);
     const entriesIdsToUpdate = entry.localizations.map(prop('id'));
     updates.push({ entriesIdsToUpdate, attributesValues });
   }
   return updates;
 };
 
+const getSortedLocales = async ({ transacting } = {}) => {
+  const localeService = getService('locales');
+
+  let defaultLocale;
+  try {
+    const storeRes = await strapi
+      .query('core_store')
+      .findOne({ key: 'plugin_i18n_default_locale' }, null, { transacting });
+    defaultLocale = JSON.parse(storeRes.value);
+  } catch (e) {
+    throw new Error("Could not migrate because the default locale doesn't exist");
+  }
+
+  const locales = await localeService.find({}, null, { transacting });
+  if (isEmpty(locales)) {
+    throw new Error('Could not migrate because no locale exist');
+  }
+
+  // Put default locale first
+  return pipe(
+    map(locale => ({ code: locale.code, isDefault: locale.code === defaultLocale })),
+    orderBy(['isDefault', 'code'], ['desc', 'asc']),
+    map(prop('code'))
+  )(locales);
+};
+
+const areScalarAttrsOnly = ({ model, attributes }) =>
+  pipe(pick(attributes), every(isScalarAttribute))(model.attributes);
+
 module.exports = {
   shouldBeProcessed,
   getUpdatesInfo,
+  getSortedLocales,
+  areScalarAttrsOnly,
 };

packages/strapi-plugin-i18n/services/content-types.js

@@ -101,8 +101,9 @@ const getNonLocalizedAttributes = model => {
 };
 
 const removeId = value => {
-  if (typeof value === 'object' && has('id', value)) {
+  if (typeof value === 'object' && (has('id', value) || has('_id', value))) {
     delete value.id;
+    delete value._id;
   }
 };