From c086bcc9d71109302f5dbb1dafd1762d7c0d66ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Thu, 8 Apr 2021 18:05:12 +0200 Subject: [PATCH] handle compo, dz and media migration when disabling i18n on a field --- .../strapi-connector-bookshelf/lib/queries.js | 4 +- .../strapi-database/lib/database-manager.js | 27 ++-- .../lib/queries/paginated-queries.js | 10 +- .../migrations/field/__tests__/utils.test.js | 4 +- .../functions/migrations/field/index.js | 10 +- .../migrations/field/migrate-for-bookshelf.js | 142 +++--------------- .../migrations/field/migrate-for-mongoose.js | 71 ++------- .../functions/migrations/field/migrate.js | 56 +++++++ .../functions/migrations/field/utils.js | 39 ++++- .../services/content-types.js | 3 +- 10 files changed, 155 insertions(+), 211 deletions(-) create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/field/migrate.js diff --git a/packages/strapi-connector-bookshelf/lib/queries.js b/packages/strapi-connector-bookshelf/lib/queries.js index 907c8c06bd..c6469cf0fa 100644 --- a/packages/strapi-connector-bookshelf/lib/queries.js +++ b/packages/strapi-connector-bookshelf/lib/queries.js @@ -89,12 +89,12 @@ module.exports = function createQueryBuilder({ model, strapi }) { /** * Count entries based on filters */ - function count(params = {}) { + function count(params = {}, { transacting } = {}) { const filters = pickCountFilters(convertRestQueryParams(params)); return model .query(buildQuery({ model, filters })) - .count() + .count({ transacting }) .then(Number); } diff --git a/packages/strapi-database/lib/database-manager.js b/packages/strapi-database/lib/database-manager.js index f610c22374..fccc52ee27 100644 --- a/packages/strapi-database/lib/database-manager.js +++ b/packages/strapi-database/lib/database-manager.js @@ -38,10 +38,10 @@ class DatabaseManager { validateModelSchemas({ strapi: this.strapi, manager: this }); - await this.connectors.initialize(); - this.initializeModelsMap(); + await this.connectors.initialize(); + return this; } @@ -73,12 +73,7 @@ class DatabaseManager { throw new Error(`argument entity is required`); } - const normalizedName = entity.toLowerCase(); - - // get by uid or name / plugin - const model = this.models.has(entity) - ? this.models.get(entity) - : this.getModel(normalizedName, plugin); + const model = this.getModel(entity, plugin); if (!model) { throw new Error(`The model ${entity} can't be found.`); @@ -101,11 +96,8 @@ class DatabaseManager { return query; } - getModel(name, plugin) { + getModelFromStrapi(name, plugin) { const key = _.toLower(name); - - if (this.models.has(key)) return this.models.get(key); - if (plugin === 'admin') { return _.get(strapi.admin, ['models', key]); } @@ -117,6 +109,17 @@ class DatabaseManager { return _.get(strapi, ['models', key]) || _.get(strapi, ['components', key]); } + getModel(name, plugin) { + const key = _.toLower(name); + + if (this.models.has(key)) { + const { modelName, plugin: pluginName } = this.models.get(key); + return this.getModelFromStrapi(modelName, pluginName); + } else { + return this.getModelFromStrapi(key, plugin); + } + } + getModelByAssoc(assoc) { return this.getModel(assoc.collection || assoc.model, assoc.plugin); } diff --git a/packages/strapi-database/lib/queries/paginated-queries.js b/packages/strapi-database/lib/queries/paginated-queries.js index eff74597e9..99b8b94cb6 100644 --- a/packages/strapi-database/lib/queries/paginated-queries.js +++ b/packages/strapi-database/lib/queries/paginated-queries.js @@ -4,11 +4,10 @@ const _ = require('lodash'); const createPaginatedQuery = ({ fetch, count }) => async (queryParams, ...args) => { const params = _.omit(queryParams, ['page', 'pageSize']); - const pagination = await getPaginationInfos(queryParams, count); + const pagination = await getPaginationInfos(queryParams, count, ...args); Object.assign(params, paginationToQueryParams(pagination)); - - const results = await fetch(params, ...args); + const results = await fetch(params, undefined, ...args); return { results, pagination }; }; @@ -18,11 +17,10 @@ const createSearchPageQuery = ({ search, countSearch }) => const createFindPageQuery = ({ find, count }) => createPaginatedQuery({ fetch: find, count }); -const getPaginationInfos = async (queryParams, count) => { +const getPaginationInfos = async (queryParams, count, ...args) => { const { page, pageSize, ...params } = withDefaultPagination(queryParams); - const total = await count(params); - + const total = await count(params, ...args); return { page, pageSize, diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js index 5af244f3f8..f05b54a2ec 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js @@ -44,8 +44,8 @@ describe('i18n - migration utils', () => { ], ]; - test.each(testData)('%j', (entriesToProcess, attributesToMigrate, expectedResult) => { - const result = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); + test.each(testData)('%j', (entriesToProcess, attrsToMigrate, expectedResult) => { + const result = getUpdatesInfo({ entriesToProcess, attrsToMigrate }); expect(result).toEqual(expectedResult); }); diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js index 87a0bbed56..d62f8e80a3 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js @@ -1,6 +1,6 @@ 'use strict'; -const { difference, intersection } = require('lodash/fp'); +const { difference, keys, intersection, isEmpty } = require('lodash/fp'); const { getService } = require('../../../../utils'); const migrateForMongoose = require('./migrate-for-mongoose'); const migrateForBookshelf = require('./migrate-for-bookshelf'); @@ -16,16 +16,16 @@ const after = async ({ model, definition, previousDefinition, ORM }) => { const localizedAttributes = ctService.getLocalizedAttributes(definition); const prevLocalizedAttributes = ctService.getLocalizedAttributes(previousDefinition); const attributesDisabled = difference(prevLocalizedAttributes, localizedAttributes); - const attributesToMigrate = intersection(Object.keys(definition.attributes), attributesDisabled); + const attrsToMigrate = intersection(keys(definition.attributes), attributesDisabled); - if (attributesToMigrate.length === 0) { + if (isEmpty(attrsToMigrate)) { return; } if (model.orm === 'bookshelf') { - await migrateForBookshelf({ ORM, model, attributesToMigrate }); + await migrateForBookshelf({ ORM, model, attrsToMigrate }); } else if (model.orm === 'mongoose') { - await migrateForMongoose({ model, attributesToMigrate }); + await migrateForMongoose({ model, attrsToMigrate }); } }; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js index 13f403c407..f51fe314cb 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js @@ -1,14 +1,11 @@ 'use strict'; -const { singular } = require('pluralize'); -const { has, omit, pick, orderBy } = require('lodash/fp'); -const { shouldBeProcessed, getUpdatesInfo } = require('./utils'); - -const BATCH_SIZE = 1000; +const { migrate } = require('./migrate'); +const { areScalarAttrsOnly } = require('./utils'); const TMP_TABLE_NAME = '__tmp__i18n_field_migration'; -const batchInsertInTmpTable = async (updatesInfo, trx) => { +const batchInsertInTmpTable = async ({ updatesInfo }, { transacting: trx }) => { const tmpEntries = []; updatesInfo.forEach(({ entriesIdsToUpdate, attributesValues }) => { entriesIdsToUpdate.forEach(id => { @@ -18,36 +15,26 @@ const batchInsertInTmpTable = async (updatesInfo, trx) => { await trx.batchInsert(TMP_TABLE_NAME, tmpEntries, 100); }; -const batchUpdate = async (updatesInfo, trx, model) => { - const promises = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => - trx - .from(model.collectionName) - .update(attributesValues) - .whereIn('id', entriesIdsToUpdate) - ); - await Promise.all(promises); -}; - -const updateFromTmpTable = async ({ model, trx, attributesToMigrate }) => { +const updateFromTmpTable = async ({ model, attrsToMigrate }, { transacting: trx }) => { const collectionName = model.collectionName; if (model.client === 'pg') { - const substitutes = attributesToMigrate.map(() => '?? = ??.??').join(','); + const substitutes = attrsToMigrate.map(() => '?? = ??.??').join(','); const bindings = [collectionName]; - attributesToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr)); + attrsToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr)); bindings.push(TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME); await trx.raw(`UPDATE ?? SET ${substitutes} FROM ?? WHERE ??.id = ??.id;`, bindings); } else if (model.client === 'mysql') { - const substitutes = attributesToMigrate.map(() => '??.?? = ??.??').join(','); + const substitutes = attrsToMigrate.map(() => '??.?? = ??.??').join(','); const bindings = [collectionName, TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME]; - attributesToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr)); + attrsToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr)); await trx.raw(`UPDATE ?? JOIN ?? ON ??.id = ??.id SET ${substitutes};`, bindings); } }; -const createTmpTable = async ({ ORM, attributesToMigrate, model }) => { - const columnsToCopy = ['id', ...attributesToMigrate]; +const createTmpTable = async ({ ORM, attrsToMigrate, model }) => { + const columnsToCopy = ['id', ...attrsToMigrate]; await deleteTmpTable({ ORM }); await ORM.knex.raw(`CREATE TABLE ?? AS ??`, [ TMP_TABLE_NAME, @@ -60,110 +47,21 @@ const createTmpTable = async ({ ORM, attributesToMigrate, model }) => { const deleteTmpTable = ({ ORM }) => ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); -const getSortedLocales = async trx => { - let defaultLocale; - try { - const defaultLocaleRow = await trx('core_store') - .select('value') - .where({ key: 'plugin_i18n_default_locale' }) - .first(); - defaultLocale = JSON.parse(defaultLocaleRow.value); - } catch (e) { - throw new Error("Could not migrate because the default locale doesn't exist"); - } +const migrateForBookshelf = async ({ ORM, model, attrsToMigrate }) => { + const onlyScalarAttrs = areScalarAttrsOnly({ model, attributes: attrsToMigrate }); - let locales; - try { - locales = await trx(strapi.plugins.i18n.models.locale.collectionName).select('code'); - } catch (e) { - throw new Error('Could not migrate because no locale exist'); - } - - locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale)); - return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first -}; - -const processEntriesWith = async (processFn, { trx, model, attributesToMigrate }) => { - const locales = await getSortedLocales(trx); - const localizationAssoc = model.associations.find(a => a.alias === 'localizations'); - const localizationTableName = localizationAssoc.tableCollectionName; - - const locsAttr = model.attributes.localizations; - const foreignKey = `${singular(model.collectionName)}_${model.primaryKey}`; - const relatedKey = `${locsAttr.attribute}_${locsAttr.column}`; - const processedLocaleCodes = []; - for (const locale of locales) { - let offset = 0; - // eslint-disable-next-line no-constant-condition - while (true) { - let batch = await trx - .select([ - ...attributesToMigrate.map(attr => `model.${attr}`), - 'model.id as __strapi_rootId', - 'relModel.id as id', - 'relModel.locale', - ]) - .join( - `${localizationTableName} as loc`, - `model.${model.primaryKey}`, - '=', - `loc.${foreignKey}` - ) - .join( - `${model.collectionName} as relModel`, - `loc.${relatedKey}`, - '=', - `relModel.${model.primaryKey}` - ) - .from( - trx - .select('*') - .from(`${model.collectionName} as subModel`) - .orderBy(`subModel.${model.primaryKey}`) - .where(`subModel.locale`, locale.code) - .offset(offset) - .limit(BATCH_SIZE) - .as('model') - ); - let entries = batch.reduce((entries, entry) => { - if (has(entry.__strapi_rootId, entries)) { - entries[entry.__strapi_rootId].localizations.push(pick(['id', 'locale'], entry)); - } else { - entries[entry.__strapi_rootId] = omit(['id', 'locale', '__strapi_rootId'], entry); - entries[entry.__strapi_rootId].localizations = [pick(['id', 'locale'], entry)]; - } - - return entries; - }, {}); - entries = Object.values(entries); - - offset += BATCH_SIZE; - - const entriesToProcess = entries.filter(shouldBeProcessed(processedLocaleCodes)); - const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); - - await processFn(updatesInfo, trx, model); - - if (entries.length < BATCH_SIZE) { - break; - } - } - processedLocaleCodes.push(locale.code); - } -}; - -const migrateForBookshelf = async ({ ORM, model, attributesToMigrate }) => { - if (['pg', 'mysql'].includes(model.client)) { + // optimize migration for pg and mysql when there are only scalar attributes to migrate + if (onlyScalarAttrs && ['pg', 'mysql'].includes(model.client)) { // create table outside of the transaction because mysql doesn't accept the creation inside - await createTmpTable({ ORM, attributesToMigrate, model }); - await ORM.knex.transaction(async trx => { - await processEntriesWith(batchInsertInTmpTable, { ORM, trx, model, attributesToMigrate }); - await updateFromTmpTable({ model, trx, attributesToMigrate }); + await createTmpTable({ ORM, attrsToMigrate, model }); + await ORM.knex.transaction(async transacting => { + await migrate({ model, attrsToMigrate }, { migrateFn: batchInsertInTmpTable, transacting }); + await updateFromTmpTable({ model, attrsToMigrate }, { transacting }); }); await deleteTmpTable({ ORM }); } else { - await ORM.knex.transaction(async trx => { - await processEntriesWith(batchUpdate, { ORM, trx, model, attributesToMigrate }); + await ORM.knex.transaction(async transacting => { + await migrate({ model, attrsToMigrate }, { transacting }); }); } }; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js index 450bee2fe1..38d5aa5593 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js @@ -1,68 +1,23 @@ 'use strict'; -const { orderBy } = require('lodash/fp'); -const { shouldBeProcessed, getUpdatesInfo } = require('./utils'); +const { migrate } = require('./migrate'); +const { areScalarAttrsOnly } = require('./utils'); -const BATCH_SIZE = 1000; +const batchUpdate = async ({ updatesInfo, model }) => { + const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({ + updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, + })); -const getSortedLocales = async () => { - let defaultLocale; - try { - const defaultLocaleRow = await strapi.models['core_store'].findOne({ - key: 'plugin_i18n_default_locale', - }); - defaultLocale = JSON.parse(defaultLocaleRow.value); - } catch (e) { - throw new Error("Could not migrate because the default locale doesn't exist"); - } - - let locales; - try { - strapi.models; - locales = await strapi.plugins.i18n.models.locale.find(); - } catch (e) { - throw new Error('Could not migrate because no locale exist'); - } - - locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale)); - return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first + await model.bulkWrite(updates); }; -const migrateForMongoose = async ({ model, attributesToMigrate }) => { - const locales = await getSortedLocales(); - const processedLocaleCodes = []; - for (const locale of locales) { - let lastId; - // eslint-disable-next-line no-constant-condition - while (true) { - const findParams = { locale: locale.code }; - if (lastId) { - findParams._id = { $gt: lastId }; - } +const migrateForMongoose = async ({ model, attrsToMigrate }) => { + const onlyScalarAttrs = areScalarAttrsOnly({ model, attributes: attrsToMigrate }); - const batch = await model - .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) - .populate('localizations', 'locale id') - .sort({ _id: 1 }) - .limit(BATCH_SIZE); - - if (batch.length > 0) { - lastId = batch[batch.length - 1]._id; - } - const entriesToProcess = batch.filter(shouldBeProcessed(processedLocaleCodes)); - - const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); - const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({ - updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, - })); - - await model.bulkWrite(updates); - - if (batch.length < BATCH_SIZE) { - break; - } - } - processedLocaleCodes.push(locale.code); + if (onlyScalarAttrs) { + await migrate({ model, attrsToMigrate }, { migrateFn: batchUpdate }); + } else { + await migrate({ model, attrsToMigrate }); } }; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate.js new file mode 100644 index 0000000000..ef7adbbf25 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate.js @@ -0,0 +1,56 @@ +'use strict'; + +const { pick, prop } = require('lodash/fp'); +const { getService } = require('../../../../utils'); +const { shouldBeProcessed, getUpdatesInfo, getSortedLocales } = require('./utils'); + +const BATCH_SIZE = 1000; + +const migrateBatch = async (entries, { model, attrsToMigrate }, { transacting }) => { + const { copyNonLocalizedAttributes } = getService('content-types'); + + const updatePromises = entries.map(entity => { + const updateValues = pick(attrsToMigrate, copyNonLocalizedAttributes(model, entity)); + const entriesIdsToUpdate = entity.localizations.map(prop('id')); + console.log('updateValues', JSON.stringify(updateValues, null, 2)); + return Promise.all( + entriesIdsToUpdate.map(id => + strapi.query(model.uid).update({ id }, updateValues, { transacting }) + ) + ); + }); + + await Promise.all(updatePromises); +}; + +const migrate = async ({ model, attrsToMigrate }, { migrateFn, transacting } = {}) => { + const locales = await getSortedLocales({ transacting }); + const processedLocaleCodes = []; + for (const locale of locales) { + let page = 1; + // eslint-disable-next-line no-constant-condition + while (true) { + const { results } = await strapi + .query(model.uid) + .findPage({ locale, page, pageSize: BATCH_SIZE }, { transacting }); + const entriesToProcess = results.filter(shouldBeProcessed(processedLocaleCodes)); + + if (migrateFn) { + const updatesInfo = getUpdatesInfo({ entriesToProcess, attrsToMigrate }); + await migrateFn({ updatesInfo, model }, { transacting }); + } else { + await migrateBatch(entriesToProcess, { model, attrsToMigrate }, { transacting }); + } + + if (results.length < BATCH_SIZE) { + break; + } + page += 1; + } + processedLocaleCodes.push(locale); + } +}; + +module.exports = { + migrate, +}; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js index 39fb9baf41..8bbf676263 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js @@ -1,6 +1,8 @@ 'use strict'; -const { pick, prop, intersection } = require('lodash/fp'); +const { isScalarAttribute } = require('strapi-utils').contentTypes; +const { pick, prop, map, intersection, isEmpty, orderBy, pipe, every } = require('lodash/fp'); +const { getService } = require('../../../../utils'); const shouldBeProcessed = processedLocaleCodes => entry => { return ( @@ -9,17 +11,48 @@ const shouldBeProcessed = processedLocaleCodes => entry => { ); }; -const getUpdatesInfo = ({ entriesToProcess, attributesToMigrate }) => { +const getUpdatesInfo = ({ entriesToProcess, attrsToMigrate }) => { const updates = []; for (const entry of entriesToProcess) { - const attributesValues = pick(attributesToMigrate, entry); + const attributesValues = pick(attrsToMigrate, entry); const entriesIdsToUpdate = entry.localizations.map(prop('id')); updates.push({ entriesIdsToUpdate, attributesValues }); } return updates; }; +const getSortedLocales = async ({ transacting } = {}) => { + const localeService = getService('locales'); + + let defaultLocale; + try { + const storeRes = await strapi + .query('core_store') + .findOne({ key: 'plugin_i18n_default_locale' }, null, { transacting }); + defaultLocale = JSON.parse(storeRes.value); + } catch (e) { + throw new Error("Could not migrate because the default locale doesn't exist"); + } + + const locales = await localeService.find({}, null, { transacting }); + if (isEmpty(locales)) { + throw new Error('Could not migrate because no locale exist'); + } + + // Put default locale first + return pipe( + map(locale => ({ code: locale.code, isDefault: locale.code === defaultLocale })), + orderBy(['isDefault', 'code'], ['desc', 'asc']), + map(prop('code')) + )(locales); +}; + +const areScalarAttrsOnly = ({ model, attributes }) => + pipe(pick(attributes), every(isScalarAttribute))(model.attributes); + module.exports = { shouldBeProcessed, getUpdatesInfo, + getSortedLocales, + areScalarAttrsOnly, }; diff --git a/packages/strapi-plugin-i18n/services/content-types.js b/packages/strapi-plugin-i18n/services/content-types.js index c1532727d9..cf3eb751fa 100644 --- a/packages/strapi-plugin-i18n/services/content-types.js +++ b/packages/strapi-plugin-i18n/services/content-types.js @@ -101,8 +101,9 @@ const getNonLocalizedAttributes = model => { }; const removeId = value => { - if (typeof value === 'object' && has('id', value)) { + if (typeof value === 'object' && (has('id', value) || has('_id', value))) { delete value.id; + delete value._id; } };