From 9950bbe4c09c1df48a51b3e941d1a6ae6b2dfcfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Tue, 16 Feb 2021 17:43:41 +0100 Subject: [PATCH 01/13] add previousDefinition in migration options --- .../lib/build-database-schema.js | 10 +++++++++- .../lib/migrations/draft-publish.js | 15 ++++++--------- .../lib/migrations/draft-publish.js | 13 +++++-------- .../strapi-connector-mongoose/lib/mount-models.js | 10 +++++++++- .../migrations/disable-localization-field.js | 10 ++++++++++ 5 files changed, 39 insertions(+), 19 deletions(-) create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js diff --git a/packages/strapi-connector-bookshelf/lib/build-database-schema.js b/packages/strapi-connector-bookshelf/lib/build-database-schema.js index 9f727713ce..ef9e8dfe86 100644 --- a/packages/strapi-connector-bookshelf/lib/build-database-schema.js +++ b/packages/strapi-connector-bookshelf/lib/build-database-schema.js @@ -4,7 +4,11 @@ const _ = require('lodash'); const { singular } = require('pluralize'); const { contentTypes: contentTypesUtils } = require('strapi-utils'); -const { storeDefinition, getColumnsWhereDefinitionChanged } = require('./utils/store-definition'); +const { + getDefinitionFromStore, + storeDefinition, + getColumnsWhereDefinitionChanged, +} = require('./utils/store-definition'); const { getManyRelations } = require('./utils/associations'); const migrateSchemas = async ({ ORM, loadedModel, definition, connection, model }, context) => { @@ -397,10 +401,14 @@ const createOrUpdateTable = async ({ table, attributes, definition, ORM, model } }; module.exports = async ({ ORM, loadedModel, definition, connection, model }) => { + const previousDefinitionRow = await getDefinitionFromStore(definition, ORM); + const previousDefinition = JSON.parse(_.get(previousDefinitionRow, 'value', null)); + // run migrations await strapi.db.migrations.run(migrateSchemas, { ORM, loadedModel, + previousDefinition, definition, connection, model, diff --git a/packages/strapi-connector-bookshelf/lib/migrations/draft-publish.js b/packages/strapi-connector-bookshelf/lib/migrations/draft-publish.js index 39e52449ad..6781a0aeea 100644 --- a/packages/strapi-connector-bookshelf/lib/migrations/draft-publish.js +++ b/packages/strapi-connector-bookshelf/lib/migrations/draft-publish.js @@ -4,12 +4,9 @@ const _ = require('lodash'); const { contentTypes: contentTypesUtils } = require('strapi-utils'); const { PUBLISHED_AT_ATTRIBUTE } = contentTypesUtils.constants; -const { getDefinitionFromStore } = require('../utils/store-definition'); -const getDraftAndPublishMigrationWay = async ({ definition, ORM }) => { - const previousDefRow = await getDefinitionFromStore(definition, ORM); - const previousDef = JSON.parse(_.get(previousDefRow, 'value', null)); - const previousDraftAndPublish = contentTypesUtils.hasDraftAndPublish(previousDef); +const getDraftAndPublishMigrationWay = async ({ definition, previousDefinition }) => { + const previousDraftAndPublish = contentTypesUtils.hasDraftAndPublish(previousDefinition); const actualDraftAndPublish = contentTypesUtils.hasDraftAndPublish(definition); if (previousDraftAndPublish === actualDraftAndPublish) { @@ -23,8 +20,8 @@ const getDraftAndPublishMigrationWay = async ({ definition, ORM }) => { } }; -const before = async ({ definition, ORM }, context) => { - const way = await getDraftAndPublishMigrationWay({ definition, ORM }); +const before = async ({ definition, previousDefinition, ORM }, context) => { + const way = await getDraftAndPublishMigrationWay({ definition, previousDefinition }); if (way === 'disable') { const publishedAtColumnExists = await ORM.knex.schema.hasColumn( @@ -50,8 +47,8 @@ const before = async ({ definition, ORM }, context) => { } }; -const after = async ({ definition, ORM }) => { - const way = await getDraftAndPublishMigrationWay({ definition, ORM }); +const after = async ({ definition, previousDefinition, ORM }) => { + const way = await getDraftAndPublishMigrationWay({ definition, previousDefinition }); if (way === 'enable') { const now = new Date(); diff --git a/packages/strapi-connector-mongoose/lib/migrations/draft-publish.js b/packages/strapi-connector-mongoose/lib/migrations/draft-publish.js index 139823c76c..2b3ea17bcb 100644 --- a/packages/strapi-connector-mongoose/lib/migrations/draft-publish.js +++ b/packages/strapi-connector-mongoose/lib/migrations/draft-publish.js @@ -4,15 +4,12 @@ const _ = require('lodash'); const { contentTypes: contentTypesUtils } = require('strapi-utils'); const { PUBLISHED_AT_ATTRIBUTE } = contentTypesUtils.constants; -const { getDefinitionFromStore } = require('../utils/store-definition'); -const getDraftAndPublishMigrationWay = async (definition, ORM) => { - const previousDefRow = await getDefinitionFromStore(definition, ORM); - const previousDef = JSON.parse(_.get(previousDefRow, 'value', null)); - const previousDraftAndPublish = contentTypesUtils.hasDraftAndPublish(previousDef); +const getDraftAndPublishMigrationWay = async ({ definition, previousDefinition }) => { + const previousDraftAndPublish = contentTypesUtils.hasDraftAndPublish(previousDefinition); const actualDraftAndPublish = contentTypesUtils.hasDraftAndPublish(definition); - if (!previousDefRow || previousDraftAndPublish === actualDraftAndPublish) { + if (!previousDefinition || previousDraftAndPublish === actualDraftAndPublish) { return 'none'; } if (!previousDraftAndPublish && actualDraftAndPublish) { @@ -23,8 +20,8 @@ const getDraftAndPublishMigrationWay = async (definition, ORM) => { } }; -const migrateDraftAndPublish = async ({ definition, model, ORM }) => { - let way = await getDraftAndPublishMigrationWay(definition, ORM); +const migrateDraftAndPublish = async ({ definition, previousDefinition, model }) => { + let way = await getDraftAndPublishMigrationWay({ definition, previousDefinition }); if (way === 'enable') { const createdAtCol = _.get(definition, 'timestamps.createdAt', 'createdAt'); diff --git a/packages/strapi-connector-mongoose/lib/mount-models.js b/packages/strapi-connector-mongoose/lib/mount-models.js index 9a9222f650..0b11dbc1a9 100644 --- a/packages/strapi-connector-mongoose/lib/mount-models.js +++ b/packages/strapi-connector-mongoose/lib/mount-models.js @@ -8,7 +8,11 @@ const utils = require('./utils'); const populateQueries = require('./utils/populate-queries'); const relations = require('./relations'); const { findComponentByGlobalId } = require('./utils/helpers'); -const { didDefinitionChange, storeDefinition } = require('./utils/store-definition'); +const { + didDefinitionChange, + storeDefinition, + getDefinitionFromStore, +} = require('./utils/store-definition'); const { PUBLISHED_AT_ATTRIBUTE, @@ -342,9 +346,13 @@ module.exports = async ({ models, target }, ctx) => { const modelInstance = target[model]; const definitionDidChange = await didDefinitionChange(definition, instance); + const previousDefinitionRow = await getDefinitionFromStore(definition, instance); + const previousDefinition = JSON.parse(_.get(previousDefinitionRow, 'value', null)); + // run migrations await strapi.db.migrations.run(migrateSchema, { definition, + previousDefinition, model: modelInstance, ORM: instance, }); diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js b/packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js new file mode 100644 index 0000000000..9814939433 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js @@ -0,0 +1,10 @@ +'use strict'; +// +// const before = ({ definition, ORM }) => { +// const previousDefRow = await getDefinitionFromStore(definition, ORM); +// +// }; +// +// module.exports = { +// before, +// }; From ec0aca2de8fc796563b158ed5dae9c4cd122c21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Thu, 18 Feb 2021 18:42:28 +0100 Subject: [PATCH 02/13] add migration for SQL DBs --- .../lib/build-database-schema.js | 2 +- .../migrations/disable-localization-field.js | 10 -- .../config/functions/migrations/field.js | 102 ++++++++++++++++++ .../config/functions/register.js | 6 +- .../services/content-types.js | 12 +++ packages/strapi/lib/Strapi.js | 7 +- 6 files changed, 120 insertions(+), 19 deletions(-) delete mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/field.js diff --git a/packages/strapi-connector-bookshelf/lib/build-database-schema.js b/packages/strapi-connector-bookshelf/lib/build-database-schema.js index ef9e8dfe86..3caf0ddcbb 100644 --- a/packages/strapi-connector-bookshelf/lib/build-database-schema.js +++ b/packages/strapi-connector-bookshelf/lib/build-database-schema.js @@ -98,7 +98,7 @@ const migrateSchemas = async ({ ORM, loadedModel, definition, connection, model } } - // Remove from attributes (auto handled by bookshlef and not displayed on ctb) + // Remove from attributes (auto handled by bookshelf and not displayed on ctb) if (loadedModel.hasTimestamps) { delete definition.attributes[loadedModel.hasTimestamps[0]]; delete definition.attributes[loadedModel.hasTimestamps[1]]; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js b/packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js deleted file mode 100644 index 9814939433..0000000000 --- a/packages/strapi-plugin-i18n/config/functions/migrations/disable-localization-field.js +++ /dev/null @@ -1,10 +0,0 @@ -'use strict'; -// -// const before = ({ definition, ORM }) => { -// const previousDefRow = await getDefinitionFromStore(definition, ORM); -// -// }; -// -// module.exports = { -// before, -// }; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field.js b/packages/strapi-plugin-i18n/config/functions/migrations/field.js new file mode 100644 index 0000000000..6b22590925 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field.js @@ -0,0 +1,102 @@ +'use strict'; + +const { difference, pick, orderBy, prop, intersection } = require('lodash/fp'); +const { getService } = require('../../../utils'); + +const getSubstitute = arr => arr.map(() => '??').join(', '); + +const before = () => {}; + +const after = async ({ model, definition, previousDefinition, ORM }) => { + const ctService = getService('content-types'); + const localeService = getService('locales'); + + if (!ctService.isLocalized(model)) { + return; + } + + const localizedAttributes = ctService.getLocalizedAttributes(definition); + const prevLocalizedAttributes = ctService.getLocalizedAttributes(previousDefinition); + const attributesDisabled = difference(prevLocalizedAttributes, localizedAttributes); + const attributesToMigrate = intersection(Object.keys(definition.attributes), attributesDisabled); + + if (attributesToMigrate.length === 0) { + return; + } + + let locales = await localeService.find(); + locales = await localeService.setIsDefault(locales); + locales = orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first + + const processedLocaleCodes = []; + + if (model.orm === 'bookshelf') { + const trx = await ORM.knex.transaction(); + try { + const columnsToCopy = ['id', ...attributesToMigrate]; + + await trx.raw('DROP TABLE IF EXISTS __tmp__i18n_field_migration'); + await trx.raw( + `CREATE TABLE __tmp__i18n_field_migration AS SELECT ${getSubstitute( + columnsToCopy + )} FROM ?? WHERE 0`, + [...columnsToCopy, model.collectionName] + ); + + for (const locale of locales) { + const batchSize = 1000; + let offset = 0; + let batchCount = 1000; + while (batchCount === batchSize) { + const batch = await trx + .select([...attributesToMigrate, 'locale', 'localizations']) + .from(model.collectionName) + .where('locale', locale.code) + .orderBy('id') + .offset(offset) + .limit(batchSize); + batch.forEach(entry => (entry.localizations = JSON.parse(entry.localizations))); + + batchCount = batch.length; + const entriesToProcess = batch.filter( + entry => + entry.localizations.length > 1 && + intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === + 0 + ); + + const tempEntries = entriesToProcess.reduce((entries, entry) => { + const attributesValues = pick(attributesToMigrate, entry); + const entriesIdsToUpdate = entry.localizations + .filter(related => related.locale !== locale.code) + .map(prop('id')); + + return entries.concat(entriesIdsToUpdate.map(id => ({ id, ...attributesValues }))); + }, []); + + await trx.batchInsert('__tmp__i18n_field_migration', tempEntries, 100); + + offset += batchSize; + } + processedLocaleCodes.push(locale.code); + } + + const attributesToMigrateSub = getSubstitute(attributesToMigrate); + await trx.raw( + `UPDATE ?? SET (${attributesToMigrateSub}) = (SELECT ${attributesToMigrateSub} FROM __tmp__i18n_field_migration as tmp WHERE tmp.id = ??.id) WHERE id IN(SELECT id from __tmp__i18n_field_migration)`, + [model.collectionName, ...attributesToMigrate, ...attributesToMigrate, model.collectionName] + ); + await trx.raw('DROP TABLE __tmp__i18n_field_migration'); + trx.commit(); + } catch (e) { + trx.rollback(); + } + } else if (model.orm === 'mongoose') { + // to do + } +}; + +module.exports = { + before, + after, +}; diff --git a/packages/strapi-plugin-i18n/config/functions/register.js b/packages/strapi-plugin-i18n/config/functions/register.js index 46216f6c32..6da08e999c 100644 --- a/packages/strapi-plugin-i18n/config/functions/register.js +++ b/packages/strapi-plugin-i18n/config/functions/register.js @@ -4,6 +4,7 @@ const _ = require('lodash'); const { PUBLISHED_AT_ATTRIBUTE } = require('strapi-utils').contentTypes.constants; const { getService } = require('../../utils'); +const fieldMigration = require('./migrations/field'); module.exports = () => { const contentTypeService = getService('content-types'); @@ -34,8 +35,5 @@ module.exports = () => { } }); - strapi.db.migrations.register({ - before() {}, - after() {}, - }); + strapi.db.migrations.register(fieldMigration); }; diff --git a/packages/strapi-plugin-i18n/services/content-types.js b/packages/strapi-plugin-i18n/services/content-types.js index d174aef646..ff34b27df4 100644 --- a/packages/strapi-plugin-i18n/services/content-types.js +++ b/packages/strapi-plugin-i18n/services/content-types.js @@ -95,10 +95,22 @@ const copyNonLocalizedAttributes = (model, entry) => { return pipe(pick(nonLocalizedAttributes), removeIds)(entry); }; +/** + * Returns the list of attribute names that are localized + * @param {object} model + * @returns {string[]} + */ +const getLocalizedAttributes = model => { + return getVisibleAttributes(model).filter( + attributeName => isLocalizedAttribute(model, attributeName) + ); +} + module.exports = { isLocalized, getValidLocale, getNewLocalizationsFor, + getLocalizedAttributes, getNonLocalizedAttributes, copyNonLocalizedAttributes, }; diff --git a/packages/strapi/lib/Strapi.js b/packages/strapi/lib/Strapi.js index 78f5dca7b9..614294459c 100644 --- a/packages/strapi/lib/Strapi.js +++ b/packages/strapi/lib/Strapi.js @@ -341,15 +341,14 @@ class Strapi { this.models['strapi_webhooks'] = webhookModel(this.config); this.db = createDatabaseManager(this); - - await this.runLifecyclesFunctions(LIFECYCLES.REGISTER); - await this.db.initialize(); - this.store = createCoreStore({ environment: this.config.environment, db: this.db, }); + await this.runLifecyclesFunctions(LIFECYCLES.REGISTER); + await this.db.initialize(); + this.webhookStore = createWebhookStore({ db: this.db }); await this.startWebhooks(); From 0a08ea32ced43519aa25d8def7a7407f202cb46a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Tue, 23 Feb 2021 16:52:30 +0100 Subject: [PATCH 03/13] add migration for mongo DB --- .../config/functions/migrations/field.js | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field.js b/packages/strapi-plugin-i18n/config/functions/migrations/field.js index 6b22590925..63ed31a9db 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field.js @@ -7,6 +7,7 @@ const getSubstitute = arr => arr.map(() => '??').join(', '); const before = () => {}; +// Migration when i18n is disabled on a field of a content-type that have i18n enabled const after = async ({ model, definition, previousDefinition, ORM }) => { const ctService = getService('content-types'); const localeService = getService('locales'); @@ -90,9 +91,50 @@ const after = async ({ model, definition, previousDefinition, ORM }) => { trx.commit(); } catch (e) { trx.rollback(); + throw e; } } else if (model.orm === 'mongoose') { - // to do + for (const locale of locales) { + const batchSize = 1000; + let batchCount = 1000; + let lastId; + while (batchCount === batchSize) { + const findParams = { locale: locale.code }; + if (lastId) { + findParams._id = { $gt: lastId }; + } + + const batch = await model + .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) + .sort({ _id: 1 }) + .limit(batchSize); + + if (batch.length > 0) { + lastId = batch[batch.length - 1]._id; + } + batchCount = batch.length; + + const entriesToProcess = batch.filter( + entry => + entry.localizations.length > 1 && + intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0 + ); + + const updates = entriesToProcess.reduce((entries, entry) => { + const attributesValues = pick(attributesToMigrate, entry); + const entriesIdsToUpdate = entry.localizations + .filter(related => related.locale !== locale.code) + .map(prop('id')); + + return entries.concat({ + updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, + }); + }, []); + + await model.bulkWrite(updates); + } + processedLocaleCodes.push(locale.code); + } } }; From 21659075f029eb1c8d43a044f8a433526e2aedb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Thu, 25 Feb 2021 17:40:14 +0100 Subject: [PATCH 04/13] refacto --- .../migrations/__tests__/field.test.js | 146 +++++++++++ .../config/functions/migrations/field.js | 239 ++++++++++-------- 2 files changed, 279 insertions(+), 106 deletions(-) create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js new file mode 100644 index 0000000000..95412dfe37 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js @@ -0,0 +1,146 @@ +'use strict'; + +const { cloneDeep } = require('lodash/fp'); +const { before } = require('../field'); + +describe('i18n - Migration - disable localization on a field', () => { + describe('before', () => { + describe('Should not migrate', () => { + test("Doesn't migrate if model isn't localized", async () => { + const find = jest.fn(); + global.strapi = { + query: () => { + find; + }, + }; + + const model = { + collectionName: 'dogs', + info: { name: 'dog' }, + attributes: { + name: { type: 'string' }, + code: { type: 'string' }, + }, + }; + + const previousDefinition = { + collectionName: 'dogs', + info: { name: 'dog' }, + attributes: { + name: { type: 'string' }, + }, + }; + + await before({ model, definition: model, previousDefinition }); + expect(find).not.toHaveBeenCalled(); + }); + + test("Doesn't migrate if no attribute changed (without i18n)", async () => { + const find = jest.fn(); + global.strapi = { + query: () => { + find; + }, + }; + + const model = { + collectionName: 'dogs', + info: { name: 'dog' }, + attributes: { + name: { type: 'string' }, + code: { type: 'string' }, + }, + }; + + const previousDefinition = model; + + await before({ model, definition: model, previousDefinition }); + expect(find).not.toHaveBeenCalled(); + }); + + test("Doesn't migrate if no attribute changed (with i18n)", async () => { + const find = jest.fn(); + global.strapi = { + query: () => { + find; + }, + }; + + const model = { + collectionName: 'dogs', + info: { name: 'dog' }, + pluginOptions: { i18n: { localized: true } }, + attributes: { + name: { + type: 'string', + pluginOptions: { i18n: { localized: true } }, + }, + code: { + type: 'string', + pluginOptions: { i18n: { localized: false } }, + }, + }, + }; + + const previousDefinition = model; + + await before({ model, definition: model, previousDefinition }); + expect(find).not.toHaveBeenCalled(); + }); + + test("Doesn't migrate if field not localized and pluginOptions removed", async () => { + const find = jest.fn(); + global.strapi = { + query: () => { + find; + }, + }; + + const model = { + collectionName: 'dogs', + info: { name: 'dog' }, + pluginOptions: { i18n: { localized: true } }, + attributes: { + name: { + type: 'string', + pluginOptions: { i18n: { localized: false } }, + }, + }, + }; + + const previousDefinition = cloneDeep(model); + delete previousDefinition.attributes.name.pluginOptions; + + await before({ model, definition: model, previousDefinition }); + expect(find).not.toHaveBeenCalled(); + }); + + test("Doesn't migrate if field becomes localized", async () => { + const find = jest.fn(); + global.strapi = { + query: () => { + find; + }, + }; + + const model = { + collectionName: 'dogs', + info: { name: 'dog' }, + pluginOptions: { i18n: { localized: true } }, + attributes: { + name: { + type: 'string', + pluginOptions: { i18n: { localized: true } }, + }, + }, + }; + + const previousDefinition = cloneDeep(model); + previousDefinition.attributes.name.pluginOptions.i18n.localized = false; + + await before({ model, definition: model, previousDefinition }); + expect(find).not.toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field.js b/packages/strapi-plugin-i18n/config/functions/migrations/field.js index 63ed31a9db..cda3f0ad4c 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field.js @@ -3,9 +3,136 @@ const { difference, pick, orderBy, prop, intersection } = require('lodash/fp'); const { getService } = require('../../../utils'); -const getSubstitute = arr => arr.map(() => '??').join(', '); +const BATCH_SIZE = 1000; -const before = () => {}; +const shouldBeProceed = processedLocaleCodes => entry => + entry.localizations.length > 1 && + intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0; + +const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) => { + // Create tmp table with all updates to make (faster than make updates one by one) + const TMP_TABLE_NAME = '__tmp__i18n_field_migration'; + const columnsToCopy = ['id', ...attributesToMigrate]; + await ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); + await ORM.knex.raw(`CREATE TABLE ?? AS ??`, [ + TMP_TABLE_NAME, + ORM.knex + .select(columnsToCopy) + .from(model.collectionName) + .whereRaw('?', 0), + ]); + + // Transaction is started after DDL because of MySQL (https://dev.mysql.com/doc/refman/8.0/en/implicit-commit.html) + const trx = await ORM.knex.transaction(); + + // bulk insert updates in tmp table + try { + const processedLocaleCodes = []; + for (const locale of locales) { + const batchSize = BATCH_SIZE; + let offset = 0; + let batchCount = BATCH_SIZE; + while (batchCount === batchSize) { + const batch = await trx + .select([...attributesToMigrate, 'locale', 'localizations']) + .from(model.collectionName) + .where('locale', locale.code) + .orderBy('id') + .offset(offset) + .limit(batchSize); + + // postgres automatically parses JSON, but not slite nor mysql + batch.forEach(entry => { + if (typeof entry.localizations === 'string') { + entry.localizations = JSON.parse(entry.localizations); + } + }); + + batchCount = batch.length; + const entriesToProcess = batch.filter(shouldBeProceed(processedLocaleCodes)); + + const tempEntries = entriesToProcess.reduce((entries, entry) => { + const attributesValues = pick(attributesToMigrate, entry); + const entriesIdsToUpdate = entry.localizations + .filter(related => related.locale !== locale.code) + .map(prop('id')); + + return entries.concat(entriesIdsToUpdate.map(id => ({ id, ...attributesValues }))); + }, []); + + console.log('tempEntries', tempEntries); + + await trx.batchInsert(TMP_TABLE_NAME, tempEntries, 100); + + offset += batchSize; + } + processedLocaleCodes.push(locale.code); + } + + const getSubquery = cl => + trx + .select(cl) + .from(TMP_TABLE_NAME) + .where(`${TMP_TABLE_NAME}.id`, trx.raw('??', [`${model.collectionName}.id`])); + const updates = attributesToMigrate.reduce( + (updates, cl) => ({ ...updates, [cl]: getSubquery(cl) }), + {} + ); + await trx + .from(model.collectionName) + .update(updates) + .whereIn('id', qb => qb.select(['id']).from(TMP_TABLE_NAME)); + + // Transaction is ended before DDL + await trx.commit(); + + await ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); + } catch (e) { + await trx.rollback(); + throw e; + } +}; + +const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { + const processedLocaleCodes = []; + for (const locale of locales) { + const batchSize = BATCH_SIZE; + let batchCount = BATCH_SIZE; + let lastId; + while (batchCount === batchSize) { + const findParams = { locale: locale.code }; + if (lastId) { + findParams._id = { $gt: lastId }; + } + + const batch = await model + .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) + .sort({ _id: 1 }) + .limit(batchSize); + + if (batch.length > 0) { + lastId = batch[batch.length - 1]._id; + } + batchCount = batch.length; + + const entriesToProcess = batch.filter(shouldBeProceed); + + const updates = entriesToProcess.reduce((entries, entry) => { + const attributesValues = pick(attributesToMigrate, entry); + const entriesIdsToUpdate = entry.localizations + .filter(related => related.locale !== locale.code) + .map(prop('id')); + + return entries.concat({ + updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, + }); + }, []); + + await model.bulkWrite(updates); + } + processedLocaleCodes.push(locale.code); + } +}; // Migration when i18n is disabled on a field of a content-type that have i18n enabled const after = async ({ model, definition, previousDefinition, ORM }) => { @@ -29,115 +156,15 @@ const after = async ({ model, definition, previousDefinition, ORM }) => { locales = await localeService.setIsDefault(locales); locales = orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first - const processedLocaleCodes = []; - if (model.orm === 'bookshelf') { - const trx = await ORM.knex.transaction(); - try { - const columnsToCopy = ['id', ...attributesToMigrate]; - - await trx.raw('DROP TABLE IF EXISTS __tmp__i18n_field_migration'); - await trx.raw( - `CREATE TABLE __tmp__i18n_field_migration AS SELECT ${getSubstitute( - columnsToCopy - )} FROM ?? WHERE 0`, - [...columnsToCopy, model.collectionName] - ); - - for (const locale of locales) { - const batchSize = 1000; - let offset = 0; - let batchCount = 1000; - while (batchCount === batchSize) { - const batch = await trx - .select([...attributesToMigrate, 'locale', 'localizations']) - .from(model.collectionName) - .where('locale', locale.code) - .orderBy('id') - .offset(offset) - .limit(batchSize); - batch.forEach(entry => (entry.localizations = JSON.parse(entry.localizations))); - - batchCount = batch.length; - const entriesToProcess = batch.filter( - entry => - entry.localizations.length > 1 && - intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === - 0 - ); - - const tempEntries = entriesToProcess.reduce((entries, entry) => { - const attributesValues = pick(attributesToMigrate, entry); - const entriesIdsToUpdate = entry.localizations - .filter(related => related.locale !== locale.code) - .map(prop('id')); - - return entries.concat(entriesIdsToUpdate.map(id => ({ id, ...attributesValues }))); - }, []); - - await trx.batchInsert('__tmp__i18n_field_migration', tempEntries, 100); - - offset += batchSize; - } - processedLocaleCodes.push(locale.code); - } - - const attributesToMigrateSub = getSubstitute(attributesToMigrate); - await trx.raw( - `UPDATE ?? SET (${attributesToMigrateSub}) = (SELECT ${attributesToMigrateSub} FROM __tmp__i18n_field_migration as tmp WHERE tmp.id = ??.id) WHERE id IN(SELECT id from __tmp__i18n_field_migration)`, - [model.collectionName, ...attributesToMigrate, ...attributesToMigrate, model.collectionName] - ); - await trx.raw('DROP TABLE __tmp__i18n_field_migration'); - trx.commit(); - } catch (e) { - trx.rollback(); - throw e; - } + await migrateForBookshelf({ ORM, model, attributesToMigrate, locales }); } else if (model.orm === 'mongoose') { - for (const locale of locales) { - const batchSize = 1000; - let batchCount = 1000; - let lastId; - while (batchCount === batchSize) { - const findParams = { locale: locale.code }; - if (lastId) { - findParams._id = { $gt: lastId }; - } - - const batch = await model - .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) - .sort({ _id: 1 }) - .limit(batchSize); - - if (batch.length > 0) { - lastId = batch[batch.length - 1]._id; - } - batchCount = batch.length; - - const entriesToProcess = batch.filter( - entry => - entry.localizations.length > 1 && - intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0 - ); - - const updates = entriesToProcess.reduce((entries, entry) => { - const attributesValues = pick(attributesToMigrate, entry); - const entriesIdsToUpdate = entry.localizations - .filter(related => related.locale !== locale.code) - .map(prop('id')); - - return entries.concat({ - updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, - }); - }, []); - - await model.bulkWrite(updates); - } - processedLocaleCodes.push(locale.code); - } + await migrateForMongoose({ model, attributesToMigrate, locales }); } }; +const before = () => {}; + module.exports = { before, after, From 13837d30040bf84075e2fbe39249ae474e12b1ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Mon, 1 Mar 2021 11:26:44 +0100 Subject: [PATCH 05/13] refacto --- .../config/functions/migrations/field.js | 91 +++++++++++-------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field.js b/packages/strapi-plugin-i18n/config/functions/migrations/field.js index cda3f0ad4c..ec4375be92 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field.js @@ -5,14 +5,36 @@ const { getService } = require('../../../utils'); const BATCH_SIZE = 1000; -const shouldBeProceed = processedLocaleCodes => entry => - entry.localizations.length > 1 && - intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0; +const shouldBeProcesseed = processedLocaleCodes => entry => { + return ( + entry.localizations.length > 1 && + intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0 + ); +}; + +const getUpdates = ({ entriesToProcess, formatUpdate, locale, attributesToMigrate }) => { + return entriesToProcess.reduce((updates, entry) => { + const attributesValues = pick(attributesToMigrate, entry); + const entriesIdsToUpdate = entry.localizations + .filter(related => related.locale !== locale.code) + .map(prop('id')); + + return updates.concat(formatUpdate(entriesIdsToUpdate, attributesValues)); + }, []); +}; + +const formatMongooseUpdate = (entriesIdsToUpdate, attributesValues) => ({ + updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, +}); + +const formatBookshelfUpdate = (entriesIdsToUpdate, attributesValues) => + entriesIdsToUpdate.map(id => ({ id, ...attributesValues })); const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) => { - // Create tmp table with all updates to make (faster than make updates one by one) + // Create tmp table with all updates to make (faster than making updates one by one) const TMP_TABLE_NAME = '__tmp__i18n_field_migration'; const columnsToCopy = ['id', ...attributesToMigrate]; + await ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); await ORM.knex.raw(`CREATE TABLE ?? AS ??`, [ TMP_TABLE_NAME, @@ -25,23 +47,21 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) // Transaction is started after DDL because of MySQL (https://dev.mysql.com/doc/refman/8.0/en/implicit-commit.html) const trx = await ORM.knex.transaction(); - // bulk insert updates in tmp table try { const processedLocaleCodes = []; for (const locale of locales) { - const batchSize = BATCH_SIZE; let offset = 0; let batchCount = BATCH_SIZE; - while (batchCount === batchSize) { + while (batchCount === BATCH_SIZE) { const batch = await trx .select([...attributesToMigrate, 'locale', 'localizations']) .from(model.collectionName) .where('locale', locale.code) .orderBy('id') .offset(offset) - .limit(batchSize); + .limit(BATCH_SIZE); - // postgres automatically parses JSON, but not slite nor mysql + // postgres automatically parses JSON, but not sqlite nor mysql batch.forEach(entry => { if (typeof entry.localizations === 'string') { entry.localizations = JSON.parse(entry.localizations); @@ -49,35 +69,33 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) }); batchCount = batch.length; - const entriesToProcess = batch.filter(shouldBeProceed(processedLocaleCodes)); + const entriesToProcess = batch.filter(shouldBeProcesseed(processedLocaleCodes)); - const tempEntries = entriesToProcess.reduce((entries, entry) => { - const attributesValues = pick(attributesToMigrate, entry); - const entriesIdsToUpdate = entry.localizations - .filter(related => related.locale !== locale.code) - .map(prop('id')); + const tmpEntries = getUpdates({ + entriesToProcess, + formatUpdate: formatBookshelfUpdate, + locale, + attributesToMigrate, + }); - return entries.concat(entriesIdsToUpdate.map(id => ({ id, ...attributesValues }))); - }, []); + await trx.batchInsert(TMP_TABLE_NAME, tmpEntries, 100); - console.log('tempEntries', tempEntries); - - await trx.batchInsert(TMP_TABLE_NAME, tempEntries, 100); - - offset += batchSize; + offset += BATCH_SIZE; } processedLocaleCodes.push(locale.code); } - const getSubquery = cl => + const getSubquery = columnName => trx - .select(cl) + .select(columnName) .from(TMP_TABLE_NAME) .where(`${TMP_TABLE_NAME}.id`, trx.raw('??', [`${model.collectionName}.id`])); + const updates = attributesToMigrate.reduce( - (updates, cl) => ({ ...updates, [cl]: getSubquery(cl) }), + (updates, columnName) => ({ ...updates, [columnName]: getSubquery(columnName) }), {} ); + await trx .from(model.collectionName) .update(updates) @@ -96,10 +114,9 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { const processedLocaleCodes = []; for (const locale of locales) { - const batchSize = BATCH_SIZE; let batchCount = BATCH_SIZE; let lastId; - while (batchCount === batchSize) { + while (batchCount === BATCH_SIZE) { const findParams = { locale: locale.code }; if (lastId) { findParams._id = { $gt: lastId }; @@ -108,25 +125,21 @@ const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { const batch = await model .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) .sort({ _id: 1 }) - .limit(batchSize); + .limit(BATCH_SIZE); if (batch.length > 0) { lastId = batch[batch.length - 1]._id; } batchCount = batch.length; - const entriesToProcess = batch.filter(shouldBeProceed); + const entriesToProcess = batch.filter(shouldBeProcesseed); - const updates = entriesToProcess.reduce((entries, entry) => { - const attributesValues = pick(attributesToMigrate, entry); - const entriesIdsToUpdate = entry.localizations - .filter(related => related.locale !== locale.code) - .map(prop('id')); - - return entries.concat({ - updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, - }); - }, []); + const updates = getUpdates({ + entriesToProcess, + formatUpdate: formatMongooseUpdate, + locale, + attributesToMigrate, + }); await model.bulkWrite(updates); } From a662db9691a03be1b9e71e5859dbd6b3703a1704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Tue, 2 Mar 2021 16:42:17 +0100 Subject: [PATCH 06/13] refacto --- .../config/functions/migrations/field.js | 138 +++++++++++------- 1 file changed, 87 insertions(+), 51 deletions(-) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field.js b/packages/strapi-plugin-i18n/config/functions/migrations/field.js index ec4375be92..1d32d62ce8 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field.js @@ -5,6 +5,8 @@ const { getService } = require('../../../utils'); const BATCH_SIZE = 1000; +// Common functions + const shouldBeProcesseed = processedLocaleCodes => entry => { return ( entry.localizations.length > 1 && @@ -12,29 +14,63 @@ const shouldBeProcesseed = processedLocaleCodes => entry => { ); }; -const getUpdates = ({ entriesToProcess, formatUpdate, locale, attributesToMigrate }) => { - return entriesToProcess.reduce((updates, entry) => { +const getUpdatesInfo = ({ entriesToProcess, locale, attributesToMigrate }) => { + const updates = []; + for (const entry of entriesToProcess) { const attributesValues = pick(attributesToMigrate, entry); const entriesIdsToUpdate = entry.localizations .filter(related => related.locale !== locale.code) .map(prop('id')); - - return updates.concat(formatUpdate(entriesIdsToUpdate, attributesValues)); - }, []); + updates.push({ entriesIdsToUpdate, attributesValues }); + } + return updates; }; -const formatMongooseUpdate = (entriesIdsToUpdate, attributesValues) => ({ - updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, -}); +// Bookshelf -const formatBookshelfUpdate = (entriesIdsToUpdate, attributesValues) => - entriesIdsToUpdate.map(id => ({ id, ...attributesValues })); +const TMP_TABLE_NAME = '__tmp__i18n_field_migration'; -const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) => { - // Create tmp table with all updates to make (faster than making updates one by one) - const TMP_TABLE_NAME = '__tmp__i18n_field_migration'; +const batchInsertInTmpTable = async (updatesInfo, trx) => { + const tmpEntries = []; + updatesInfo.forEach(({ entriesIdsToUpdate, attributesValues }) => { + entriesIdsToUpdate.forEach(id => { + tmpEntries.push({ id, ...attributesValues }); + }); + }); + await trx.batchInsert(TMP_TABLE_NAME, tmpEntries, 100); +}; + +const batchUpdate = async (updatesInfo, trx, model) => { + const promises = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => + trx + .from(model.collectionName) + .update(attributesValues) + .whereIn('id', entriesIdsToUpdate) + ); + await Promise.all(promises); +}; + +const updateFromTmpTable = async ({ model, trx, attributesToMigrate }) => { + const collectionName = model.collectionName; + let bindings = []; + if (model.client === 'pg') { + const substitutes = attributesToMigrate.map(() => '?? = ??.??').join(','); + bindings.push(collectionName); + attributesToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr)); + bindings.push(TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME); + + await trx.raw(`UPDATE ?? SET ${substitutes} FROM ?? WHERE ??.id = ??.id;`, bindings); + } else if (model.client === 'mysql') { + const substitutes = attributesToMigrate.map(() => '??.?? = ??.??').join(','); + bindings.push(collectionName, TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME); + attributesToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr)); + + await trx.raw(`UPDATE ?? JOIN ?? ON ??.id = ??.id SET ${substitutes};`, bindings); + } +}; + +const createTmpTable = async ({ ORM, attributesToMigrate, model }) => { const columnsToCopy = ['id', ...attributesToMigrate]; - await ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); await ORM.knex.raw(`CREATE TABLE ?? AS ??`, [ TMP_TABLE_NAME, @@ -43,16 +79,25 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) .from(model.collectionName) .whereRaw('?', 0), ]); +}; + +const deleteTmpTable = ({ ORM }) => ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); + +const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) => { + // The migration is custom for pg and mysql for better perfomance + const isPgOrMysql = ['pg', 'mysql'].includes(model.client); + + if (isPgOrMysql) { + await createTmpTable({ ORM, attributesToMigrate, model }); + } - // Transaction is started after DDL because of MySQL (https://dev.mysql.com/doc/refman/8.0/en/implicit-commit.html) const trx = await ORM.knex.transaction(); - try { const processedLocaleCodes = []; for (const locale of locales) { let offset = 0; - let batchCount = BATCH_SIZE; - while (batchCount === BATCH_SIZE) { + // eslint-disable-next-line no-constant-condition + while (true) { const batch = await trx .select([...attributesToMigrate, 'locale', 'localizations']) .from(model.collectionName) @@ -61,6 +106,8 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) .offset(offset) .limit(BATCH_SIZE); + offset += BATCH_SIZE; + // postgres automatically parses JSON, but not sqlite nor mysql batch.forEach(entry => { if (typeof entry.localizations === 'string') { @@ -68,49 +115,39 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) } }); - batchCount = batch.length; const entriesToProcess = batch.filter(shouldBeProcesseed(processedLocaleCodes)); + const updatesInfo = getUpdatesInfo({ entriesToProcess, locale, attributesToMigrate }); - const tmpEntries = getUpdates({ - entriesToProcess, - formatUpdate: formatBookshelfUpdate, - locale, - attributesToMigrate, - }); + if (isPgOrMysql) { + await batchInsertInTmpTable(updatesInfo, trx); + } else { + await batchUpdate(updatesInfo, trx, model); + } - await trx.batchInsert(TMP_TABLE_NAME, tmpEntries, 100); - - offset += BATCH_SIZE; + if (batch.length < BATCH_SIZE) { + break; + } } processedLocaleCodes.push(locale.code); } - const getSubquery = columnName => - trx - .select(columnName) - .from(TMP_TABLE_NAME) - .where(`${TMP_TABLE_NAME}.id`, trx.raw('??', [`${model.collectionName}.id`])); + if (isPgOrMysql) { + await updateFromTmpTable({ model, trx, attributesToMigrate }); + } - const updates = attributesToMigrate.reduce( - (updates, columnName) => ({ ...updates, [columnName]: getSubquery(columnName) }), - {} - ); - - await trx - .from(model.collectionName) - .update(updates) - .whereIn('id', qb => qb.select(['id']).from(TMP_TABLE_NAME)); - - // Transaction is ended before DDL await trx.commit(); - await ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); + if (isPgOrMysql) { + await deleteTmpTable({ ORM }); + } } catch (e) { await trx.rollback(); throw e; } }; +// Mongoose + const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { const processedLocaleCodes = []; for (const locale of locales) { @@ -134,12 +171,10 @@ const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { const entriesToProcess = batch.filter(shouldBeProcesseed); - const updates = getUpdates({ - entriesToProcess, - formatUpdate: formatMongooseUpdate, - locale, - attributesToMigrate, - }); + const updatesInfo = getUpdatesInfo({ entriesToProcess, locale, attributesToMigrate }); + const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({ + updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, + })); await model.bulkWrite(updates); } @@ -174,6 +209,7 @@ const after = async ({ model, definition, previousDefinition, ORM }) => { } else if (model.orm === 'mongoose') { await migrateForMongoose({ model, attributesToMigrate, locales }); } + throw new Error('Done'); }; const before = () => {}; From 54512c6049036766f607b890c1b87c4a4e69caf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Tue, 2 Mar 2021 17:09:35 +0100 Subject: [PATCH 07/13] split code into files --- .../functions/migrations/field/index.js | 42 ++++++++ .../migrateForBookshelf.js} | 99 +------------------ .../migrations/field/migrateForMongoose.js | 41 ++++++++ .../functions/migrations/field/utils.js | 27 +++++ 4 files changed, 112 insertions(+), 97 deletions(-) create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/field/index.js rename packages/strapi-plugin-i18n/config/functions/migrations/{field.js => field/migrateForBookshelf.js} (55%) create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js new file mode 100644 index 0000000000..1bd866f512 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js @@ -0,0 +1,42 @@ +'use strict'; + +const { difference, orderBy, intersection } = require('lodash/fp'); +const { getService } = require('../../../../utils'); +const migrateForMongoose = require('./migrateForMongoose'); +const migrateForBookshelf = require('./migrateForBookshelf'); + +// Migration when i18n is disabled on a field of a content-type that have i18n enabled +const after = async ({ model, definition, previousDefinition, ORM }) => { + const ctService = getService('content-types'); + const localeService = getService('locales'); + + if (!ctService.isLocalized(model)) { + return; + } + + const localizedAttributes = ctService.getLocalizedFields(definition); + const prevLocalizedAttributes = ctService.getLocalizedFields(previousDefinition); + const attributesDisabled = difference(prevLocalizedAttributes, localizedAttributes); + const attributesToMigrate = intersection(Object.keys(definition.attributes), attributesDisabled); + + if (attributesToMigrate.length === 0) { + return; + } + + let locales = await localeService.find(); + locales = await localeService.setIsDefault(locales); + locales = orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first + + if (model.orm === 'bookshelf') { + await migrateForBookshelf({ ORM, model, attributesToMigrate, locales }); + } else if (model.orm === 'mongoose') { + await migrateForMongoose({ model, attributesToMigrate, locales }); + } +}; + +const before = () => {}; + +module.exports = { + before, + after, +}; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js similarity index 55% rename from packages/strapi-plugin-i18n/config/functions/migrations/field.js rename to packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js index 1d32d62ce8..90da51e16f 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js @@ -1,33 +1,9 @@ 'use strict'; -const { difference, pick, orderBy, prop, intersection } = require('lodash/fp'); -const { getService } = require('../../../utils'); +const { shouldBeProcesseed, getUpdatesInfo } = require('./utils'); const BATCH_SIZE = 1000; -// Common functions - -const shouldBeProcesseed = processedLocaleCodes => entry => { - return ( - entry.localizations.length > 1 && - intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0 - ); -}; - -const getUpdatesInfo = ({ entriesToProcess, locale, attributesToMigrate }) => { - const updates = []; - for (const entry of entriesToProcess) { - const attributesValues = pick(attributesToMigrate, entry); - const entriesIdsToUpdate = entry.localizations - .filter(related => related.locale !== locale.code) - .map(prop('id')); - updates.push({ entriesIdsToUpdate, attributesValues }); - } - return updates; -}; - -// Bookshelf - const TMP_TABLE_NAME = '__tmp__i18n_field_migration'; const batchInsertInTmpTable = async (updatesInfo, trx) => { @@ -146,75 +122,4 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) } }; -// Mongoose - -const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { - const processedLocaleCodes = []; - for (const locale of locales) { - let batchCount = BATCH_SIZE; - let lastId; - while (batchCount === BATCH_SIZE) { - const findParams = { locale: locale.code }; - if (lastId) { - findParams._id = { $gt: lastId }; - } - - const batch = await model - .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) - .sort({ _id: 1 }) - .limit(BATCH_SIZE); - - if (batch.length > 0) { - lastId = batch[batch.length - 1]._id; - } - batchCount = batch.length; - - const entriesToProcess = batch.filter(shouldBeProcesseed); - - const updatesInfo = getUpdatesInfo({ entriesToProcess, locale, attributesToMigrate }); - const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({ - updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, - })); - - await model.bulkWrite(updates); - } - processedLocaleCodes.push(locale.code); - } -}; - -// Migration when i18n is disabled on a field of a content-type that have i18n enabled -const after = async ({ model, definition, previousDefinition, ORM }) => { - const ctService = getService('content-types'); - const localeService = getService('locales'); - - if (!ctService.isLocalized(model)) { - return; - } - - const localizedAttributes = ctService.getLocalizedAttributes(definition); - const prevLocalizedAttributes = ctService.getLocalizedAttributes(previousDefinition); - const attributesDisabled = difference(prevLocalizedAttributes, localizedAttributes); - const attributesToMigrate = intersection(Object.keys(definition.attributes), attributesDisabled); - - if (attributesToMigrate.length === 0) { - return; - } - - let locales = await localeService.find(); - locales = await localeService.setIsDefault(locales); - locales = orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first - - if (model.orm === 'bookshelf') { - await migrateForBookshelf({ ORM, model, attributesToMigrate, locales }); - } else if (model.orm === 'mongoose') { - await migrateForMongoose({ model, attributesToMigrate, locales }); - } - throw new Error('Done'); -}; - -const before = () => {}; - -module.exports = { - before, - after, -}; +module.exports = migrateForBookshelf; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js new file mode 100644 index 0000000000..8bbe44bc65 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js @@ -0,0 +1,41 @@ +'use strict'; + +const { shouldBeProcesseed, getUpdatesInfo } = require('./utils'); + +const BATCH_SIZE = 1000; + +const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { + const processedLocaleCodes = []; + for (const locale of locales) { + let batchCount = BATCH_SIZE; + let lastId; + while (batchCount === BATCH_SIZE) { + const findParams = { locale: locale.code }; + if (lastId) { + findParams._id = { $gt: lastId }; + } + + const batch = await model + .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) + .sort({ _id: 1 }) + .limit(BATCH_SIZE); + + if (batch.length > 0) { + lastId = batch[batch.length - 1]._id; + } + batchCount = batch.length; + + const entriesToProcess = batch.filter(shouldBeProcesseed); + + const updatesInfo = getUpdatesInfo({ entriesToProcess, locale, attributesToMigrate }); + const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({ + updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, + })); + + await model.bulkWrite(updates); + } + processedLocaleCodes.push(locale.code); + } +}; + +module.exports = migrateForMongoose; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js new file mode 100644 index 0000000000..c5db345031 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js @@ -0,0 +1,27 @@ +'use strict'; + +const { pick, prop, intersection } = require('lodash/fp'); + +const shouldBeProcesseed = processedLocaleCodes => entry => { + return ( + entry.localizations.length > 1 && + intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0 + ); +}; + +const getUpdatesInfo = ({ entriesToProcess, locale, attributesToMigrate }) => { + const updates = []; + for (const entry of entriesToProcess) { + const attributesValues = pick(attributesToMigrate, entry); + const entriesIdsToUpdate = entry.localizations + .filter(related => related.locale !== locale.code) + .map(prop('id')); + updates.push({ entriesIdsToUpdate, attributesValues }); + } + return updates; +}; + +module.exports = { + shouldBeProcesseed, + getUpdatesInfo, +}; From b39aa3c8bdd3fb60d912222b8160225b42260ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Thu, 11 Mar 2021 10:41:35 +0100 Subject: [PATCH 08/13] WIP --- .../migrations/__tests__/field.test.js | 156 ++++++++++-------- .../migrations/field/__tests__/utils.test.js | 53 ++++++ .../functions/migrations/field/index.js | 1 + .../migrations/field/migrateForBookshelf.js | 76 ++++++--- .../migrations/field/migrateForMongoose.js | 7 +- .../functions/migrations/field/utils.js | 12 +- packages/strapi-plugin-i18n/package.json | 1 + packages/strapi/lib/Strapi.js | 7 +- yarn.lock | 2 +- 9 files changed, 207 insertions(+), 108 deletions(-) create mode 100644 packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js index 95412dfe37..4acc432ba2 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js @@ -1,10 +1,9 @@ 'use strict'; -const { cloneDeep } = require('lodash/fp'); -const { before } = require('../field'); +const { after } = require('../field'); describe('i18n - Migration - disable localization on a field', () => { - describe('before', () => { + describe('after', () => { describe('Should not migrate', () => { test("Doesn't migrate if model isn't localized", async () => { const find = jest.fn(); @@ -12,133 +11,144 @@ describe('i18n - Migration - disable localization on a field', () => { query: () => { find; }, - }; - - const model = { - collectionName: 'dogs', - info: { name: 'dog' }, - attributes: { - name: { type: 'string' }, - code: { type: 'string' }, + plugins: { + i18n: { + services: { + 'content-types': { + isLocalized: () => false, + }, + }, + }, }, }; - const previousDefinition = { - collectionName: 'dogs', - info: { name: 'dog' }, - attributes: { - name: { type: 'string' }, - }, - }; + const model = {}; + const previousDefinition = {}; - await before({ model, definition: model, previousDefinition }); + await after({ model, definition: model, previousDefinition }); expect(find).not.toHaveBeenCalled(); }); test("Doesn't migrate if no attribute changed (without i18n)", async () => { const find = jest.fn(); + const getLocalizedFields = jest + .fn() + .mockReturnValueOnce([]) + .mockReturnValueOnce([]); + global.strapi = { query: () => { find; }, - }; - - const model = { - collectionName: 'dogs', - info: { name: 'dog' }, - attributes: { - name: { type: 'string' }, - code: { type: 'string' }, + plugins: { + i18n: { + services: { + 'content-types': { + isLocalized: () => true, + getLocalizedFields, + }, + }, + }, }, }; - const previousDefinition = model; + const model = { attributes: { name: {} } }; + const previousDefinition = { attributes: { name: {} } }; - await before({ model, definition: model, previousDefinition }); + await after({ model, definition: model, previousDefinition }); + expect(getLocalizedFields).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); test("Doesn't migrate if no attribute changed (with i18n)", async () => { const find = jest.fn(); + const getLocalizedFields = jest + .fn() + .mockReturnValueOnce(['name']) + .mockReturnValueOnce(['name']); global.strapi = { query: () => { find; }, - }; - - const model = { - collectionName: 'dogs', - info: { name: 'dog' }, - pluginOptions: { i18n: { localized: true } }, - attributes: { - name: { - type: 'string', - pluginOptions: { i18n: { localized: true } }, - }, - code: { - type: 'string', - pluginOptions: { i18n: { localized: false } }, + plugins: { + i18n: { + services: { + 'content-types': { + isLocalized: () => true, + getLocalizedFields, + }, + }, }, }, }; - const previousDefinition = model; + const model = { attributes: { name: {} } }; + const previousDefinition = { attributes: { name: {} } }; - await before({ model, definition: model, previousDefinition }); + await after({ model, definition: model, previousDefinition }); + expect(getLocalizedFields).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); - test("Doesn't migrate if field not localized and pluginOptions removed", async () => { + test("Doesn't migrate if field become localized", async () => { const find = jest.fn(); + const getLocalizedFields = jest + .fn() + .mockReturnValueOnce(['name']) + .mockReturnValueOnce([]); + global.strapi = { query: () => { find; }, - }; - - const model = { - collectionName: 'dogs', - info: { name: 'dog' }, - pluginOptions: { i18n: { localized: true } }, - attributes: { - name: { - type: 'string', - pluginOptions: { i18n: { localized: false } }, + plugins: { + i18n: { + services: { + 'content-types': { + isLocalized: () => true, + getLocalizedFields, + }, + }, }, }, }; - const previousDefinition = cloneDeep(model); - delete previousDefinition.attributes.name.pluginOptions; + const model = { attributes: { name: {} } }; + const previousDefinition = { attributes: { name: {} } }; - await before({ model, definition: model, previousDefinition }); + await after({ model, definition: model, previousDefinition }); + expect(getLocalizedFields).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); - test("Doesn't migrate if field becomes localized", async () => { + test("Doesn't migrate if field is deleted", async () => { const find = jest.fn(); + const getLocalizedFields = jest + .fn() + .mockReturnValueOnce([]) + .mockReturnValueOnce(['name']); + global.strapi = { query: () => { find; }, - }; - - const model = { - collectionName: 'dogs', - info: { name: 'dog' }, - pluginOptions: { i18n: { localized: true } }, - attributes: { - name: { - type: 'string', - pluginOptions: { i18n: { localized: true } }, + plugins: { + i18n: { + services: { + 'content-types': { + isLocalized: () => true, + getLocalizedFields, + }, + }, }, }, }; - const previousDefinition = cloneDeep(model); - previousDefinition.attributes.name.pluginOptions.i18n.localized = false; + const model = { attributes: {} }; + const previousDefinition = { attributes: { name: {} } }; - await before({ model, definition: model, previousDefinition }); + await after({ model, definition: model, previousDefinition }); + expect(getLocalizedFields).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); }); diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js new file mode 100644 index 0000000000..5af244f3f8 --- /dev/null +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js @@ -0,0 +1,53 @@ +'use strict'; + +const { shouldBeProcessed, getUpdatesInfo } = require('../utils'); + +describe('i18n - migration utils', () => { + describe('shouldBeProcessed', () => { + const testData = [ + [[], [], false], + [['en'], [], false], + [['en', 'fr'], [], false], + [['en', 'fr'], [{ locale: 'en' }], false], + [['en', 'fr'], [{ locale: 'fr' }], false], + [['en'], [{ locale: 'fr' }, { locale: 'en' }], false], + [['en', 'fr'], [{ locale: 'fr' }, { locale: 'en' }], false], + [[], [{ locale: 'en' }], true], + [['en'], [{ locale: 'fr' }], true], + [['en', 'fr'], [{ locale: 'it' }], true], + ]; + + test.each(testData)('%p %j : %p', (processedLocaleCodes, localizations, expectedResult) => { + const result = shouldBeProcessed(processedLocaleCodes)({ localizations }); + + expect(result).toBe(expectedResult); + }); + }); + + describe('getUpdatesInfo', () => { + const testData = [ + [ + [{ name: 'Name', nickname: 'Nickname', localizations: [{ id: 1 }, { id: 2 }] }], + ['name'], + [{ entriesIdsToUpdate: [1, 2], attributesValues: { name: 'Name' } }], + ], + [ + [ + { name: 'Name 1', nickname: 'Nickname 1', localizations: [{ id: 1 }, { id: 2 }] }, + { name: 'Name 2', nickname: 'Nickname 2', localizations: [{ id: 3 }, { id: 4 }] }, + ], + ['name'], + [ + { entriesIdsToUpdate: [1, 2], attributesValues: { name: 'Name 1' } }, + { entriesIdsToUpdate: [3, 4], attributesValues: { name: 'Name 2' } }, + ], + ], + ]; + + test.each(testData)('%j', (entriesToProcess, attributesToMigrate, expectedResult) => { + const result = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); + + expect(result).toEqual(expectedResult); + }); + }); +}); diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js index 1bd866f512..cb52ffe7ae 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js @@ -32,6 +32,7 @@ const after = async ({ model, definition, previousDefinition, ORM }) => { } else if (model.orm === 'mongoose') { await migrateForMongoose({ model, attributesToMigrate, locales }); } + throw new Error('pouet'); }; const before = () => {}; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js index 90da51e16f..aa79c83d70 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js @@ -1,6 +1,8 @@ 'use strict'; -const { shouldBeProcesseed, getUpdatesInfo } = require('./utils'); +const { singular } = require('pluralize'); +const { has, omit, pick } = require('lodash/fp'); +const { shouldBeProcessed, getUpdatesInfo } = require('./utils'); const BATCH_SIZE = 1000; @@ -28,17 +30,16 @@ const batchUpdate = async (updatesInfo, trx, model) => { const updateFromTmpTable = async ({ model, trx, attributesToMigrate }) => { const collectionName = model.collectionName; - let bindings = []; if (model.client === 'pg') { const substitutes = attributesToMigrate.map(() => '?? = ??.??').join(','); - bindings.push(collectionName); + const bindings = [collectionName]; attributesToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr)); bindings.push(TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME); await trx.raw(`UPDATE ?? SET ${substitutes} FROM ?? WHERE ??.id = ??.id;`, bindings); } else if (model.client === 'mysql') { const substitutes = attributesToMigrate.map(() => '??.?? = ??.??').join(','); - bindings.push(collectionName, TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME); + const bindings = [collectionName, TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME]; attributesToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr)); await trx.raw(`UPDATE ?? JOIN ?? ON ??.id = ??.id SET ${substitutes};`, bindings); @@ -60,6 +61,9 @@ const createTmpTable = async ({ ORM, attributesToMigrate, model }) => { const deleteTmpTable = ({ ORM }) => ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) => { + const localizationAssoc = model.associations.find(a => a.alias === 'localizations'); + const localizationTableName = localizationAssoc.tableCollectionName; + // The migration is custom for pg and mysql for better perfomance const isPgOrMysql = ['pg', 'mysql'].includes(model.client); @@ -68,31 +72,61 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) } const trx = await ORM.knex.transaction(); + + const locsAttr = model.attributes.localizations; + const foreignKey = `${singular(model.collectionName)}_${model.primaryKey}`; + const relatedKey = `${locsAttr.attribute}_${locsAttr.column}`; try { const processedLocaleCodes = []; for (const locale of locales) { let offset = 0; // eslint-disable-next-line no-constant-condition while (true) { - const batch = await trx - .select([...attributesToMigrate, 'locale', 'localizations']) - .from(model.collectionName) - .where('locale', locale.code) - .orderBy('id') - .offset(offset) - .limit(BATCH_SIZE); + let batch = await trx + .select([ + ...attributesToMigrate.map(attr => `model.${attr}`), + 'model.id as __strapi_rootId', + 'relModel.id as id', + 'relModel.locale', + ]) + .join( + `${localizationTableName} as loc`, + `model.${model.primaryKey}`, + '=', + `loc.${foreignKey}` + ) + .join( + `${model.collectionName} as relModel`, + `loc.${relatedKey}`, + '=', + `relModel.${model.primaryKey}` + ) + .from( + trx + .select('*') + .from(`${model.collectionName} as subModel`) + .orderBy(`subModel.${model.primaryKey}`) + .where(`subModel.locale`, locale.code) + .offset(offset) + .limit(BATCH_SIZE) + .as('model') + ); + let entries = batch.reduce((entries, entry) => { + if (has(entry.__strapi_rootId, entries)) { + entries[entry.__strapi_rootId].localizations.push(pick(['id', 'locale'], entry)); + } else { + entries[entry.__strapi_rootId] = omit(['id', 'locale', '__strapi_rootId'], entry); + entries[entry.__strapi_rootId].localizations = [pick(['id', 'locale'], entry)]; + } + + return entries; + }, {}); + entries = Object.values(entries); offset += BATCH_SIZE; - // postgres automatically parses JSON, but not sqlite nor mysql - batch.forEach(entry => { - if (typeof entry.localizations === 'string') { - entry.localizations = JSON.parse(entry.localizations); - } - }); - - const entriesToProcess = batch.filter(shouldBeProcesseed(processedLocaleCodes)); - const updatesInfo = getUpdatesInfo({ entriesToProcess, locale, attributesToMigrate }); + const entriesToProcess = batch.filter(shouldBeProcessed(processedLocaleCodes)); + const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); if (isPgOrMysql) { await batchInsertInTmpTable(updatesInfo, trx); @@ -100,7 +134,7 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) await batchUpdate(updatesInfo, trx, model); } - if (batch.length < BATCH_SIZE) { + if (entries.length < BATCH_SIZE) { break; } } diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js index 8bbe44bc65..31bc8ad0a0 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js @@ -1,6 +1,6 @@ 'use strict'; -const { shouldBeProcesseed, getUpdatesInfo } = require('./utils'); +const { shouldBeProcessed, getUpdatesInfo } = require('./utils'); const BATCH_SIZE = 1000; @@ -17,6 +17,7 @@ const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { const batch = await model .find(findParams, [...attributesToMigrate, 'locale', 'localizations']) + .populate('localizations', 'locale id') .sort({ _id: 1 }) .limit(BATCH_SIZE); @@ -25,9 +26,9 @@ const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { } batchCount = batch.length; - const entriesToProcess = batch.filter(shouldBeProcesseed); + const entriesToProcess = batch.filter(shouldBeProcessed(processedLocaleCodes)); - const updatesInfo = getUpdatesInfo({ entriesToProcess, locale, attributesToMigrate }); + const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({ updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues }, })); diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js index c5db345031..39fb9baf41 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js @@ -2,26 +2,24 @@ const { pick, prop, intersection } = require('lodash/fp'); -const shouldBeProcesseed = processedLocaleCodes => entry => { +const shouldBeProcessed = processedLocaleCodes => entry => { return ( - entry.localizations.length > 1 && + entry.localizations.length > 0 && intersection(entry.localizations.map(prop('locale')), processedLocaleCodes).length === 0 ); }; -const getUpdatesInfo = ({ entriesToProcess, locale, attributesToMigrate }) => { +const getUpdatesInfo = ({ entriesToProcess, attributesToMigrate }) => { const updates = []; for (const entry of entriesToProcess) { const attributesValues = pick(attributesToMigrate, entry); - const entriesIdsToUpdate = entry.localizations - .filter(related => related.locale !== locale.code) - .map(prop('id')); + const entriesIdsToUpdate = entry.localizations.map(prop('id')); updates.push({ entriesIdsToUpdate, attributesValues }); } return updates; }; module.exports = { - shouldBeProcesseed, + shouldBeProcessed, getUpdatesInfo, }; diff --git a/packages/strapi-plugin-i18n/package.json b/packages/strapi-plugin-i18n/package.json index efedb8a418..6a37cb9fc2 100644 --- a/packages/strapi-plugin-i18n/package.json +++ b/packages/strapi-plugin-i18n/package.json @@ -10,6 +10,7 @@ }, "dependencies": { "lodash": "4.17.20", + "pluralize": "8.0.0", "strapi-utils": "3.5.3" }, "author": { diff --git a/packages/strapi/lib/Strapi.js b/packages/strapi/lib/Strapi.js index 614294459c..78f5dca7b9 100644 --- a/packages/strapi/lib/Strapi.js +++ b/packages/strapi/lib/Strapi.js @@ -341,14 +341,15 @@ class Strapi { this.models['strapi_webhooks'] = webhookModel(this.config); this.db = createDatabaseManager(this); + + await this.runLifecyclesFunctions(LIFECYCLES.REGISTER); + await this.db.initialize(); + this.store = createCoreStore({ environment: this.config.environment, db: this.db, }); - await this.runLifecyclesFunctions(LIFECYCLES.REGISTER); - await this.db.initialize(); - this.webhookStore = createWebhookStore({ db: this.db }); await this.startWebhooks(); diff --git a/yarn.lock b/yarn.lock index 2b859fa7ac..f7484b4b2a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15164,7 +15164,7 @@ please-upgrade-node@^3.2.0: dependencies: semver-compare "^1.0.0" -pluralize@^8.0.0: +pluralize@8.0.0, pluralize@^8.0.0: version "8.0.0" resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== From ce84bd0325ec38fcd986daa5620fc478ed648519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Mon, 15 Mar 2021 18:51:32 +0100 Subject: [PATCH 09/13] refacto --- .../functions/migrations/field/index.js | 14 +++------ .../migrations/field/migrateForBookshelf.js | 29 +++++++++++++++++-- .../migrations/field/migrateForMongoose.js | 27 ++++++++++++++++- 3 files changed, 56 insertions(+), 14 deletions(-) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js index cb52ffe7ae..f9f3251338 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js @@ -1,6 +1,6 @@ 'use strict'; -const { difference, orderBy, intersection } = require('lodash/fp'); +const { difference, intersection } = require('lodash/fp'); const { getService } = require('../../../../utils'); const migrateForMongoose = require('./migrateForMongoose'); const migrateForBookshelf = require('./migrateForBookshelf'); @@ -8,9 +8,8 @@ const migrateForBookshelf = require('./migrateForBookshelf'); // Migration when i18n is disabled on a field of a content-type that have i18n enabled const after = async ({ model, definition, previousDefinition, ORM }) => { const ctService = getService('content-types'); - const localeService = getService('locales'); - if (!ctService.isLocalized(model)) { + if (!ctService.isLocalized(model) || !previousDefinition) { return; } @@ -23,16 +22,11 @@ const after = async ({ model, definition, previousDefinition, ORM }) => { return; } - let locales = await localeService.find(); - locales = await localeService.setIsDefault(locales); - locales = orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first - if (model.orm === 'bookshelf') { - await migrateForBookshelf({ ORM, model, attributesToMigrate, locales }); + await migrateForBookshelf({ ORM, model, attributesToMigrate }); } else if (model.orm === 'mongoose') { - await migrateForMongoose({ model, attributesToMigrate, locales }); + await migrateForMongoose({ model, attributesToMigrate }); } - throw new Error('pouet'); }; const before = () => {}; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js index aa79c83d70..d6296c7243 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js @@ -1,7 +1,7 @@ 'use strict'; const { singular } = require('pluralize'); -const { has, omit, pick } = require('lodash/fp'); +const { has, omit, pick, orderBy } = require('lodash/fp'); const { shouldBeProcessed, getUpdatesInfo } = require('./utils'); const BATCH_SIZE = 1000; @@ -60,7 +60,30 @@ const createTmpTable = async ({ ORM, attributesToMigrate, model }) => { const deleteTmpTable = ({ ORM }) => ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); -const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) => { +const getSortedLocales = async ORM => { + let defaultLocale; + try { + const defaultLocaleRow = await ORM.knex('core_store') + .select('value') + .where({ key: 'plugin_i18n_default_locale' }); + defaultLocale = defaultLocaleRow[0].value; + } catch (e) { + throw new Error("Could not migrate because the default locale doesn't exist"); + } + + let locales; + try { + locales = await ORM.knex(strapi.plugins.i18n.models.locale.collectionName).select('code'); + } catch (e) { + throw new Error('Could not migrate because no locale exist'); + } + + locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale)); + return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first +}; + +const migrateForBookshelf = async ({ ORM, model, attributesToMigrate }) => { + const locales = await getSortedLocales(ORM); const localizationAssoc = model.associations.find(a => a.alias === 'localizations'); const localizationTableName = localizationAssoc.tableCollectionName; @@ -125,7 +148,7 @@ const migrateForBookshelf = async ({ ORM, model, attributesToMigrate, locales }) offset += BATCH_SIZE; - const entriesToProcess = batch.filter(shouldBeProcessed(processedLocaleCodes)); + const entriesToProcess = entries.filter(shouldBeProcessed(processedLocaleCodes)); const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); if (isPgOrMysql) { diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js index 31bc8ad0a0..5a7e0715df 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js @@ -1,10 +1,35 @@ 'use strict'; +const { orderBy } = require('lodash/fp'); const { shouldBeProcessed, getUpdatesInfo } = require('./utils'); const BATCH_SIZE = 1000; -const migrateForMongoose = async ({ model, attributesToMigrate, locales }) => { +const getSortedLocales = async () => { + let defaultLocale; + try { + const defaultLocaleRow = await strapi.models['core_store'].findOne({ + key: 'plugin_i18n_default_locale', + }); + defaultLocale = JSON.parse(defaultLocaleRow.value); + } catch (e) { + throw new Error("Could not migrate because the default locale doesn't exist"); + } + + let locales; + try { + strapi.models; + locales = await strapi.plugins.i18n.models.locale.find(); + } catch (e) { + throw new Error('Could not migrate because no locale exist'); + } + + locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale)); + return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first +}; + +const migrateForMongoose = async ({ model, attributesToMigrate }) => { + const locales = await getSortedLocales(); const processedLocaleCodes = []; for (const locale of locales) { let batchCount = BATCH_SIZE; From f9c95eed259aaa5616030bf5f9be8a1479c4ef03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Thu, 18 Mar 2021 18:27:15 +0100 Subject: [PATCH 10/13] refacto --- .../functions/migrations/field/index.js | 4 +- ...rBookshelf.js => migrate-for-bookshelf.js} | 155 ++++++++---------- ...ForMongoose.js => migrate-for-mongoose.js} | 10 +- 3 files changed, 79 insertions(+), 90 deletions(-) rename packages/strapi-plugin-i18n/config/functions/migrations/field/{migrateForBookshelf.js => migrate-for-bookshelf.js} (56%) rename packages/strapi-plugin-i18n/config/functions/migrations/field/{migrateForMongoose.js => migrate-for-mongoose.js} (93%) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js index f9f3251338..1052803fe1 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js @@ -2,8 +2,8 @@ const { difference, intersection } = require('lodash/fp'); const { getService } = require('../../../../utils'); -const migrateForMongoose = require('./migrateForMongoose'); -const migrateForBookshelf = require('./migrateForBookshelf'); +const migrateForMongoose = require('./migrate-for-mongoose'); +const migrateForBookshelf = require('./migrate-for-bookshelf'); // Migration when i18n is disabled on a field of a content-type that have i18n enabled const after = async ({ model, definition, previousDefinition, ORM }) => { diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js similarity index 56% rename from packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js rename to packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js index d6296c7243..a33fc5ef46 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForBookshelf.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js @@ -60,10 +60,10 @@ const createTmpTable = async ({ ORM, attributesToMigrate, model }) => { const deleteTmpTable = ({ ORM }) => ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); -const getSortedLocales = async ORM => { +const getSortedLocales = async trx => { let defaultLocale; try { - const defaultLocaleRow = await ORM.knex('core_store') + const defaultLocaleRow = await trx('core_store') .select('value') .where({ key: 'plugin_i18n_default_locale' }); defaultLocale = defaultLocaleRow[0].value; @@ -73,7 +73,7 @@ const getSortedLocales = async ORM => { let locales; try { - locales = await ORM.knex(strapi.plugins.i18n.models.locale.collectionName).select('code'); + locales = await trx(strapi.plugins.i18n.models.locale.collectionName).select('code'); } catch (e) { throw new Error('Could not migrate because no locale exist'); } @@ -82,100 +82,87 @@ const getSortedLocales = async ORM => { return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first }; -const migrateForBookshelf = async ({ ORM, model, attributesToMigrate }) => { - const locales = await getSortedLocales(ORM); +const processEntriesWith = async (processFn, { trx, model, attributesToMigrate }) => { + const locales = await getSortedLocales(trx); const localizationAssoc = model.associations.find(a => a.alias === 'localizations'); const localizationTableName = localizationAssoc.tableCollectionName; - // The migration is custom for pg and mysql for better perfomance - const isPgOrMysql = ['pg', 'mysql'].includes(model.client); - - if (isPgOrMysql) { - await createTmpTable({ ORM, attributesToMigrate, model }); - } - - const trx = await ORM.knex.transaction(); - const locsAttr = model.attributes.localizations; const foreignKey = `${singular(model.collectionName)}_${model.primaryKey}`; const relatedKey = `${locsAttr.attribute}_${locsAttr.column}`; - try { - const processedLocaleCodes = []; - for (const locale of locales) { - let offset = 0; - // eslint-disable-next-line no-constant-condition - while (true) { - let batch = await trx - .select([ - ...attributesToMigrate.map(attr => `model.${attr}`), - 'model.id as __strapi_rootId', - 'relModel.id as id', - 'relModel.locale', - ]) - .join( - `${localizationTableName} as loc`, - `model.${model.primaryKey}`, - '=', - `loc.${foreignKey}` - ) - .join( - `${model.collectionName} as relModel`, - `loc.${relatedKey}`, - '=', - `relModel.${model.primaryKey}` - ) - .from( - trx - .select('*') - .from(`${model.collectionName} as subModel`) - .orderBy(`subModel.${model.primaryKey}`) - .where(`subModel.locale`, locale.code) - .offset(offset) - .limit(BATCH_SIZE) - .as('model') - ); - let entries = batch.reduce((entries, entry) => { - if (has(entry.__strapi_rootId, entries)) { - entries[entry.__strapi_rootId].localizations.push(pick(['id', 'locale'], entry)); - } else { - entries[entry.__strapi_rootId] = omit(['id', 'locale', '__strapi_rootId'], entry); - entries[entry.__strapi_rootId].localizations = [pick(['id', 'locale'], entry)]; - } - - return entries; - }, {}); - entries = Object.values(entries); - - offset += BATCH_SIZE; - - const entriesToProcess = entries.filter(shouldBeProcessed(processedLocaleCodes)); - const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); - - if (isPgOrMysql) { - await batchInsertInTmpTable(updatesInfo, trx); + const processedLocaleCodes = []; + for (const locale of locales) { + let offset = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + let batch = await trx + .select([ + ...attributesToMigrate.map(attr => `model.${attr}`), + 'model.id as __strapi_rootId', + 'relModel.id as id', + 'relModel.locale', + ]) + .join( + `${localizationTableName} as loc`, + `model.${model.primaryKey}`, + '=', + `loc.${foreignKey}` + ) + .join( + `${model.collectionName} as relModel`, + `loc.${relatedKey}`, + '=', + `relModel.${model.primaryKey}` + ) + .from( + trx + .select('*') + .from(`${model.collectionName} as subModel`) + .orderBy(`subModel.${model.primaryKey}`) + .where(`subModel.locale`, locale.code) + .offset(offset) + .limit(BATCH_SIZE) + .as('model') + ); + let entries = batch.reduce((entries, entry) => { + if (has(entry.__strapi_rootId, entries)) { + entries[entry.__strapi_rootId].localizations.push(pick(['id', 'locale'], entry)); } else { - await batchUpdate(updatesInfo, trx, model); + entries[entry.__strapi_rootId] = omit(['id', 'locale', '__strapi_rootId'], entry); + entries[entry.__strapi_rootId].localizations = [pick(['id', 'locale'], entry)]; } - if (entries.length < BATCH_SIZE) { - break; - } + return entries; + }, {}); + entries = Object.values(entries); + + offset += BATCH_SIZE; + + const entriesToProcess = entries.filter(shouldBeProcessed(processedLocaleCodes)); + const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); + + await processFn(updatesInfo, trx, model); + + if (entries.length < BATCH_SIZE) { + break; } - processedLocaleCodes.push(locale.code); } + processedLocaleCodes.push(locale.code); + } +}; - if (isPgOrMysql) { +const migrateForBookshelf = async ({ ORM, model, attributesToMigrate }) => { + if (['pg', 'mysql'].includes(model.client)) { + await createTmpTable({ ORM, attributesToMigrate, model }); + await ORM.knex.transaction(async trx => { + await processEntriesWith(batchInsertInTmpTable, { ORM, trx, model, attributesToMigrate }); await updateFromTmpTable({ model, trx, attributesToMigrate }); - } - - await trx.commit(); - - if (isPgOrMysql) { - await deleteTmpTable({ ORM }); - } - } catch (e) { - await trx.rollback(); - throw e; + }); + await deleteTmpTable({ ORM }); + } else { + await ORM.knex.transaction(async trx => { + await processEntriesWith(batchUpdate, { ORM, trx, model, attributesToMigrate }); + }); } }; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js similarity index 93% rename from packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js rename to packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js index 5a7e0715df..450bee2fe1 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrateForMongoose.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js @@ -32,9 +32,9 @@ const migrateForMongoose = async ({ model, attributesToMigrate }) => { const locales = await getSortedLocales(); const processedLocaleCodes = []; for (const locale of locales) { - let batchCount = BATCH_SIZE; let lastId; - while (batchCount === BATCH_SIZE) { + // eslint-disable-next-line no-constant-condition + while (true) { const findParams = { locale: locale.code }; if (lastId) { findParams._id = { $gt: lastId }; @@ -49,8 +49,6 @@ const migrateForMongoose = async ({ model, attributesToMigrate }) => { if (batch.length > 0) { lastId = batch[batch.length - 1]._id; } - batchCount = batch.length; - const entriesToProcess = batch.filter(shouldBeProcessed(processedLocaleCodes)); const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate }); @@ -59,6 +57,10 @@ const migrateForMongoose = async ({ model, attributesToMigrate }) => { })); await model.bulkWrite(updates); + + if (batch.length < BATCH_SIZE) { + break; + } } processedLocaleCodes.push(locale.code); } From bd823964aa9b772ac753e0eb6ed3e1a625f20dd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Thu, 18 Mar 2021 18:53:29 +0100 Subject: [PATCH 11/13] fix rebase --- .../migrations/__tests__/field.test.js | 24 +++++++++---------- .../functions/migrations/field/index.js | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js index 4acc432ba2..2fa1d029d9 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js @@ -31,7 +31,7 @@ describe('i18n - Migration - disable localization on a field', () => { test("Doesn't migrate if no attribute changed (without i18n)", async () => { const find = jest.fn(); - const getLocalizedFields = jest + const getLocalizedAttributes = jest .fn() .mockReturnValueOnce([]) .mockReturnValueOnce([]); @@ -45,7 +45,7 @@ describe('i18n - Migration - disable localization on a field', () => { services: { 'content-types': { isLocalized: () => true, - getLocalizedFields, + getLocalizedAttributes, }, }, }, @@ -56,13 +56,13 @@ describe('i18n - Migration - disable localization on a field', () => { const previousDefinition = { attributes: { name: {} } }; await after({ model, definition: model, previousDefinition }); - expect(getLocalizedFields).toHaveBeenCalledTimes(2); + expect(getLocalizedAttributes).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); test("Doesn't migrate if no attribute changed (with i18n)", async () => { const find = jest.fn(); - const getLocalizedFields = jest + const getLocalizedAttributes = jest .fn() .mockReturnValueOnce(['name']) .mockReturnValueOnce(['name']); @@ -75,7 +75,7 @@ describe('i18n - Migration - disable localization on a field', () => { services: { 'content-types': { isLocalized: () => true, - getLocalizedFields, + getLocalizedAttributes, }, }, }, @@ -86,13 +86,13 @@ describe('i18n - Migration - disable localization on a field', () => { const previousDefinition = { attributes: { name: {} } }; await after({ model, definition: model, previousDefinition }); - expect(getLocalizedFields).toHaveBeenCalledTimes(2); + expect(getLocalizedAttributes).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); test("Doesn't migrate if field become localized", async () => { const find = jest.fn(); - const getLocalizedFields = jest + const getLocalizedAttributes = jest .fn() .mockReturnValueOnce(['name']) .mockReturnValueOnce([]); @@ -106,7 +106,7 @@ describe('i18n - Migration - disable localization on a field', () => { services: { 'content-types': { isLocalized: () => true, - getLocalizedFields, + getLocalizedAttributes, }, }, }, @@ -117,13 +117,13 @@ describe('i18n - Migration - disable localization on a field', () => { const previousDefinition = { attributes: { name: {} } }; await after({ model, definition: model, previousDefinition }); - expect(getLocalizedFields).toHaveBeenCalledTimes(2); + expect(getLocalizedAttributes).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); test("Doesn't migrate if field is deleted", async () => { const find = jest.fn(); - const getLocalizedFields = jest + const getLocalizedAttributes = jest .fn() .mockReturnValueOnce([]) .mockReturnValueOnce(['name']); @@ -137,7 +137,7 @@ describe('i18n - Migration - disable localization on a field', () => { services: { 'content-types': { isLocalized: () => true, - getLocalizedFields, + getLocalizedAttributes, }, }, }, @@ -148,7 +148,7 @@ describe('i18n - Migration - disable localization on a field', () => { const previousDefinition = { attributes: { name: {} } }; await after({ model, definition: model, previousDefinition }); - expect(getLocalizedFields).toHaveBeenCalledTimes(2); + expect(getLocalizedAttributes).toHaveBeenCalledTimes(2); expect(find).not.toHaveBeenCalled(); }); }); diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js index 1052803fe1..7f43dc1b43 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/index.js @@ -13,8 +13,8 @@ const after = async ({ model, definition, previousDefinition, ORM }) => { return; } - const localizedAttributes = ctService.getLocalizedFields(definition); - const prevLocalizedAttributes = ctService.getLocalizedFields(previousDefinition); + const localizedAttributes = ctService.getLocalizedAttributes(definition); + const prevLocalizedAttributes = ctService.getLocalizedAttributes(previousDefinition); const attributesDisabled = difference(prevLocalizedAttributes, localizedAttributes); const attributesToMigrate = intersection(Object.keys(definition.attributes), attributesDisabled); From 113f2b093b24a941ba905ff49ab1ad69f25a8574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Fri, 19 Mar 2021 10:21:56 +0100 Subject: [PATCH 12/13] add comment --- .../config/functions/migrations/field/migrate-for-bookshelf.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js index a33fc5ef46..796e029737 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js @@ -153,6 +153,7 @@ const processEntriesWith = async (processFn, { trx, model, attributesToMigrate } const migrateForBookshelf = async ({ ORM, model, attributesToMigrate }) => { if (['pg', 'mysql'].includes(model.client)) { + // create table outside of the transaction because mysql doesn't accept the creation inside await createTmpTable({ ORM, attributesToMigrate, model }); await ORM.knex.transaction(async trx => { await processEntriesWith(batchInsertInTmpTable, { ORM, trx, model, attributesToMigrate }); From 9d50d97706cbd4f06ef3c05e1de3d67b77dd26cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre=20No=C3=ABl?= Date: Fri, 19 Mar 2021 12:12:51 +0100 Subject: [PATCH 13/13] refacto --- .../functions/migrations/__tests__/field.test.js | 10 ++-------- .../migrations/field/migrate-for-bookshelf.js | 7 ++++--- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js index 2fa1d029d9..43c046d7cb 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/__tests__/field.test.js @@ -31,10 +31,7 @@ describe('i18n - Migration - disable localization on a field', () => { test("Doesn't migrate if no attribute changed (without i18n)", async () => { const find = jest.fn(); - const getLocalizedAttributes = jest - .fn() - .mockReturnValueOnce([]) - .mockReturnValueOnce([]); + const getLocalizedAttributes = jest.fn(() => []); global.strapi = { query: () => { @@ -62,10 +59,7 @@ describe('i18n - Migration - disable localization on a field', () => { test("Doesn't migrate if no attribute changed (with i18n)", async () => { const find = jest.fn(); - const getLocalizedAttributes = jest - .fn() - .mockReturnValueOnce(['name']) - .mockReturnValueOnce(['name']); + const getLocalizedAttributes = jest.fn(() => ['name']); global.strapi = { query: () => { find; diff --git a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js index 796e029737..13f403c407 100644 --- a/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js +++ b/packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js @@ -48,7 +48,7 @@ const updateFromTmpTable = async ({ model, trx, attributesToMigrate }) => { const createTmpTable = async ({ ORM, attributesToMigrate, model }) => { const columnsToCopy = ['id', ...attributesToMigrate]; - await ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME); + await deleteTmpTable({ ORM }); await ORM.knex.raw(`CREATE TABLE ?? AS ??`, [ TMP_TABLE_NAME, ORM.knex @@ -65,8 +65,9 @@ const getSortedLocales = async trx => { try { const defaultLocaleRow = await trx('core_store') .select('value') - .where({ key: 'plugin_i18n_default_locale' }); - defaultLocale = defaultLocaleRow[0].value; + .where({ key: 'plugin_i18n_default_locale' }) + .first(); + defaultLocale = JSON.parse(defaultLocaleRow.value); } catch (e) { throw new Error("Could not migrate because the default locale doesn't exist"); }