yujunjun/strapi
1
0
Fork 0
mirror of https://github.com/strapi/strapi.git synced 2025-11-01 02:16:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

handle compo, dz and media migration when disabling i18n on a field

Browse Source
This commit is contained in:
Pierre Noël 2021-04-08 18:05:12 +02:00
parent f42f685a0a
commit c086bcc9d7
10 changed files with 155 additions and 211 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
packages/strapi-connector-bookshelf/lib/queries.js
View File

@ -89,12 +89,12 @@ module.exports = function createQueryBuilder({ model, strapi }) {
/**
* Count entries based on filters
*/
function count(params = {}) {
function count(params = {}, { transacting } = {}) {
const filters = pickCountFilters(convertRestQueryParams(params));
return model
.query(buildQuery({ model, filters }))
.count()
.count({ transacting })
.then(Number);
}

27
packages/strapi-database/lib/database-manager.js
View File

@ -38,10 +38,10 @@ class DatabaseManager {
validateModelSchemas({ strapi: this.strapi, manager: this });
await this.connectors.initialize();
this.initializeModelsMap();
await this.connectors.initialize();
return this;
}
@ -73,12 +73,7 @@ class DatabaseManager {
throw new Error(`argument entity is required`);
}
const normalizedName = entity.toLowerCase();
// get by uid or name / plugin
const model = this.models.has(entity)
? this.models.get(entity)
: this.getModel(normalizedName, plugin);
const model = this.getModel(entity, plugin);
if (!model) {
throw new Error(`The model ${entity} can't be found.`);
@ -101,11 +96,8 @@ class DatabaseManager {
return query;
}
getModel(name, plugin) {
getModelFromStrapi(name, plugin) {
const key = _.toLower(name);
if (this.models.has(key)) return this.models.get(key);
if (plugin === 'admin') {
return _.get(strapi.admin, ['models', key]);
}
@ -117,6 +109,17 @@ class DatabaseManager {
return _.get(strapi, ['models', key]) || _.get(strapi, ['components', key]);
}
getModel(name, plugin) {
const key = _.toLower(name);
if (this.models.has(key)) {
const { modelName, plugin: pluginName } = this.models.get(key);
return this.getModelFromStrapi(modelName, pluginName);
} else {
return this.getModelFromStrapi(key, plugin);
}
}
getModelByAssoc(assoc) {
return this.getModel(assoc.collection || assoc.model, assoc.plugin);
}

10
packages/strapi-database/lib/queries/paginated-queries.js
View File

@ -4,11 +4,10 @@ const _ = require('lodash');
const createPaginatedQuery = ({ fetch, count }) => async (queryParams, ...args) => {
const params = _.omit(queryParams, ['page', 'pageSize']);
const pagination = await getPaginationInfos(queryParams, count);
const pagination = await getPaginationInfos(queryParams, count, ...args);
Object.assign(params, paginationToQueryParams(pagination));
const results = await fetch(params, ...args);
const results = await fetch(params, undefined, ...args);
return { results, pagination };
};
@ -18,11 +17,10 @@ const createSearchPageQuery = ({ search, countSearch }) =>
const createFindPageQuery = ({ find, count }) => createPaginatedQuery({ fetch: find, count });
const getPaginationInfos = async (queryParams, count) => {
const getPaginationInfos = async (queryParams, count, ...args) => {
const { page, pageSize, ...params } = withDefaultPagination(queryParams);
const total = await count(params);
const total = await count(params, ...args);
return {
page,
pageSize,

4
packages/strapi-plugin-i18n/config/functions/migrations/field/__tests__/utils.test.js
View File

@ -44,8 +44,8 @@ describe('i18n - migration utils', () => {
],
];
test.each(testData)('%j', (entriesToProcess, attributesToMigrate, expectedResult) => {
const result = getUpdatesInfo({ entriesToProcess, attributesToMigrate });
test.each(testData)('%j', (entriesToProcess, attrsToMigrate, expectedResult) => {
const result = getUpdatesInfo({ entriesToProcess, attrsToMigrate });
expect(result).toEqual(expectedResult);
});

10
packages/strapi-plugin-i18n/config/functions/migrations/field/index.js
View File

@ -1,6 +1,6 @@
'use strict';
const { difference, intersection } = require('lodash/fp');
const { difference, keys, intersection, isEmpty } = require('lodash/fp');
const { getService } = require('../../../../utils');
const migrateForMongoose = require('./migrate-for-mongoose');
const migrateForBookshelf = require('./migrate-for-bookshelf');
@ -16,16 +16,16 @@ const after = async ({ model, definition, previousDefinition, ORM }) => {
const localizedAttributes = ctService.getLocalizedAttributes(definition);
const prevLocalizedAttributes = ctService.getLocalizedAttributes(previousDefinition);
const attributesDisabled = difference(prevLocalizedAttributes, localizedAttributes);
const attributesToMigrate = intersection(Object.keys(definition.attributes), attributesDisabled);
const attrsToMigrate = intersection(keys(definition.attributes), attributesDisabled);
if (attributesToMigrate.length === 0) {
if (isEmpty(attrsToMigrate)) {
return;
}
if (model.orm === 'bookshelf') {
await migrateForBookshelf({ ORM, model, attributesToMigrate });
await migrateForBookshelf({ ORM, model, attrsToMigrate });
} else if (model.orm === 'mongoose') {
await migrateForMongoose({ model, attributesToMigrate });
await migrateForMongoose({ model, attrsToMigrate });
}
};

142
packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-bookshelf.js
View File

@ -1,14 +1,11 @@
'use strict';
const { singular } = require('pluralize');
const { has, omit, pick, orderBy } = require('lodash/fp');
const { shouldBeProcessed, getUpdatesInfo } = require('./utils');
const BATCH_SIZE = 1000;
const { migrate } = require('./migrate');
const { areScalarAttrsOnly } = require('./utils');
const TMP_TABLE_NAME = '__tmp__i18n_field_migration';
const batchInsertInTmpTable = async (updatesInfo, trx) => {
const batchInsertInTmpTable = async ({ updatesInfo }, { transacting: trx }) => {
const tmpEntries = [];
updatesInfo.forEach(({ entriesIdsToUpdate, attributesValues }) => {
entriesIdsToUpdate.forEach(id => {
@ -18,36 +15,26 @@ const batchInsertInTmpTable = async (updatesInfo, trx) => {
await trx.batchInsert(TMP_TABLE_NAME, tmpEntries, 100);
};
const batchUpdate = async (updatesInfo, trx, model) => {
const promises = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) =>
trx
.from(model.collectionName)
.update(attributesValues)
.whereIn('id', entriesIdsToUpdate)
);
await Promise.all(promises);
};
const updateFromTmpTable = async ({ model, trx, attributesToMigrate }) => {
const updateFromTmpTable = async ({ model, attrsToMigrate }, { transacting: trx }) => {
const collectionName = model.collectionName;
if (model.client === 'pg') {
const substitutes = attributesToMigrate.map(() => '?? = ??.??').join(',');
const substitutes = attrsToMigrate.map(() => '?? = ??.??').join(',');
const bindings = [collectionName];
attributesToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr));
attrsToMigrate.forEach(attr => bindings.push(attr, TMP_TABLE_NAME, attr));
bindings.push(TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME);
await trx.raw(`UPDATE ?? SET ${substitutes} FROM ?? WHERE ??.id = ??.id;`, bindings);
} else if (model.client === 'mysql') {
const substitutes = attributesToMigrate.map(() => '??.?? = ??.??').join(',');
const substitutes = attrsToMigrate.map(() => '??.?? = ??.??').join(',');
const bindings = [collectionName, TMP_TABLE_NAME, collectionName, TMP_TABLE_NAME];
attributesToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr));
attrsToMigrate.forEach(attr => bindings.push(collectionName, attr, TMP_TABLE_NAME, attr));
await trx.raw(`UPDATE ?? JOIN ?? ON ??.id = ??.id SET ${substitutes};`, bindings);
}
};
const createTmpTable = async ({ ORM, attributesToMigrate, model }) => {
const columnsToCopy = ['id', ...attributesToMigrate];
const createTmpTable = async ({ ORM, attrsToMigrate, model }) => {
const columnsToCopy = ['id', ...attrsToMigrate];
await deleteTmpTable({ ORM });
await ORM.knex.raw(`CREATE TABLE ?? AS ??`, [
TMP_TABLE_NAME,
@ -60,110 +47,21 @@ const createTmpTable = async ({ ORM, attributesToMigrate, model }) => {
const deleteTmpTable = ({ ORM }) => ORM.knex.schema.dropTableIfExists(TMP_TABLE_NAME);
const getSortedLocales = async trx => {
let defaultLocale;
try {
const defaultLocaleRow = await trx('core_store')
.select('value')
.where({ key: 'plugin_i18n_default_locale' })
.first();
defaultLocale = JSON.parse(defaultLocaleRow.value);
} catch (e) {
throw new Error("Could not migrate because the default locale doesn't exist");
}
const migrateForBookshelf = async ({ ORM, model, attrsToMigrate }) => {
const onlyScalarAttrs = areScalarAttrsOnly({ model, attributes: attrsToMigrate });
let locales;
try {
locales = await trx(strapi.plugins.i18n.models.locale.collectionName).select('code');
} catch (e) {
throw new Error('Could not migrate because no locale exist');
}
locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale));
return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first
};
const processEntriesWith = async (processFn, { trx, model, attributesToMigrate }) => {
const locales = await getSortedLocales(trx);
const localizationAssoc = model.associations.find(a => a.alias === 'localizations');
const localizationTableName = localizationAssoc.tableCollectionName;
const locsAttr = model.attributes.localizations;
const foreignKey = `${singular(model.collectionName)}_${model.primaryKey}`;
const relatedKey = `${locsAttr.attribute}_${locsAttr.column}`;
const processedLocaleCodes = [];
for (const locale of locales) {
let offset = 0;
// eslint-disable-next-line no-constant-condition
while (true) {
let batch = await trx
.select([
...attributesToMigrate.map(attr => `model.${attr}`),
'model.id as __strapi_rootId',
'relModel.id as id',
'relModel.locale',
])
.join(
`${localizationTableName} as loc`,
`model.${model.primaryKey}`,
'=',
`loc.${foreignKey}`
)
.join(
`${model.collectionName} as relModel`,
`loc.${relatedKey}`,
'=',
`relModel.${model.primaryKey}`
)
.from(
trx
.select('*')
.from(`${model.collectionName} as subModel`)
.orderBy(`subModel.${model.primaryKey}`)
.where(`subModel.locale`, locale.code)
.offset(offset)
.limit(BATCH_SIZE)
.as('model')
);
let entries = batch.reduce((entries, entry) => {
if (has(entry.__strapi_rootId, entries)) {
entries[entry.__strapi_rootId].localizations.push(pick(['id', 'locale'], entry));
} else {
entries[entry.__strapi_rootId] = omit(['id', 'locale', '__strapi_rootId'], entry);
entries[entry.__strapi_rootId].localizations = [pick(['id', 'locale'], entry)];
}
return entries;
}, {});
entries = Object.values(entries);
offset += BATCH_SIZE;
const entriesToProcess = entries.filter(shouldBeProcessed(processedLocaleCodes));
const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate });
await processFn(updatesInfo, trx, model);
if (entries.length < BATCH_SIZE) {
break;
}
}
processedLocaleCodes.push(locale.code);
}
};
const migrateForBookshelf = async ({ ORM, model, attributesToMigrate }) => {
if (['pg', 'mysql'].includes(model.client)) {
// optimize migration for pg and mysql when there are only scalar attributes to migrate
if (onlyScalarAttrs && ['pg', 'mysql'].includes(model.client)) {
// create table outside of the transaction because mysql doesn't accept the creation inside
await createTmpTable({ ORM, attributesToMigrate, model });
await ORM.knex.transaction(async trx => {
await processEntriesWith(batchInsertInTmpTable, { ORM, trx, model, attributesToMigrate });
await updateFromTmpTable({ model, trx, attributesToMigrate });
await createTmpTable({ ORM, attrsToMigrate, model });
await ORM.knex.transaction(async transacting => {
await migrate({ model, attrsToMigrate }, { migrateFn: batchInsertInTmpTable, transacting });
await updateFromTmpTable({ model, attrsToMigrate }, { transacting });
});
await deleteTmpTable({ ORM });
} else {
await ORM.knex.transaction(async trx => {
await processEntriesWith(batchUpdate, { ORM, trx, model, attributesToMigrate });
await ORM.knex.transaction(async transacting => {
await migrate({ model, attrsToMigrate }, { transacting });
});
}
};

71
packages/strapi-plugin-i18n/config/functions/migrations/field/migrate-for-mongoose.js
View File

@ -1,68 +1,23 @@
'use strict';
const { orderBy } = require('lodash/fp');
const { shouldBeProcessed, getUpdatesInfo } = require('./utils');
const { migrate } = require('./migrate');
const { areScalarAttrsOnly } = require('./utils');
const BATCH_SIZE = 1000;
const batchUpdate = async ({ updatesInfo, model }) => {
const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({
updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues },
}));
const getSortedLocales = async () => {
let defaultLocale;
try {
const defaultLocaleRow = await strapi.models['core_store'].findOne({
key: 'plugin_i18n_default_locale',
});
defaultLocale = JSON.parse(defaultLocaleRow.value);
} catch (e) {
throw new Error("Could not migrate because the default locale doesn't exist");
}
let locales;
try {
strapi.models;
locales = await strapi.plugins.i18n.models.locale.find();
} catch (e) {
throw new Error('Could not migrate because no locale exist');
}
locales.forEach(locale => (locale.isDefault = locale.code === defaultLocale));
return orderBy(['isDefault', 'code'], ['desc', 'asc'])(locales); // Put default locale first
await model.bulkWrite(updates);
};
const migrateForMongoose = async ({ model, attributesToMigrate }) => {
const locales = await getSortedLocales();
const processedLocaleCodes = [];
for (const locale of locales) {
let lastId;
// eslint-disable-next-line no-constant-condition
while (true) {
const findParams = { locale: locale.code };
if (lastId) {
findParams._id = { $gt: lastId };
}
const migrateForMongoose = async ({ model, attrsToMigrate }) => {
const onlyScalarAttrs = areScalarAttrsOnly({ model, attributes: attrsToMigrate });
const batch = await model
.find(findParams, [...attributesToMigrate, 'locale', 'localizations'])
.populate('localizations', 'locale id')
.sort({ _id: 1 })
.limit(BATCH_SIZE);
if (batch.length > 0) {
lastId = batch[batch.length - 1]._id;
}
const entriesToProcess = batch.filter(shouldBeProcessed(processedLocaleCodes));
const updatesInfo = getUpdatesInfo({ entriesToProcess, attributesToMigrate });
const updates = updatesInfo.map(({ entriesIdsToUpdate, attributesValues }) => ({
updateMany: { filter: { _id: { $in: entriesIdsToUpdate } }, update: attributesValues },
}));
await model.bulkWrite(updates);
if (batch.length < BATCH_SIZE) {
break;
}
}
processedLocaleCodes.push(locale.code);
if (onlyScalarAttrs) {
await migrate({ model, attrsToMigrate }, { migrateFn: batchUpdate });
} else {
await migrate({ model, attrsToMigrate });
}
};

56
packages/strapi-plugin-i18n/config/functions/migrations/field/migrate.js Normal file
View File

@ -0,0 +1,56 @@
'use strict';
const { pick, prop } = require('lodash/fp');
const { getService } = require('../../../../utils');
const { shouldBeProcessed, getUpdatesInfo, getSortedLocales } = require('./utils');
const BATCH_SIZE = 1000;
const migrateBatch = async (entries, { model, attrsToMigrate }, { transacting }) => {
const { copyNonLocalizedAttributes } = getService('content-types');
const updatePromises = entries.map(entity => {
const updateValues = pick(attrsToMigrate, copyNonLocalizedAttributes(model, entity));
const entriesIdsToUpdate = entity.localizations.map(prop('id'));
console.log('updateValues', JSON.stringify(updateValues, null, 2));
return Promise.all(
entriesIdsToUpdate.map(id =>
strapi.query(model.uid).update({ id }, updateValues, { transacting })
)
);
});
await Promise.all(updatePromises);
};
const migrate = async ({ model, attrsToMigrate }, { migrateFn, transacting } = {}) => {
const locales = await getSortedLocales({ transacting });
const processedLocaleCodes = [];
for (const locale of locales) {
let page = 1;
// eslint-disable-next-line no-constant-condition
while (true) {
const { results } = await strapi
.query(model.uid)
.findPage({ locale, page, pageSize: BATCH_SIZE }, { transacting });
const entriesToProcess = results.filter(shouldBeProcessed(processedLocaleCodes));
if (migrateFn) {
const updatesInfo = getUpdatesInfo({ entriesToProcess, attrsToMigrate });
await migrateFn({ updatesInfo, model }, { transacting });
} else {
await migrateBatch(entriesToProcess, { model, attrsToMigrate }, { transacting });
}
if (results.length < BATCH_SIZE) {
break;
}
page += 1;
}
processedLocaleCodes.push(locale);
}
};
module.exports = {
migrate,
};

39
packages/strapi-plugin-i18n/config/functions/migrations/field/utils.js
View File

@ -1,6 +1,8 @@
'use strict';
const { pick, prop, intersection } = require('lodash/fp');
const { isScalarAttribute } = require('strapi-utils').contentTypes;
const { pick, prop, map, intersection, isEmpty, orderBy, pipe, every } = require('lodash/fp');
const { getService } = require('../../../../utils');
const shouldBeProcessed = processedLocaleCodes => entry => {
return (
@ -9,17 +11,48 @@ const shouldBeProcessed = processedLocaleCodes => entry => {
);
};
const getUpdatesInfo = ({ entriesToProcess, attributesToMigrate }) => {
const getUpdatesInfo = ({ entriesToProcess, attrsToMigrate }) => {
const updates = [];
for (const entry of entriesToProcess) {
const attributesValues = pick(attributesToMigrate, entry);
const attributesValues = pick(attrsToMigrate, entry);
const entriesIdsToUpdate = entry.localizations.map(prop('id'));
updates.push({ entriesIdsToUpdate, attributesValues });
}
return updates;
};
const getSortedLocales = async ({ transacting } = {}) => {
const localeService = getService('locales');
let defaultLocale;
try {
const storeRes = await strapi
.query('core_store')
.findOne({ key: 'plugin_i18n_default_locale' }, null, { transacting });
defaultLocale = JSON.parse(storeRes.value);
} catch (e) {
throw new Error("Could not migrate because the default locale doesn't exist");
}
const locales = await localeService.find({}, null, { transacting });
if (isEmpty(locales)) {
throw new Error('Could not migrate because no locale exist');
}
// Put default locale first
return pipe(
map(locale => ({ code: locale.code, isDefault: locale.code === defaultLocale })),
orderBy(['isDefault', 'code'], ['desc', 'asc']),
map(prop('code'))
)(locales);
};
const areScalarAttrsOnly = ({ model, attributes }) =>
pipe(pick(attributes), every(isScalarAttribute))(model.attributes);
module.exports = {
shouldBeProcessed,
getUpdatesInfo,
getSortedLocales,
areScalarAttrsOnly,
};

3
packages/strapi-plugin-i18n/services/content-types.js
View File

@ -101,8 +101,9 @@ const getNonLocalizedAttributes = model => {
};
const removeId = value => {
if (typeof value === 'object' && has('id', value)) {
if (typeof value === 'object' && (has('id', value) || has('_id', value))) {
delete value.id;
delete value._id;
}
};