From ecfbe42ae2997d12a79bb7c32fd1867394787e35 Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 21 Dec 2022 09:49:47 +0100 Subject: [PATCH] Update the import's ID mapping table with new components ID --- .../strategies/restore/entities.ts | 68 +++++++++++++++++- .../local-strapi-source-provider/entities.ts | 61 ++-------------- .../lib/providers/shared/strapi/entity.ts | 70 ++++++++++++++++++- packages/core/data-transfer/package.json | 1 + 4 files changed, 139 insertions(+), 61 deletions(-) diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts index 7b6a4a929c..4c75cd879a 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts @@ -1,4 +1,9 @@ import type { SchemaUID } from '@strapi/strapi/lib/types/utils'; + +// eslint-disable-next-line @typescript-eslint/ban-ts-comment +// @ts-ignore +import { traverseEntity } from '@strapi/utils'; +import { castArray, get } from 'lodash/fp'; import { Writable } from 'stream'; import type { IEntity } from '../../../../../types'; @@ -9,6 +14,8 @@ interface IEntitiesRestoreStreamOptions { updateMappingTable(type: T, oldID: number, newID: number): void; } +type EntityIDMap = { [path: string]: { type: string; old?: number; new?: number } }; + const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { const { strapi, updateMappingTable } = options; const query = shared.strapi.entity.createEntityQuery(strapi); @@ -18,11 +25,40 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { async write(entity: IEntity, _encoding, callback) { const { type, id, data } = entity; + const { create, getDeepPopulateComponentLikeQuery } = query(type); + const contentType = strapi.getModel(type); + + const ids: EntityIDMap = { + // Register current entity ID + id: { type, old: id }, + }; + + const extractOldIDs = extractEntityIDs(ids, 'old'); + const extractNewIDs = extractEntityIDs(ids, 'new'); try { - const created = await query(type).create({ data }); + // Register old IDs + await traverseEntity(extractOldIDs, { schema: contentType }, data); - updateMappingTable(type, id, created.id); + // Create the entity + const created = await create({ + data, + populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }), + select: 'id', + }); + + // Register new IDs + ids.id.new = parseInt(created.id, 10); + await traverseEntity(extractNewIDs, { schema: contentType }, created); + + // Save old/new IDs in the mapping table + for (const idMap of Object.values(ids)) { + const { new: newID, old: oldID } = idMap; + + if (oldID && newID) { + updateMappingTable(idMap.type, oldID, newID); + } + } } catch (e) { if (e instanceof Error) { return callback(e); @@ -36,4 +72,32 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { }); }; +const extractEntityIDs = (ids: EntityIDMap, property: 'old' | 'new') => (opts: any) => { + const { path, attribute, value } = opts; + + const extract = (type: string, id: string) => { + const parsedID = parseInt(id, 10); + + if (!(path in ids)) { + Object.assign(ids, { [path]: { type } }); + } + + Object.assign(ids[path], { [property]: parsedID }); + }; + + if (attribute.type === 'component') { + const { component } = attribute; + + castArray(value) + .map(get('id')) + .forEach((componentID: string) => extract(component, componentID)); + } + + if (attribute.type === 'dynamiczone') { + value.forEach((item: { __component: string; id: string }) => + extract(item.__component, item.id) + ); + } +}; + export { createEntitiesWriteStream }; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts index 215efa628f..8f45069d09 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts @@ -1,7 +1,7 @@ import type { ContentTypeSchema } from '@strapi/strapi'; -import { isObject, isArray, isEmpty, size } from 'lodash/fp'; import { Readable, PassThrough } from 'stream'; +import * as shared from '../shared/strapi'; import { IEntity } from '../../../types'; /** @@ -12,13 +12,15 @@ export const createEntitiesStream = (strapi: Strapi.Strapi): Readable => { async function* contentTypeStreamGenerator() { for (const contentType of contentTypes) { + const query = shared.entity.createEntityQuery(strapi).call(null, contentType.uid); + const stream: Readable = strapi.db // Create a query builder instance (default type is 'select') .queryBuilder(contentType.uid) // Fetch all columns .select('*') // Apply the populate - .populate(getPopulateAttributes(strapi, contentType)) + .populate(query.deepPopulateComponentLikeQuery) // Get a readable stream .stream(); @@ -61,58 +63,3 @@ export const createEntitiesTransformStream = (): PassThrough => { }, }); }; - -/** - * Get the list of attributes that needs to be populated for the entities streaming - */ -const getPopulateAttributes = (strapi: Strapi.Strapi, contentType: ContentTypeSchema) => { - const { attributes } = contentType; - - const populate: any = {}; - - const entries: [string, any][] = Object.entries(attributes); - - for (const [key, attribute] of entries) { - if (attribute.type === 'component') { - const component = strapi.getModel(attribute.component); - const subPopulate = getPopulateAttributes(strapi, component); - - if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) { - populate[key] = { populate: subPopulate }; - } - - if (isArray(subPopulate) && isEmpty(subPopulate)) { - populate[key] = []; - } - } - - if (attribute.type === 'dynamiczone') { - const { components: componentsUID } = attribute; - - const on: any = {}; - - for (const componentUID of componentsUID) { - const component = strapi.getModel(componentUID); - const subPopulate = getPopulateAttributes(strapi, component); - - if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) { - on[componentUID] = { populate: subPopulate }; - } - - if (isArray(subPopulate) && isEmpty(subPopulate)) { - on[componentUID] = []; - } - } - - populate[key] = size(on) > 0 ? { on } : true; - } - } - - const values = Object.values(populate); - - if (values.every((value) => value === true)) { - return Object.keys(populate); - } - - return populate; -}; diff --git a/packages/core/data-transfer/lib/providers/shared/strapi/entity.ts b/packages/core/data-transfer/lib/providers/shared/strapi/entity.ts index a8a4bdd581..82003d0e08 100644 --- a/packages/core/data-transfer/lib/providers/shared/strapi/entity.ts +++ b/packages/core/data-transfer/lib/providers/shared/strapi/entity.ts @@ -3,7 +3,7 @@ import type { ContentTypeSchema } from '@strapi/strapi'; // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore import * as componentsService from '@strapi/strapi/lib/services/entity-service/components'; -import { assign, map, omit } from 'lodash/fp'; +import { assign, isArray, isEmpty, isObject, map, omit, size } from 'lodash/fp'; const sanitizeComponentLikeAttributes = (model: ContentTypeSchema, data: T) => { const { attributes } = model; @@ -74,7 +74,73 @@ const createEntityQuery = (strapi: Strapi.Strapi) => { return deletedEntities; }; - return { create, createMany, deleteMany }; + const getDeepPopulateComponentLikeQuery = ( + contentType: ContentTypeSchema, + params = { select: '*' } + ) => { + const { attributes } = contentType; + + const populate: any = {}; + + const entries: [string, any][] = Object.entries(attributes); + + for (const [key, attribute] of entries) { + if (attribute.type === 'component') { + const component = strapi.getModel(attribute.component); + const subPopulate = getDeepPopulateComponentLikeQuery(component, params); + + if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) { + populate[key] = { ...params, populate: subPopulate }; + } + + if (isArray(subPopulate) && isEmpty(subPopulate)) { + populate[key] = { ...params }; + } + } + + if (attribute.type === 'dynamiczone') { + const { components: componentsUID } = attribute; + + const on: any = {}; + + for (const componentUID of componentsUID) { + const component = strapi.getModel(componentUID); + const subPopulate = getDeepPopulateComponentLikeQuery(component, params); + + if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) { + on[componentUID] = { ...params, populate: subPopulate }; + } + + if (isArray(subPopulate) && isEmpty(subPopulate)) { + on[componentUID] = { ...params }; + } + } + + populate[key] = size(on) > 0 ? { on } : true; + } + } + + const values = Object.values(populate); + + if (values.every((value) => value === true)) { + return Object.keys(populate); + } + + return populate; + }; + + return { + create, + createMany, + deleteMany, + getDeepPopulateComponentLikeQuery, + + get deepPopulateComponentLikeQuery() { + const contentType = strapi.getModel(uid); + + return getDeepPopulateComponentLikeQuery(contentType); + }, + }; }; return query; diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index 0c4f3e093e..8cf5e9cc59 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -39,6 +39,7 @@ "dependencies": { "@strapi/logger": "4.5.4", "@strapi/strapi": "4.5.4", + "@strapi/utils": "4.5.4", "chalk": "4.1.2", "fs-extra": "10.0.0", "lodash": "4.17.21",