Update the import's ID mapping table with new components ID

packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts

@@ -1,4 +1,9 @@
 import type { SchemaUID } from '@strapi/strapi/lib/types/utils';
+
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-ignore
+import { traverseEntity } from '@strapi/utils';
+import { castArray, get } from 'lodash/fp';
 import { Writable } from 'stream';
 
 import type { IEntity } from '../../../../../types';
@@ -9,6 +14,8 @@ interface IEntitiesRestoreStreamOptions {
   updateMappingTable<T extends SchemaUID | string>(type: T, oldID: number, newID: number): void;
 }
 
+type EntityIDMap = { [path: string]: { type: string; old?: number; new?: number } };
+
 const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
   const { strapi, updateMappingTable } = options;
   const query = shared.strapi.entity.createEntityQuery(strapi);
@@ -18,11 +25,40 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
 
     async write(entity: IEntity, _encoding, callback) {
       const { type, id, data } = entity;
+      const { create, getDeepPopulateComponentLikeQuery } = query(type);
+      const contentType = strapi.getModel(type);
+
+      const ids: EntityIDMap = {
+        // Register current entity ID
+        id: { type, old: id },
+      };
+
+      const extractOldIDs = extractEntityIDs(ids, 'old');
+      const extractNewIDs = extractEntityIDs(ids, 'new');
 
       try {
-        const created = await query(type).create({ data });
+        // Register old IDs
+        await traverseEntity(extractOldIDs, { schema: contentType }, data);
 
-        updateMappingTable(type, id, created.id);
+        // Create the entity
+        const created = await create({
+          data,
+          populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }),
+          select: 'id',
+        });
+
+        // Register new IDs
+        ids.id.new = parseInt(created.id, 10);
+        await traverseEntity(extractNewIDs, { schema: contentType }, created);
+
+        // Save old/new IDs in the mapping table
+        for (const idMap of Object.values(ids)) {
+          const { new: newID, old: oldID } = idMap;
+
+          if (oldID && newID) {
+            updateMappingTable(idMap.type, oldID, newID);
+          }
+        }
       } catch (e) {
         if (e instanceof Error) {
           return callback(e);
@@ -36,4 +72,32 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
   });
 };
 
+const extractEntityIDs = (ids: EntityIDMap, property: 'old' | 'new') => (opts: any) => {
+  const { path, attribute, value } = opts;
+
+  const extract = (type: string, id: string) => {
+    const parsedID = parseInt(id, 10);
+
+    if (!(path in ids)) {
+      Object.assign(ids, { [path]: { type } });
+    }
+
+    Object.assign(ids[path], { [property]: parsedID });
+  };
+
+  if (attribute.type === 'component') {
+    const { component } = attribute;
+
+    castArray(value)
+      .map(get('id'))
+      .forEach((componentID: string) => extract(component, componentID));
+  }
+
+  if (attribute.type === 'dynamiczone') {
+    value.forEach((item: { __component: string; id: string }) =>
+      extract(item.__component, item.id)
+    );
+  }
+};
+
 export { createEntitiesWriteStream };

packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts

@@ -1,7 +1,7 @@
 import type { ContentTypeSchema } from '@strapi/strapi';
 
-import { isObject, isArray, isEmpty, size } from 'lodash/fp';
 import { Readable, PassThrough } from 'stream';
+import * as shared from '../shared/strapi';
 import { IEntity } from '../../../types';
 
 /**
@@ -12,13 +12,15 @@ export const createEntitiesStream = (strapi: Strapi.Strapi): Readable => {
 
   async function* contentTypeStreamGenerator() {
     for (const contentType of contentTypes) {
+      const query = shared.entity.createEntityQuery(strapi).call(null, contentType.uid);
+
       const stream: Readable = strapi.db
         // Create a query builder instance (default type is 'select')
         .queryBuilder(contentType.uid)
         // Fetch all columns
         .select('*')
         // Apply the populate
-        .populate(getPopulateAttributes(strapi, contentType))
+        .populate(query.deepPopulateComponentLikeQuery)
         // Get a readable stream
         .stream();
 
@@ -61,58 +63,3 @@ export const createEntitiesTransformStream = (): PassThrough => {
     },
   });
 };
-
-/**
- * Get the list of attributes that needs to be populated for the entities streaming
- */
-const getPopulateAttributes = (strapi: Strapi.Strapi, contentType: ContentTypeSchema) => {
-  const { attributes } = contentType;
-
-  const populate: any = {};
-
-  const entries: [string, any][] = Object.entries(attributes);
-
-  for (const [key, attribute] of entries) {
-    if (attribute.type === 'component') {
-      const component = strapi.getModel(attribute.component);
-      const subPopulate = getPopulateAttributes(strapi, component);
-
-      if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) {
-        populate[key] = { populate: subPopulate };
-      }
-
-      if (isArray(subPopulate) && isEmpty(subPopulate)) {
-        populate[key] = [];
-      }
-    }
-
-    if (attribute.type === 'dynamiczone') {
-      const { components: componentsUID } = attribute;
-
-      const on: any = {};
-
-      for (const componentUID of componentsUID) {
-        const component = strapi.getModel(componentUID);
-        const subPopulate = getPopulateAttributes(strapi, component);
-
-        if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) {
-          on[componentUID] = { populate: subPopulate };
-        }
-
-        if (isArray(subPopulate) && isEmpty(subPopulate)) {
-          on[componentUID] = [];
-        }
-      }
-
-      populate[key] = size(on) > 0 ? { on } : true;
-    }
-  }
-
-  const values = Object.values(populate);
-
-  if (values.every((value) => value === true)) {
-    return Object.keys(populate);
-  }
-
-  return populate;
-};

packages/core/data-transfer/lib/providers/shared/strapi/entity.ts

@@ -3,7 +3,7 @@ import type { ContentTypeSchema } from '@strapi/strapi';
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import * as componentsService from '@strapi/strapi/lib/services/entity-service/components';
-import { assign, map, omit } from 'lodash/fp';
+import { assign, isArray, isEmpty, isObject, map, omit, size } from 'lodash/fp';
 
 const sanitizeComponentLikeAttributes = <T extends object>(model: ContentTypeSchema, data: T) => {
   const { attributes } = model;
@@ -74,7 +74,73 @@ const createEntityQuery = (strapi: Strapi.Strapi) => {
       return deletedEntities;
     };
 
-    return { create, createMany, deleteMany };
+    const getDeepPopulateComponentLikeQuery = (
+      contentType: ContentTypeSchema,
+      params = { select: '*' }
+    ) => {
+      const { attributes } = contentType;
+
+      const populate: any = {};
+
+      const entries: [string, any][] = Object.entries(attributes);
+
+      for (const [key, attribute] of entries) {
+        if (attribute.type === 'component') {
+          const component = strapi.getModel(attribute.component);
+          const subPopulate = getDeepPopulateComponentLikeQuery(component, params);
+
+          if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) {
+            populate[key] = { ...params, populate: subPopulate };
+          }
+
+          if (isArray(subPopulate) && isEmpty(subPopulate)) {
+            populate[key] = { ...params };
+          }
+        }
+
+        if (attribute.type === 'dynamiczone') {
+          const { components: componentsUID } = attribute;
+
+          const on: any = {};
+
+          for (const componentUID of componentsUID) {
+            const component = strapi.getModel(componentUID);
+            const subPopulate = getDeepPopulateComponentLikeQuery(component, params);
+
+            if ((isArray(subPopulate) || isObject(subPopulate)) && size(subPopulate) > 0) {
+              on[componentUID] = { ...params, populate: subPopulate };
+            }
+
+            if (isArray(subPopulate) && isEmpty(subPopulate)) {
+              on[componentUID] = { ...params };
+            }
+          }
+
+          populate[key] = size(on) > 0 ? { on } : true;
+        }
+      }
+
+      const values = Object.values(populate);
+
+      if (values.every((value) => value === true)) {
+        return Object.keys(populate);
+      }
+
+      return populate;
+    };
+
+    return {
+      create,
+      createMany,
+      deleteMany,
+      getDeepPopulateComponentLikeQuery,
+
+      get deepPopulateComponentLikeQuery() {
+        const contentType = strapi.getModel(uid);
+
+        return getDeepPopulateComponentLikeQuery(contentType);
+      },
+    };
   };
 
   return query;

packages/core/data-transfer/package.json

@@ -39,6 +39,7 @@
   "dependencies": {
     "@strapi/logger": "4.5.4",
     "@strapi/strapi": "4.5.4",
+    "@strapi/utils": "4.5.4",
     "chalk": "4.1.2",
     "fs-extra": "10.0.0",
     "lodash": "4.17.21",