data-transfer: Move from lib to src & update folder architecture

packages/core/data-transfer/.gitignore

@@ -0,0 +1 @@
+lib/

packages/core/data-transfer/.npmignore

@@ -1,9 +1,9 @@
 #############################
-# DATA TRANSFER X
+# DATA TRANSFER
 ############################
 
 *.js.map
-lib/
+src/
 types/
 tsconfig.json
 
@@ -19,14 +19,12 @@ Icon
 .Trashes
 ._*
 
-
 ############################
 # Linux
 ############################
 
 *~
 
-
 ############################
 # Windows
 ############################
@@ -40,7 +38,6 @@ $RECYCLE.BIN/
 *.msm
 *.msp
 
-
 ############################
 # Packages
 ############################
@@ -69,7 +66,6 @@ $RECYCLE.BIN/
 *.out
 *.pid
 
-
 ############################
 # Logs and databases
 ############################
@@ -79,7 +75,6 @@ $RECYCLE.BIN/
 *.sql
 *.sqlite
 
-
 ############################
 # Misc.
 ############################
@@ -89,7 +84,6 @@ $RECYCLE.BIN/
 nbproject
 .vscode/
 
-
 ############################
 # Node.js
 ############################
@@ -107,7 +101,6 @@ package-lock.json
 !docs/package-lock.json
 *.heapsnapshot
 
-
 ############################
 # Tests
 ############################

packages/core/data-transfer/lib/bootstrap/controllers/index.ts

@@ -1,2 +0,0 @@
-export * from './push';
-export { default as createTransferController } from './transfer';

packages/core/data-transfer/lib/index.ts

@@ -1,4 +0,0 @@
-export * from './engine';
-export * from './providers';
-
-export { default as register } from './register';

packages/core/data-transfer/lib/providers/__tests__/register.test.ts

@@ -1,32 +0,0 @@
-import { createTransferController } from '../../bootstrap/controllers';
-import register from '../../register';
-
-afterEach(() => {
-  jest.clearAllMocks();
-});
-
-const strapiMock = {
-  admin: {
-    routes: {
-      push: jest.fn(),
-    },
-  },
-};
-
-jest.mock('../../bootstrap/controllers', () => ({
-  createTransferController: jest.fn(),
-}));
-
-describe('Register the Transfer route', () => {
-  test('registers the /transfer route', () => {
-    register(strapiMock);
-    expect(strapiMock.admin.routes.push).toHaveBeenCalledWith({
-      method: 'GET',
-      path: '/transfer',
-      handler: createTransferController(),
-      config: {
-        auth: false,
-      },
-    });
-  });
-});

packages/core/data-transfer/lib/providers/index.ts

@@ -1,9 +0,0 @@
-// source providers
-export * from './local-file-source-provider';
-export * from './local-strapi-source-provider';
-
-// destination providers
-export * from './local-file-destination-provider';
-export * from './local-strapi-destination-provider';
-
-export * from './remote-strapi-destination-provider';

packages/core/data-transfer/lib/providers/shared/index.ts

@@ -1 +0,0 @@
-export * as strapi from './strapi';

packages/core/data-transfer/lib/providers/test-utils/index.ts

@@ -1,83 +0,0 @@
-import { Readable } from 'stream';
-
-/**
- * Collect every entity in a Readable stream
- */
-export const collect = <T = unknown>(stream: Readable): Promise<T[]> => {
-  const chunks: T[] = [];
-
-  return new Promise((resolve, reject) => {
-    stream
-      .on('data', (chunk) => chunks.push(chunk))
-      .on('close', () => resolve(chunks))
-      .on('error', reject);
-  });
-};
-
-/**
- * Create a "Strapi" like object factory based on the
- * given params and cast it to the correct type
- */
-export const getStrapiFactory =
-  <
-    T extends {
-      [key in keyof Partial<Strapi.Strapi>]: unknown;
-    }
-  >(
-    properties?: T
-  ) =>
-  () => {
-    return { ...properties } as Strapi.Strapi;
-  };
-
-/**
- * Union type used to represent the default content types available
- */
-export type ContentType = 'foo' | 'bar';
-
-/**
- * Factory to get default content types test values
- */
-export const getContentTypes = (): {
-  [key in ContentType]: { uid: key; attributes: { [attribute: string]: unknown } };
-} => ({
-  foo: { uid: 'foo', attributes: { title: { type: 'string' } } },
-  bar: { uid: 'bar', attributes: { age: { type: 'number' } } },
-});
-
-/**
- * Create a factory of readable streams (wrapped with a jest mock function)
- */
-export const createMockedReadableFactory = <T extends string = ContentType>(source: {
-  [ct in T]: Array<{ id: number; [key: string]: unknown }>;
-}) =>
-  jest.fn((uid: T) => {
-    return Readable.from(source[uid] || []);
-  });
-
-/**
- * Create a factory of mocked query builders
- */
-export const createMockedQueryBuilder = <T extends string = ContentType>(data: {
-  [key in T]: unknown[];
-}) =>
-  jest.fn((uid: T) => {
-    const state: { [key: string]: unknown } = { populate: undefined };
-
-    return {
-      populate(populate: unknown) {
-        state.populate = populate;
-        return this;
-      },
-      stream() {
-        return Readable.from(data[uid]);
-      },
-    };
-  });
-
-/**
- * Update the global store with the given strapi value
- */
-export const setGlobalStrapi = (strapi: Strapi.Strapi): void => {
-  (global as unknown as Global).strapi = strapi;
-};

packages/core/data-transfer/lib/register.ts

@@ -1,16 +0,0 @@
-import { createTransferController } from './bootstrap/controllers';
-
-const registerTransferRoute = (strapi: any) => {
-  strapi.admin.routes.push({
-    method: 'GET',
-    path: '/transfer',
-    handler: createTransferController(),
-    config: { auth: false },
-  });
-};
-
-const register = (strapi: any) => {
-  registerTransferRoute(strapi);
-};
-
-export default register;

packages/core/data-transfer/lib/utils/schema.ts

@@ -1,20 +0,0 @@
-import type { Schema } from '@strapi/strapi';
-import { mapValues, pick } from 'lodash/fp';
-
-const schemaSelectedKeys = [
-  'collectionName',
-  'info',
-  'options',
-  'pluginOptions',
-  'attributes',
-  'kind',
-  'modelType',
-  'modelName',
-  'uid',
-  'plugin',
-  'globalId',
-];
-
-export const mapSchemasValues = (schemas: Record<string, Schema>) => {
-  return mapValues(pick(schemaSelectedKeys), schemas);
-};

packages/core/data-transfer/lib/utils/stream.ts

@@ -1,48 +0,0 @@
-import { Transform, Readable } from 'stream';
-
-type TransformOptions = ConstructorParameters<typeof Transform>[0];
-
-export const filter = <T>(
-  predicate: (value: T) => boolean | Promise<boolean>,
-  options: TransformOptions = { objectMode: true }
-): Transform => {
-  return new Transform({
-    ...options,
-
-    async transform(chunk, _encoding, callback) {
-      const keep = await predicate(chunk);
-
-      callback(null, keep ? chunk : undefined);
-    },
-  });
-};
-
-export const map = <T>(
-  predicate: (value: T) => T | Promise<T>,
-  options: TransformOptions = { objectMode: true }
-): Transform => {
-  return new Transform({
-    ...options,
-
-    async transform(chunk, _encoding, callback) {
-      const mappedValue = await predicate(chunk);
-
-      callback(null, mappedValue);
-    },
-  });
-};
-
-/**
- * Collect every entity in a Readable stream
- */
-export const collect = <T = unknown>(stream: Readable): Promise<T[]> => {
-  const chunks: T[] = [];
-
-  return new Promise((resolve) => {
-    stream.on('data', (chunk) => chunks.push(chunk));
-    stream.on('end', () => {
-      stream.destroy();
-      resolve(chunks);
-    });
-  });
-};

packages/core/data-transfer/package.json

@@ -24,17 +24,17 @@
       "url": "https://strapi.io"
     }
   ],
-  "main": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "main": "./lib/index.js",
+  "types": "./lib/index.d.ts",
   "scripts": {
     "build": "tsc -p tsconfig.json",
-    "clean": "rimraf ./dist",
+    "clean": "rimraf ./lib",
     "build:clean": "yarn clean && yarn build",
     "watch": "yarn build -w --preserveWatchOutput",
     "test:unit": "jest --verbose"
   },
   "directories": {
-    "lib": "./dist"
+    "lib": "./lib"
   },
   "dependencies": {
     "@strapi/logger": "4.5.4",
@@ -61,7 +61,7 @@
     "@types/tar": "6.1.3",
     "@types/tar-stream": "2.2.2",
     "@types/uuid": "9.0.0",
-    "koa": "2.14.1",
+    "@types/koa": "2.13.1",
     "rimraf": "3.0.2",
     "typescript": "4.6.2"
   },

packages/core/data-transfer/src/__tests__/test-utils.ts

@@ -102,6 +102,13 @@ export const destinationStages = [
   'getSchemasStream',
 ];
 
+/**
+ * Update the global store with the given strapi value
+ */
+export const setGlobalStrapi = (strapi: Strapi.Strapi): void => {
+  (global as unknown as Global).strapi = strapi;
+};
+
 /**
  * Add jest expect helpers
  */

packages/core/data-transfer/src/engine/index.ts

@@ -19,7 +19,7 @@ import type {
 } from '../../types';
 import type { Diff } from '../utils/json';
 
-import compareSchemas from '../strategies';
+import { compareSchemas } from './validation/schemas';
 import { filter, map } from '../utils/stream';
 
 export const TRANSFER_STAGES: ReadonlyArray<TransferStage> = Object.freeze([

packages/core/data-transfer/src/engine/validation/index.ts

@@ -0,0 +1 @@
+export * as schemas from './schemas';

packages/core/data-transfer/src/engine/validation/schemas/index.ts

@@ -1,5 +1,5 @@
-import type { Diff } from '../utils/json';
-import * as utils from '../utils';
+import type { Diff } from '../../../utils/json';
+import * as utils from '../../../utils';
 
 const strategies = {
   // No diffs
@@ -32,4 +32,4 @@ const compareSchemas = <T, P>(a: T, b: P, strategy: keyof typeof strategies) =>
   return strategies[strategy](diffs);
 };
 
-export default compareSchemas;
+export { compareSchemas };

packages/core/data-transfer/src/file/index.ts

@@ -0,0 +1 @@
+export * as providers from './providers';

packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts

@@ -6,7 +6,7 @@ jest.mock('fs');
 import fs from 'fs-extra';
 import { Writable } from 'stream-chain';
 import { createLocalFileDestinationProvider, ILocalFileDestinationProviderOptions } from '..';
-import * as encryption from '../../../encryption/encrypt';
+import * as encryption from '../../../../utils/encryption';
 import { createFilePathFactory, createTarEntryStream } from '../utils';
 
 fs.createWriteStream = jest.fn().mockReturnValue(
@@ -18,14 +18,14 @@ fs.createWriteStream = jest.fn().mockReturnValue(
 
 const filePath = './test-file';
 
-jest.mock('../../../encryption/encrypt', () => {
+jest.mock('../../../../utils/encryption', () => {
   return {
     __esModule: true,
     createEncryptionCipher() {},
   };
 });
 
-jest.mock('../../local-file-destination-provider/utils');
+jest.mock('../utils');
 
 describe('Local File Destination Provider', () => {
   (createFilePathFactory as jest.Mock).mockImplementation(jest.fn());

packages/core/data-transfer/src/file/providers/destination/index.ts

@@ -6,7 +6,7 @@ import { stringer } from 'stream-json/jsonl/Stringer';
 import { chain } from 'stream-chain';
 import { Readable, Writable } from 'stream';
 
-import { createEncryptionCipher } from '../../encryption/encrypt';
+import { createEncryptionCipher } from '../../../utils/encryption';
 import type {
   IAsset,
   IDestinationProvider,
@@ -14,7 +14,7 @@ import type {
   IMetadata,
   ProviderType,
   Stream,
-} from '../../../types';
+} from '../../../../types';
 import { createFilePathFactory, createTarEntryStream } from './utils';
 
 export interface ILocalFileDestinationProviderOptions {

packages/core/data-transfer/src/file/providers/index.ts

@@ -0,0 +1,2 @@
+export * from './source';
+export * from './destination';

packages/core/data-transfer/src/file/providers/source/index.ts

@@ -8,10 +8,9 @@ import { keyBy } from 'lodash/fp';
 import { chain } from 'stream-chain';
 import { pipeline, PassThrough } from 'stream';
 import { parser } from 'stream-json/jsonl/Parser';
-import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../types';
+import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../../types';
 
-import { createDecryptionCipher } from '../../encryption';
-import * as utils from '../../utils';
+import * as utils from '../../../utils';
 
 type StreamItemArray = Parameters<typeof chain>[0];
 
@@ -152,7 +151,7 @@ class LocalFileSourceProvider implements ISourceProvider {
     }
 
     if (encryption.enabled && encryption.key) {
-      streams.push(createDecryptionCipher(encryption.key));
+      streams.push(utils.encryption.createDecryptionCipher(encryption.key));
     }
 
     if (compression.enabled) {
@@ -227,20 +226,22 @@ class LocalFileSourceProvider implements ISourceProvider {
               return entryPath === filePath && entry.type === 'File';
             },
 
-            /**
-             * Whenever an entry passes the filter method, process it
-             */
             async onentry(entry) {
               // Collect all the content of the entry file
               const content = await entry.collect();
-              // Parse from buffer to string to JSON
-              const parsedContent = JSON.parse(content.toString());
 
-              // Resolve the Promise with the parsed content
-              resolve(parsedContent);
+              try {
+                // Parse from buffer to string to JSON
+                const parsedContent = JSON.parse(content.toString());
 
-              // Cleanup (close the stream associated to the entry)
-              entry.destroy();
+                // Resolve the Promise with the parsed content
+                resolve(parsedContent);
+              } catch (e) {
+                reject(e);
+              } finally {
+                // Cleanup (close the stream associated to the entry)
+                entry.destroy();
+              }
             },
           }),
         ],

packages/core/data-transfer/src/index.ts

@@ -0,0 +1,4 @@
+export * as engine from './engine';
+export * as strapi from './strapi';
+export * as file from './file';
+export * as utils from './utils';

packages/core/data-transfer/src/strapi/__tests__/register.test.ts

@@ -0,0 +1,36 @@
+import { getStrapiFactory } from '../../__tests__/test-utils';
+
+import { createTransferHandler } from '../remote/handlers';
+import register from '../register';
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+const strapiMockFactory = getStrapiFactory({
+  admin: {
+    routes: {
+      push: jest.fn(),
+    },
+  },
+});
+
+jest.mock('../remote/handlers', () => ({
+  createTransferHandler: jest.fn(),
+}));
+
+describe('Register the Transfer route', () => {
+  test('registers the /transfer route', () => {
+    const strapi = strapiMockFactory();
+
+    register(strapi);
+    expect(strapi.admin.routes.push).toHaveBeenCalledWith({
+      method: 'GET',
+      path: '/transfer',
+      handler: createTransferHandler(),
+      config: {
+        auth: false,
+      },
+    });
+  });
+});

packages/core/data-transfer/src/strapi/index.ts

@@ -0,0 +1,5 @@
+export * as providers from './providers';
+export * as queries from './queries';
+export * as remote from './remote';
+
+export { default as register } from './register';

packages/core/data-transfer/src/strapi/providers/index.ts

@@ -0,0 +1,6 @@
+// Local
+export * from './local-destination';
+export * from './local-source';
+
+// Remote
+export * from './remote-destination';

packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts

@@ -1,8 +1,8 @@
 import fse from 'fs-extra';
 import { Writable, Readable } from 'stream';
-import type { IAsset } from '../../../../types';
+import type { IAsset } from '../../../../../types';
 
-import { getStrapiFactory } from '../../../__tests__/test-utils';
+import { getStrapiFactory } from '../../../../__tests__/test-utils';
 import { createLocalStrapiDestinationProvider } from '../index';
 
 const write = jest.fn((_chunk, _encoding, callback) => {

packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts

@@ -1,6 +1,10 @@
 import { createLocalStrapiDestinationProvider } from '../index';
 import * as restoreApi from '../strategies/restore';
-import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils';
+import {
+  getStrapiFactory,
+  getContentTypes,
+  setGlobalStrapi,
+} from '../../../../__tests__/test-utils';
 
 afterEach(() => {
   jest.clearAllMocks();

packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts

@@ -1,6 +1,10 @@
 import { deleteRecords, restoreConfigs } from '../strategies/restore';
-import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils';
-import { IConfiguration } from '../../../../types';
+import {
+  getStrapiFactory,
+  getContentTypes,
+  setGlobalStrapi,
+} from '../../../../__tests__/test-utils';
+import { IConfiguration } from '../../../../../types';
 
 const entities = [
   {

packages/core/data-transfer/src/strapi/providers/local-destination/index.ts

@@ -1,10 +1,10 @@
 import { Writable } from 'stream';
 import path from 'path';
 import * as fse from 'fs-extra';
-import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../types';
+import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../../types';
 
 import { restore } from './strategies';
-import * as utils from '../../utils';
+import * as utils from '../../../utils';
 
 export const VALID_CONFLICT_STRATEGIES = ['restore', 'merge'];
 export const DEFAULT_CONFLICT_STRATEGY = 'restore';
@@ -25,6 +25,9 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
 
   strapi?: Strapi.Strapi;
 
+  /**
+   * The entities mapper is used to map old entities to their new IDs
+   */
   #entitiesMapper: { [type: string]: { [id: number]: number } };
 
   constructor(options: ILocalStrapiDestinationProviderOptions) {
@@ -123,6 +126,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
     throw new Error(`Invalid strategy supplied: "${strategy}"`);
   }
 
+  // TODO: Move this logic to the restore strategy
   async getAssetsStream(): Promise<Writable> {
     if (!this.strapi) {
       throw new Error('Not able to stream Assets. Strapi instance not found');

packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts

@@ -1,6 +1,6 @@
 import { Writable } from 'stream';
 import chalk from 'chalk';
-import { IConfiguration } from '../../../../../types';
+import { IConfiguration } from '../../../../../../types';
 
 const restoreCoreStore = async <T extends { value: unknown }>(strapi: Strapi.Strapi, data: T) => {
   return strapi.db.query('strapi::core-store').create({

packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts

@@ -3,9 +3,9 @@ import type { SchemaUID } from '@strapi/strapi/lib/types/utils';
 import { get } from 'lodash/fp';
 import { Writable } from 'stream';
 
-import type { IEntity } from '../../../../../types';
-import { json } from '../../../../utils';
-import * as shared from '../../../shared';
+import type { IEntity } from '../../../../../../types';
+import { json } from '../../../../../utils';
+import * as queries from '../../../../queries';
 
 interface IEntitiesRestoreStreamOptions {
   strapi: Strapi.Strapi;
@@ -14,7 +14,7 @@ interface IEntitiesRestoreStreamOptions {
 
 const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
   const { strapi, updateMappingTable } = options;
-  const query = shared.strapi.entity.createEntityQuery(strapi);
+  const query = queries.entity.createEntityQuery(strapi);
 
   return new Writable({
     objectMode: true,
@@ -24,13 +24,19 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
       const { create, getDeepPopulateComponentLikeQuery } = query(type);
       const contentType = strapi.getModel(type);
 
-      const resolveType = (paths: string[]) => {
+      /**
+       * Resolve the component UID of an entity's attribute based
+       * on a given path (components & dynamic zones only)
+       */
+      const resolveType = (paths: string[]): string | undefined => {
         let cType = contentType;
         let value: unknown = data;
 
         for (const path of paths) {
           value = get(path, value);
 
+          // Needed when the value of cType should be computed
+          // based on the next value (eg: dynamic zones)
           if (typeof cType === 'function') {
             cType = cType(value);
           }
@@ -48,24 +54,32 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
           }
         }
 
-        return cType.uid;
+        return cType?.uid;
       };
 
       try {
-        // Create the entity
         const created = await create({
           data,
           populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }),
           select: 'id',
         });
 
+        // Compute differences between original & new entities
         const diffs = json.diff(data, created);
 
         updateMappingTable(type, id, created.id);
 
+        // For each difference found on an ID attribute,
+        // update the mapping the table accordingly
         diffs.forEach((diff) => {
           if (diff.kind === 'modified' && diff.path.at(-1) === 'id') {
             const target = resolveType(diff.path);
+
+            // If no type is found for the given path, then ignore the diff
+            if (!target) {
+              return;
+            }
+
             const [oldID, newID] = diff.values as [number, number];
 
             updateMappingTable(target, oldID, newID);

packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts

@@ -1,5 +1,5 @@
 import type { ContentTypeSchema } from '@strapi/strapi';
-import * as shared from '../../../shared';
+import * as queries from '../../../../queries';
 
 export interface IRestoreOptions {
   assets?: boolean;
@@ -36,7 +36,7 @@ const deleteEntitiesRecord = async (
   options: IRestoreOptions = {}
 ): Promise<IDeleteResults> => {
   const { entities } = options;
-  const query = shared.strapi.entity.createEntityQuery(strapi);
+  const query = queries.entity.createEntityQuery(strapi);
   const contentTypes = Object.values<ContentTypeSchema>(strapi.contentTypes);
 
   const contentTypesToClear = contentTypes.filter((contentType) => {

packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts

@@ -1,6 +1,6 @@
 import { Writable } from 'stream';
-import { ILink } from '../../../../../types';
-import { createLinkQuery } from '../../../shared/strapi/link';
+import { ILink } from '../../../../../../types';
+import { createLinkQuery } from '../../../../queries/link';
 
 export const createLinksWriteStream = (
   mapID: (uid: string, id: number) => number | undefined,

packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts

@@ -1,6 +1,10 @@
 import { Readable } from 'stream';
 
-import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils';
+import {
+  collect,
+  createMockedQueryBuilder,
+  getStrapiFactory,
+} from '../../../../__tests__/test-utils';
 import { createConfigurationStream } from '../configuration';
 
 describe('Configuration', () => {

packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts

@@ -1,12 +1,12 @@
 import { Readable, PassThrough } from 'stream';
-import type { IEntity } from '../../../../types';
+import type { IEntity } from '../../../../../types';
 
 import {
   collect,
   getStrapiFactory,
   getContentTypes,
   createMockedQueryBuilder,
-} from '../../../__tests__/test-utils';
+} from '../../../../__tests__/test-utils';
 import { createEntitiesStream, createEntitiesTransformStream } from '../entities';
 
 describe('Local Strapi Source Provider - Entities Streaming', () => {

packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts

@@ -1,7 +1,11 @@
 import { Readable } from 'stream';
-import type { IEntity } from '../../../../types';
+import type { IEntity } from '../../../../../types';
 
-import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils';
+import {
+  collect,
+  createMockedQueryBuilder,
+  getStrapiFactory,
+} from '../../../../__tests__/test-utils';
 import { createLocalStrapiSourceProvider } from '..';
 
 describe('Local Strapi Source Provider', () => {

packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts

@@ -1,7 +1,7 @@
 import { Readable } from 'stream';
 
 import { createLinksStream } from '../links';
-import { collect, getStrapiFactory } from '../../../__tests__/test-utils';
+import { collect, getStrapiFactory } from '../../../../__tests__/test-utils';
 
 // TODO: entityService needs to be replaced with a mocked wrapper of db.connection and provide real metadata
 describe.skip('Local Strapi Source Provider - Entities Streaming', () => {

packages/core/data-transfer/src/strapi/providers/local-source/assets.ts

@@ -2,7 +2,7 @@ import { join } from 'path';
 import { readdir, stat, createReadStream } from 'fs-extra';
 import { Duplex } from 'stream';
 
-import type { IAsset } from '../../../types';
+import type { IAsset } from '../../../../types';
 
 const IGNORED_FILES = ['.gitkeep'];
 

packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts

@@ -2,7 +2,7 @@ import { chain } from 'stream-chain';
 import { Readable } from 'stream';
 import { set } from 'lodash/fp';
 
-import type { IConfiguration } from '../../../types';
+import type { IConfiguration } from '../../../../types';
 
 /**
  * Create a readable stream that export the Strapi app configuration

packages/core/data-transfer/src/strapi/providers/local-source/entities.ts

@@ -1,8 +1,8 @@
 import type { ContentTypeSchema } from '@strapi/strapi';
 
 import { Readable, PassThrough } from 'stream';
-import * as shared from '../shared/strapi';
-import { IEntity } from '../../../types';
+import * as shared from '../../queries';
+import { IEntity } from '../../../../types';
 
 /**
  * Generate and consume content-types streams in order to stream each entity individually

packages/core/data-transfer/src/strapi/providers/local-source/index.ts

@@ -1,12 +1,12 @@
 import { chain } from 'stream-chain';
 import { Readable } from 'stream';
 
-import type { IMetadata, ISourceProvider, ProviderType } from '../../../types';
+import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types';
 import { createEntitiesStream, createEntitiesTransformStream } from './entities';
 import { createLinksStream } from './links';
 import { createConfigurationStream } from './configuration';
 import { createAssetsStream } from './assets';
-import * as utils from '../../utils';
+import * as utils from '../../../utils';
 
 export interface ILocalStrapiSourceProviderOptions {
   getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>;

packages/core/data-transfer/src/strapi/providers/local-source/links.ts

@@ -1,7 +1,7 @@
 import { Readable } from 'stream';
 
-import type { ILink } from '../../../types';
-import { createLinkQuery } from '../shared/strapi/link';
+import type { ILink } from '../../../../types';
+import { createLinkQuery } from '../../queries/link';
 
 /**
  * Create a Readable which will stream all the links from a Strapi instance

packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts

@@ -11,8 +11,8 @@ import type {
   IConfiguration,
   TransferStage,
   IAsset,
-} from '../../../types';
-import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider';
+} from '../../../../types';
+import type { ILocalStrapiDestinationProviderOptions } from '../local-destination';
 import { dispatch } from './utils';
 
 interface ITokenAuth {

packages/core/data-transfer/src/strapi/queries/link.ts

@@ -1,5 +1,6 @@
+import { RelationAttribute } from '@strapi/strapi';
 import { clone, isNil } from 'lodash/fp';
-import { ILink } from '../../../../types';
+import { ILink } from '../../../types';
 
 // TODO: Remove any types when we'll have types for DB metadata
 

packages/core/data-transfer/src/strapi/register.ts

@@ -0,0 +1,12 @@
+import { routes } from './remote';
+
+/**
+ * This is intended to be called on Strapi register phase.
+ *
+ * It registers a transfer route in the Strapi admin router.
+ */
+const register = (strapi: Strapi.Strapi) => {
+  routes.registerAdminTransferRoute(strapi);
+};
+
+export default register;

packages/core/data-transfer/src/strapi/remote/controllers/index.ts

@@ -0,0 +1 @@
+export * from './push';

packages/core/data-transfer/src/strapi/remote/controllers/push.ts

@@ -1,6 +1,6 @@
 import { PassThrough, Writable } from 'stream-chain';
 
-import { IAsset, IMetadata, PushTransferMessage, PushTransferStage } from '../../../types';
+import { IAsset, IMetadata, PushTransferMessage, PushTransferStage } from '../../../../types';
 import {
   createLocalStrapiDestinationProvider,
   ILocalStrapiDestinationProviderOptions,
@@ -71,7 +71,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions):
           streams.entities = provider.getEntitiesStream();
         }
 
-        await writeAsync(streams.entities!, entity);
+        await writeAsync(streams.entities, entity);
       },
 
       async links(link) {
@@ -79,7 +79,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions):
           streams.links = await provider.getLinksStream();
         }
 
-        await writeAsync(streams.links!, link);
+        await writeAsync(streams.links, link);
       },
 
       async configuration(config) {
@@ -87,7 +87,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions):
           streams.configuration = await provider.getConfigurationStream();
         }
 
-        await writeAsync(streams.configuration!, config);
+        await writeAsync(streams.configuration, config);
       },
 
       async assets(payload) {

packages/core/data-transfer/src/strapi/remote/handlers.ts

@@ -1,13 +1,14 @@
+// eslint-disable-next-line node/no-extraneous-import
 import type { Context } from 'koa';
 import type { ServerOptions } from 'ws';
 
 import { v4 } from 'uuid';
 import { WebSocket } from 'ws';
 
-import type { IPushController } from './push';
+import type { IPushController } from './controllers/push';
 
 import { InitMessage, Message, TransferKind } from '../../../types';
-import createPushController from './push';
+import createPushController from './controllers/push';
 
 interface ITransferState {
   kind?: TransferKind;
@@ -15,7 +16,7 @@ interface ITransferState {
   controller?: IPushController;
 }
 
-const createTransferController =
+export const createTransferHandler =
   (options: ServerOptions = {}) =>
   async (ctx: Context) => {
     const upgradeHeader = (ctx.request.headers.upgrade || '')
@@ -143,5 +144,3 @@ const createTransferController =
       ctx.respond = false;
     }
   };
-
-export default createTransferController;

packages/core/data-transfer/src/strapi/remote/index.ts

@@ -0,0 +1,2 @@
+export * as controllers from './controllers';
+export * as routes from './routes';

packages/core/data-transfer/src/strapi/remote/routes.ts

@@ -0,0 +1,35 @@
+// eslint-disable-next-line node/no-extraneous-import
+import type { Context } from 'koa';
+
+import { createTransferHandler } from './handlers';
+
+// Extend Strapi interface type to access the admin routes' API
+// TODO: Remove this when the Strapi instances will be better typed
+declare module '@strapi/strapi' {
+  interface Strapi {
+    admin: {
+      routes: {
+        method: string;
+        path: string;
+        handler: (ctx: Context) => Promise<void>;
+        config: unknown;
+      }[];
+    };
+  }
+}
+
+/**
+ * Register a transfer route in the Strapi admin router.
+ *
+ * It exposes a WS server that can be used to run and manage transfer processes.
+ *
+ * @param strapi - A Strapi instance
+ */
+export const registerAdminTransferRoute = (strapi: Strapi.Strapi) => {
+  strapi.admin.routes.push({
+    method: 'GET',
+    path: '/transfer',
+    handler: createTransferHandler(),
+    config: { auth: false },
+  });
+};

packages/core/data-transfer/src/utils/encryption/decrypt.ts

@@ -1,5 +1,5 @@
 import { Cipher, scryptSync, CipherKey, BinaryLike, createDecipheriv } from 'crypto';
-import { EncryptionStrategy, Strategies, Algorithm } from '../../types';
+import { EncryptionStrategy, Strategies, Algorithm } from '../../../types';
 
 // different key values depending on algorithm chosen
 const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
@@ -33,6 +33,14 @@ const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
   return strategies[algorithm];
 };
 
+/**
+ * It creates a cipher instance used for decryption
+ *
+ * @param key - The decryption key
+ * @param algorithm - The algorithm to use to create the Cipher
+ *
+ * @returns A {@link Cipher} instance created with the given key & algorithm
+ */
 export const createDecryptionCipher = (
   key: string,
   algorithm: Algorithm = 'aes-128-ecb'

packages/core/data-transfer/src/utils/encryption/encrypt.ts

@@ -1,5 +1,5 @@
 import { createCipheriv, Cipher, scryptSync, CipherKey, BinaryLike } from 'crypto';
-import { EncryptionStrategy, Strategies, Algorithm } from '../../types';
+import { EncryptionStrategy, Strategies, Algorithm } from '../../../types';
 
 // different key values depending on algorithm chosen
 const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
@@ -33,6 +33,14 @@ const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
   return strategies[algorithm];
 };
 
+/**
+ * It creates a cipher instance used for encryption
+ *
+ * @param key - The encryption key
+ * @param algorithm - The algorithm to use to create the Cipher
+ *
+ * @returns A {@link Cipher} instance created with the given key & algorithm
+ */
 export const createEncryptionCipher = (
   key: string,
   algorithm: Algorithm = 'aes-128-ecb'

packages/core/data-transfer/src/utils/index.ts

@@ -1,3 +1,4 @@
+export * as encryption from './encryption';
 export * as stream from './stream';
 export * as json from './json';
 export * as schema from './schema';

packages/core/data-transfer/src/utils/json.ts

@@ -2,6 +2,13 @@ import { isArray, isObject, zip, isEqual, uniq } from 'lodash/fp';
 
 const createContext = (): Context => ({ path: [] });
 
+/**
+ * Compute differences between two JSON objects and returns them
+ *
+ * @param a - First object
+ * @param b - Second object
+ * @param ctx - Context used to keep track of the current path during recursion
+ */
 export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Diff[] => {
   const diffs: Diff[] = [];
   const { path } = ctx;
@@ -70,7 +77,7 @@ export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Di
   }
 
   if (!isEqual(a, b)) {
-    modified();
+    return modified();
   }
 
   return diffs;

packages/core/data-transfer/src/utils/schema.ts

@@ -0,0 +1,27 @@
+import type { Schema } from '@strapi/strapi';
+import { mapValues, pick } from 'lodash/fp';
+
+/**
+ * List of schema properties that should be kept when sanitizing schemas
+ */
+const VALID_SCHEMA_PROPERTIES = [
+  'collectionName',
+  'info',
+  'options',
+  'pluginOptions',
+  'attributes',
+  'kind',
+  'modelType',
+  'modelName',
+  'uid',
+  'plugin',
+  'globalId',
+];
+
+/**
+ * Sanitize a schemas dictionnary by omiting unwanted properties
+ * The list of allowed properties can be found here: {@link VALID_SCHEMA_PROPERTIES}
+ */
+export const mapSchemasValues = (schemas: Record<string, Schema>) => {
+  return mapValues(pick(VALID_SCHEMA_PROPERTIES), schemas);
+};

packages/core/data-transfer/src/utils/stream.ts

@@ -0,0 +1,72 @@
+import { Transform, Readable } from 'stream';
+
+type TransformOptions = ConstructorParameters<typeof Transform>[0];
+
+/**
+ * Create a filter stream that discard chunks which doesn't satisfies the given predicate
+ *
+ * @param predicate - A filter predicate, takes a stream data chunk as parameter and returns a boolean value
+ * @param options - Transform stream options
+ */
+export const filter = <T>(
+  predicate: (value: T) => boolean | Promise<boolean>,
+  options: TransformOptions = { objectMode: true }
+): Transform => {
+  return new Transform({
+    ...options,
+
+    async transform(chunk, _encoding, callback) {
+      const keep = await predicate(chunk);
+
+      callback(null, keep ? chunk : undefined);
+    },
+  });
+};
+
+/**
+ * Create a map stream that transform chunks using the given predicate
+ *
+ * @param predicate - A map predicate, takes a stream data chunk as parameter and returns a mapped value
+ * @param options - Transform stream options
+ */
+export const map = <T, U = T>(
+  predicate: (value: T) => U | Promise<U>,
+  options: TransformOptions = { objectMode: true }
+): Transform => {
+  return new Transform({
+    ...options,
+
+    async transform(chunk, _encoding, callback) {
+      const mappedValue = await predicate(chunk);
+
+      callback(null, mappedValue);
+    },
+  });
+};
+
+/**
+ * Collect every chunks from a Readable stream.
+ *
+ * @param stream - The redable stream to collect data from
+ * @param options.destroy - If set to true, it automatically calls `destroy()` on the given stream upon receiving the 'end' event
+ */
+export const collect = <T = unknown>(
+  stream: Readable,
+  options: { destroy: boolean } = { destroy: true }
+): Promise<T[]> => {
+  const chunks: T[] = [];
+
+  return new Promise((resolve, reject) => {
+    stream
+      .on('close', () => resolve(chunks))
+      .on('error', reject)
+      .on('data', (chunk) => chunks.push(chunk))
+      .on('end', () => {
+        if (options.destroy) {
+          stream.destroy();
+        }
+
+        resolve(chunks);
+      });
+  });
+};

packages/core/data-transfer/tsconfig.json

@@ -4,11 +4,11 @@
     "strict": true,
     "lib": ["ESNEXT"],
     "skipLibCheck": true,
-    "outDir": "dist",
+    "outDir": "lib",
     "declaration": true,
     "sourceMap": true,
     "esModuleInterop": true
   },
-  "include": ["types", "lib/**/*.ts"],
-  "exclude": ["node_modules", "lib/**/__tests__"]
+  "include": ["types", "src/**/*.ts"],
+  "exclude": ["node_modules", "src/**/__tests__"]
 }

packages/core/data-transfer/types/remote.d.ts

@@ -1,4 +1,4 @@
-import type { ILocalStrapiDestinationProviderOptions } from '../lib';
+import type { ILocalStrapiDestinationProviderOptions } from '../src/strapi/providers';
 import type { IAsset, IConfiguration, IEntity, ILink } from './common-entities';
 
 /**