Merge branch 'features/deits' of github.com:strapi/strapi into deits/beta-enhancements

packages/core/data-transfer/lib/engine/__tests__/engine.test.ts

@@ -363,50 +363,6 @@ describe('Transfer engine', () => {
   });
 
   describe('transfer', () => {
-    test('requires strategy to be either restore or merge', async () => {
-      const engineOptions = {
-        versionMatching: 'exact',
-        exclude: [],
-      } as unknown as ITransferEngineOptions;
-
-      const restoreEngine = createTransferEngine(minimalSource, minimalDestination, {
-        ...engineOptions,
-        strategy: 'restore',
-      });
-      await restoreEngine.transfer();
-      expect(restoreEngine).toBeValidTransferEngine();
-
-      const mergeEngine = createTransferEngine(minimalSource, minimalDestination, {
-        ...engineOptions,
-        strategy: 'merge',
-      });
-      await mergeEngine.transfer();
-      expect(mergeEngine).toBeValidTransferEngine();
-
-      // undefined strategy
-      await expect(
-        (async () => {
-          const invalidEngine = createTransferEngine(
-            minimalSource,
-            minimalDestination,
-            engineOptions
-          );
-          await invalidEngine.transfer();
-        })()
-      ).rejects.toThrow();
-
-      // invalid strategy
-      await expect(
-        (async () => {
-          const invalidEngine = createTransferEngine(minimalSource, minimalDestination, {
-            ...engineOptions,
-            strategy: 'foo',
-          } as unknown as ITransferEngineOptions);
-          await invalidEngine.transfer();
-        })()
-      ).rejects.toThrow();
-    });
-
     test('calls all provider stages', async () => {
       const engine = createTransferEngine(completeSource, completeDestination, defaultOptions);
       expect(completeSource).toHaveSourceStagesCalledTimes(0);

packages/core/data-transfer/lib/engine/index.ts

@@ -38,8 +38,6 @@ type TransferEngineProgress = {
   stream: PassThrough;
 };
 
-export const VALID_STRATEGIES = ['restore', 'merge'];
-
 class TransferEngine<
   S extends ISourceProvider = ISourceProvider,
   D extends IDestinationProvider = IDestinationProvider
@@ -197,7 +195,7 @@ class TransferEngine<
     if (!isEmpty(diffs)) {
       throw new Error(
         `Import process failed because the project doesn't have a matching data structure 
-        ${JSON.stringify(diffs, null, 2)}        
+        ${JSON.stringify(diffs, null, 2)}
         `
       );
     }
@@ -262,27 +260,21 @@ class TransferEngine<
     }
   }
 
-  validateTransferOptions() {
-    if (!VALID_STRATEGIES.includes(this.options.strategy)) {
-      throw new Error(`Invalid stategy ${this.options.strategy}`);
-    }
-  }
-
   async transfer(): Promise<ITransferResults<S, D>> {
     try {
-      this.validateTransferOptions();
-
       await this.bootstrap();
       await this.init();
 
       const isValidTransfer = await this.integrityCheck();
 
       if (!isValidTransfer) {
+        // TODO: provide the log from the integrity check
         throw new Error(
           `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.`
         );
       }
 
+      await this.beforeTransfer();
       // Run the transfer stages
       await this.transferSchemas();
       await this.transferEntities();
@@ -306,6 +298,11 @@ class TransferEngine<
     };
   }
 
+  async beforeTransfer(): Promise<void> {
+    await this.sourceProvider.beforeTransfer?.();
+    await this.destinationProvider.beforeTransfer?.();
+  }
+
   async transferSchemas(): Promise<void> {
     const stageName: TransferStage = 'schemas';
 

packages/core/data-transfer/lib/providers/local-file-source-provider.ts

@@ -82,7 +82,6 @@ class LocalFileSourceProvider implements ISourceProvider {
     // TODO: need to read the file & extract the metadata json file
     // => we might also need to read the schema.jsonl files & implements a custom stream-check
     const backupStream = this.#getBackupStream();
-
     return this.#parseJSONFile<IMetadata>(backupStream, METADATA_FILE_PATH);
   }
 

packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts

@@ -0,0 +1,138 @@
+import { createLocalStrapiDestinationProvider } from '../index';
+import * as restoreApi from '../restore';
+import { getStrapiFactory, getContentTypes } from '../../test-utils';
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+jest.mock('../restore', () => {
+  return {
+    __esModule: true,
+    ...jest.requireActual('../restore'),
+  };
+});
+
+describe('Local Strapi Source Destination', () => {
+  describe('Bootstrap', () => {
+    test('Should not have a defined Strapi instance if bootstrap has not been called', () => {
+      const provider = createLocalStrapiDestinationProvider({
+        getStrapi: getStrapiFactory(),
+        strategy: 'restore',
+      });
+
+      expect(provider.strapi).not.toBeDefined();
+    });
+
+    test('Should have a defined Strapi instance if bootstrap has been called', async () => {
+      const provider = createLocalStrapiDestinationProvider({
+        getStrapi: getStrapiFactory(),
+        strategy: 'restore',
+      });
+      await provider.bootstrap();
+
+      expect(provider.strapi).toBeDefined();
+    });
+  });
+
+  describe('Strategy', () => {
+    test('requires strategy to be either restore or merge', async () => {
+      const restoreProvider = createLocalStrapiDestinationProvider({
+        getStrapi: getStrapiFactory(),
+        strategy: 'restore',
+      });
+      await restoreProvider.bootstrap();
+      expect(restoreProvider.strapi).toBeDefined();
+
+      const mergeProvider = createLocalStrapiDestinationProvider({
+        getStrapi: getStrapiFactory(),
+        strategy: 'merge',
+      });
+      await mergeProvider.bootstrap();
+      expect(mergeProvider.strapi).toBeDefined();
+
+      await expect(
+        (async () => {
+          const invalidProvider = createLocalStrapiDestinationProvider({
+            getStrapi: getStrapiFactory(),
+            /* @ts-ignore: disable-next-line */
+            strategy: 'foo',
+          });
+          await invalidProvider.bootstrap();
+        })()
+      ).rejects.toThrow();
+    });
+
+    test('Should delete all entities if it is a restore', async () => {
+      const entities = [
+        {
+          entity: { id: 1, title: 'My first foo' },
+          contentType: { uid: 'foo' },
+        },
+        {
+          entity: { id: 4, title: 'Another Foo' },
+          contentType: { uid: 'foo' },
+        },
+        {
+          entity: { id: 12, title: 'Last foo' },
+          contentType: { uid: 'foo' },
+        },
+        {
+          entity: { id: 1, age: 21 },
+          contentType: { uid: 'bar' },
+        },
+        {
+          entity: { id: 2, age: 42 },
+          contentType: { uid: 'bar' },
+        },
+        {
+          entity: { id: 7, age: 84 },
+          contentType: { uid: 'bar' },
+        },
+        {
+          entity: { id: 9, age: 0 },
+          contentType: { uid: 'bar' },
+        },
+      ];
+      const deleteMany = jest.fn(async (uid: string) => ({
+        count: entities.filter((entity) => entity.contentType.uid === uid).length,
+      }));
+
+      const query = jest.fn(() => {
+        return {
+          deleteMany: jest.fn(() => ({
+            count: 0,
+          })),
+        };
+      });
+
+      const provider = createLocalStrapiDestinationProvider({
+        getStrapi: getStrapiFactory({
+          contentTypes: getContentTypes(),
+          entityService: {
+            deleteMany,
+          },
+          query,
+        }),
+        strategy: 'restore',
+      });
+      const deleteAllSpy = jest.spyOn(restoreApi, 'deleteAllRecords');
+      await provider.bootstrap();
+      await provider.beforeTransfer();
+
+      expect(deleteAllSpy).toBeCalledTimes(1);
+    });
+
+    test('Should not delete if it is a merge strategy', async () => {
+      const provider = createLocalStrapiDestinationProvider({
+        getStrapi: getStrapiFactory({}),
+        strategy: 'merge',
+      });
+      const deleteAllSpy = jest.spyOn(restoreApi, 'deleteAllRecords');
+      await provider.bootstrap();
+      await provider.beforeTransfer();
+
+      expect(deleteAllSpy).not.toBeCalled();
+    });
+  });
+});

packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts

@@ -0,0 +1,79 @@
+import { deleteAllRecords } from '../restore';
+import { getStrapiFactory, getContentTypes } from '../../test-utils';
+
+const entities = [
+  {
+    entity: { id: 1, title: 'My first foo' },
+    contentType: { uid: 'foo' },
+  },
+  {
+    entity: { id: 4, title: 'Another Foo' },
+    contentType: { uid: 'foo' },
+  },
+  {
+    entity: { id: 12, title: 'Last foo' },
+    contentType: { uid: 'foo' },
+  },
+  {
+    entity: { id: 1, age: 21 },
+    contentType: { uid: 'bar' },
+  },
+  {
+    entity: { id: 2, age: 42 },
+    contentType: { uid: 'bar' },
+  },
+  {
+    entity: { id: 7, age: 84 },
+    contentType: { uid: 'bar' },
+  },
+  {
+    entity: { id: 9, age: 0 },
+    contentType: { uid: 'bar' },
+  },
+];
+
+const deleteMany = jest.fn(async (uid: string) => ({
+  count: entities.filter((entity) => entity.contentType.uid === uid).length,
+}));
+
+const query = jest.fn(() => {
+  return {
+    deleteMany: jest.fn(() => ({
+      count: 0,
+    })),
+  };
+});
+
+describe('Restore ', () => {
+  test('Should delete all contentTypes', async () => {
+    const strapi = getStrapiFactory({
+      contentTypes: getContentTypes(),
+      entityService: {
+        deleteMany,
+      },
+      query,
+    })();
+
+    const { count } = await deleteAllRecords(strapi, {
+      /* @ts-ignore: disable-next-line */
+      contentTypes: Object.values(getContentTypes()),
+    });
+    expect(count).toBe(entities.length);
+  });
+
+  test('Should only delete chosen contentType', async () => {
+    const strapi = getStrapiFactory({
+      contentTypes: getContentTypes(),
+      entityService: {
+        deleteMany,
+      },
+      query,
+    })();
+
+    const { count } = await deleteAllRecords(strapi, {
+      /* @ts-ignore: disable-next-line */
+      contentTypes: [getContentTypes()['foo']],
+    });
+    expect(count).toBe(3);
+  });
+});

packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts

@@ -1,13 +1,18 @@
 // import { createLogger } from '@strapi/logger';
-import type { IDestinationProvider, IMetadata, ProviderType } from '../../types';
+import type { IDestinationProvider, IMetadata, ProviderType } from '../../../types';
+import { deleteAllRecords, DeleteOptions } from './restore';
 
 import chalk from 'chalk';
-import { Duplex } from 'stream';
+import { Writable } from 'stream';
 
-import { mapSchemasValues } from '../utils';
+import { mapSchemasValues } from '../../utils';
+
+export const VALID_STRATEGIES = ['restore', 'merge'];
 
 interface ILocalStrapiDestinationProviderOptions {
-  getStrapi(): Promise<Strapi.Strapi>;
+  getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>;
+  restore?: DeleteOptions;
+  strategy: 'restore' | 'merge';
 }
 
 // TODO: getting some type errors with @strapi/logger that need to be resolved first
@@ -32,6 +37,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
   }
 
   async bootstrap(): Promise<void> {
+    this.#validateOptions();
     this.strapi = await this.options.getStrapi();
   }
 
@@ -39,9 +45,42 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
     await this.strapi?.destroy?.();
   }
 
-  // TODO
+  #validateOptions() {
+    if (!VALID_STRATEGIES.includes(this.options.strategy)) {
+      throw new Error('Invalid stategy ' + this.options.strategy);
+    }
+  }
+
+  async #deleteAll() {
+    if (!this.strapi) {
+      throw new Error('Strapi instance not found');
+    }
+    return await deleteAllRecords(this.strapi, this.options.restore);
+  }
+
+  async beforeTransfer() {
+    if (this.options.strategy === 'restore') {
+      await this.#deleteAll();
+    }
+  }
+
   getMetadata(): IMetadata | Promise<IMetadata> {
-    return {};
+    const strapiVersion = strapi.config.get('info.strapi');
+    const createdAt = new Date().toISOString();
+
+    const plugins = Object.keys(strapi.plugins);
+
+    return {
+      createdAt,
+      strapi: {
+        version: strapiVersion,
+        plugins: plugins.map((name) => ({
+          name,
+          // TODO: Get the plugin actual version when it'll be available
+          version: strapiVersion,
+        })),
+      },
+    };
   }
 
   getSchemas() {
@@ -56,43 +95,4 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
 
     return mapSchemasValues(schemas);
   }
-
-  getEntitiesStream(): Duplex {
-    const self = this;
-
-    return new Duplex({
-      objectMode: true,
-      async write(entity, _encoding, callback) {
-        if (!self.strapi) {
-          callback(new Error('Strapi instance not found'));
-        }
-
-        const { type: uid, id, data } = entity;
-
-        try {
-          await strapi.entityService.create(uid, { data });
-        } catch (e: any) {
-          // TODO: remove "any" cast
-          log.warn(
-            chalk.bold(`Failed to import ${chalk.yellowBright(uid)} (${chalk.greenBright(id)})`)
-          );
-          e.details.errors
-            .map((err: any, i: number) => {
-              // TODO: add correct error type
-              const info = {
-                uid: chalk.yellowBright(`[${uid}]`),
-                path: chalk.blueBright(`[${err.path.join('.')}]`),
-                id: chalk.greenBright(`[${id}]`),
-                message: err.message,
-              };
-
-              return `(${i}) ${info.uid}${info.id}${info.path}: ${info.message}`;
-            })
-            .forEach((message: string) => log.warn(message));
-        } finally {
-          callback();
-        }
-      },
-    });
-  }
 }

packages/core/data-transfer/lib/providers/local-strapi-destination-provider/restore.ts

@@ -0,0 +1,35 @@
+import type { ContentTypeSchema } from '@strapi/strapi';
+
+export type DeleteOptions = {
+  contentTypes?: ContentTypeSchema[];
+  uidsOfModelsToDelete?: string[];
+  conditions?: {
+    [contentTypeUid: string]: {
+      params: any;
+    };
+  };
+};
+
+export const deleteAllRecords = async (strapi: Strapi.Strapi, deleteOptions?: DeleteOptions) => {
+  const conditions = deleteOptions?.conditions ?? {};
+  const contentTypes = deleteOptions?.contentTypes ?? [];
+  const uidsOfModelsToDelete = deleteOptions?.uidsOfModelsToDelete ?? [];
+  let count = 0;
+
+  await Promise.all(
+    contentTypes.map(async (contentType) => {
+      const params = conditions[contentType.uid]?.params ?? {};
+      const result = await strapi?.entityService.deleteMany(contentType.uid, params);
+      count += result ? result.count : 0;
+    })
+  );
+
+  await Promise.all(
+    uidsOfModelsToDelete.map(async (uid) => {
+      const result = await strapi?.query(uid).deleteMany();
+      count += result.count;
+    })
+  );
+
+  return { count };
+};

packages/core/data-transfer/lib/providers/test-utils/index.ts

@@ -0,0 +1,76 @@
+import { Readable } from 'stream';
+
+/**
+ * Collect every entity in a Readable stream
+ */
+export const collect = <T = unknown>(stream: Readable): Promise<T[]> => {
+  const chunks: T[] = [];
+
+  return new Promise((resolve, reject) => {
+    stream
+      .on('data', (chunk) => chunks.push(chunk))
+      .on('close', () => resolve(chunks))
+      .on('error', reject);
+  });
+};
+
+/**
+ * Create a "Strapi" like object factory based on the
+ * given params and cast it to the correct type
+ */
+export const getStrapiFactory =
+  <
+    T extends {
+      [key in keyof Partial<Strapi.Strapi>]: unknown;
+    }
+  >(
+    properties?: T
+  ) =>
+  () => {
+    return { ...properties } as Strapi.Strapi;
+  };
+
+/**
+ * Union type used to represent the default content types available
+ */
+export type ContentType = 'foo' | 'bar';
+
+/**
+ * Factory to get default content types test values
+ */
+export const getContentTypes = (): {
+  [key in ContentType]: { uid: key; attributes: { [attribute: string]: unknown } };
+} => ({
+  foo: { uid: 'foo', attributes: { title: { type: 'string' } } },
+  bar: { uid: 'bar', attributes: { age: { type: 'number' } } },
+});
+
+/**
+ * Create a factory of readable streams (wrapped with a jest mock function)
+ */
+export const createMockedReadableFactory = <T extends string = ContentType>(source: {
+  [ct in T]: Array<{ id: number; [key: string]: unknown }>;
+}) =>
+  jest.fn((uid: T) => {
+    return Readable.from(source[uid] || []);
+  });
+
+/**
+ * Create a factory of mocked query builders
+ */
+export const createMockedQueryBuilder = <T extends string = ContentType>(data: {
+  [key in T]: unknown[];
+}) =>
+  jest.fn((uid: T) => {
+    const state: { [key: string]: unknown } = { populate: undefined };
+
+    return {
+      populate(populate: unknown) {
+        state.populate = populate;
+        return this;
+      },
+      stream() {
+        return Readable.from(data[uid]);
+      },
+    };
+  });

packages/core/data-transfer/types/providers.d.ts

@@ -18,6 +18,8 @@ interface IProvider {
   getSchemas?(): any;
   close?(): Promise<void> | void;
   getMetadata(): IMetadata | null | Promise<IMetadata | null>;
+  beforeTransfer?(): Promise<void>;
+  validateOptions?(): void;
 }
 
 export interface ISourceProvider extends IProvider {

packages/core/data-transfer/types/transfer-engine.d.ts

@@ -56,6 +56,13 @@ export interface ITransferEngine<
    */
   close(): Promise<void>;
 
+  /**
+   * Run the preparations before starting a transfer for each provider
+   *
+   * related source and destination providers
+   */
+  beforeTransfer(): Promise<void>;
+
   /**
    * Start the schemas transfer by connecting the
    * related source and destination providers streams
@@ -96,11 +103,6 @@ export interface ITransferEngine<
  * Note: here, we're listing the TransferEngine options, not the individual providers' options
  */
 export interface ITransferEngineOptions {
-  /**
-   * The strategy to use when importing the data from the source to the destination
-   * Note: Should we keep this here or fully delegate the strategies logic to the destination?
-   */
-  strategy: 'restore' | 'merge';
   /**
    * What kind of version matching should be done between the source and the destination metadata?
    * @example

packages/core/strapi/lib/commands/transfer/import.js

@@ -33,9 +33,27 @@ module.exports = async (opts) => {
   /**
    * To local Strapi instance
    */
+  const strapiInstance = await strapi(await strapi.compile()).load();
+
+  const exceptions = [
+    'admin::permission',
+    'admin::user',
+    'admin::role',
+    'admin::api-token',
+    'admin::api-token-permission',
+  ];
+  const contentTypes = Object.values(strapiInstance.contentTypes);
+  const contentTypesToDelete = contentTypes.filter(
+    (contentType) => !exceptions.includes(contentType.uid)
+  );
   const destinationOptions = {
     async getStrapi() {
-      return strapi(await strapi.compile()).load();
+      return strapiInstance;
+    },
+    strategy: opts.conflictStrategy,
+    restore: {
+      contentTypes: contentTypesToDelete,
+      uidsOfModelsToDelete: ['webhook', 'strapi::core-store'],
     },
   };
   const destination = createLocalStrapiDestinationProvider(destinationOptions);