Merge branch 'features/deits' into deits/transfer-push

package.json

@@ -31,7 +31,7 @@
     "prepare": "husky install",
     "setup": "yarn && yarn clean && yarn build",
     "clean": "lerna run --stream clean --no-private",
-    "watch": "lerna run --stream watch --no-private",
+    "watch": "lerna run --stream watch --no-private --parallel",
     "build": "lerna run --stream build --no-private",
     "generate": "plop --plopfile ./packages/generators/admin/plopfile.js",
     "lint": "npm-run-all -p lint:code lint:css",

packages/core/data-transfer/lib/engine/index.ts

@@ -159,7 +159,7 @@ class TransferEngine<
     });
   }
 
-  #emitTransferUpdate(type: 'start' | 'finish' | 'error', payload?: object) {
+  #emitTransferUpdate(type: 'init' | 'start' | 'finish' | 'error', payload?: object) {
     this.progress.stream.emit(`transfer::${type}`, payload);
   }
 
@@ -352,9 +352,8 @@ class TransferEngine<
     // reset data between transfers
     this.progress.data = {};
 
-    this.#emitTransferUpdate('start');
-
     try {
+      this.#emitTransferUpdate('init');
       await this.bootstrap();
       await this.init();
 
@@ -367,6 +366,8 @@ class TransferEngine<
         );
       }
 
+      this.#emitTransferUpdate('start');
+
       await this.beforeTransfer();
 
       // Run the transfer stages

packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts

@@ -49,6 +49,8 @@ class LocalFileSourceProvider implements ISourceProvider {
 
   options: ILocalFileSourceProviderOptions;
 
+  #metadata?: IMetadata;
+
   constructor(options: ILocalFileSourceProviderOptions) {
     this.options = options;
 
@@ -60,15 +62,16 @@ class LocalFileSourceProvider implements ISourceProvider {
   }
 
   /**
-   * Pre flight checks regarding the provided options (making sure that the provided path is correct, etc...)
+   * Pre flight checks regarding the provided options, making sure that the file can be opened (decrypted, decompressed), etc.
    */
   async bootstrap() {
     const { path: filePath } = this.options.file;
+
     try {
-      // This is only to show a nicer error, it doesn't ensure the file will still exist when we try to open it later
+      // Read the metadata to ensure the file can be parsed
-      await fs.access(filePath, fs.constants.R_OK);
+      this.#metadata = await this.getMetadata();
     } catch (e) {
-      throw new Error(`Can't access file "${filePath}".`);
+      throw new Error(`Can't read file "${filePath}".`);
     }
   }
 

packages/core/data-transfer/package.json

@@ -30,7 +30,7 @@
     "build": "tsc -p tsconfig.json",
     "clean": "rimraf ./dist",
     "build:clean": "yarn clean && yarn build",
-    "watch": "yarn build -w",
+    "watch": "yarn build -w --preserveWatchOutput",
     "test:unit": "jest --verbose"
   },
   "directories": {

packages/core/strapi/lib/commands/__tests__/export.test.js

@@ -1,98 +1,146 @@
 'use strict';
 
-const utils = require('../transfer/utils');
+describe('export', () => {
+  const defaultFileName = 'defaultFilename';
 
-const mockDataTransfer = {
+  // mock @strapi/data-transfer
-  createLocalFileDestinationProvider: jest.fn(),
+  const mockDataTransfer = {
-  createLocalStrapiSourceProvider: jest.fn(),
+    createLocalFileDestinationProvider: jest.fn().mockReturnValue({ name: 'testDest' }),
-  createTransferEngine: jest.fn().mockReturnValue({
+    createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testSource' }),
+    createTransferEngine() {
+      return {
         transfer: jest.fn().mockReturnValue(Promise.resolve({})),
-  }),
+        progress: {
-};
+          on: jest.fn(),
-
+          stream: {
-jest.mock(
+            on: jest.fn(),
+          },
+        },
+        sourceProvider: { name: 'testSource' },
+        destinationProvider: { name: 'testDestination' },
+      };
+    },
+  };
+  jest.mock(
     '@strapi/data-transfer',
     () => {
       return mockDataTransfer;
     },
     { virtual: true }
-);
+  );
 
-const exportCommand = require('../transfer/export');
+  // mock utils
+  const mockUtils = {
+    createStrapiInstance() {
+      return {
+        telemetry: {
+          send: jest.fn(),
+        },
+      };
+    },
+    getDefaultExportName: jest.fn(() => defaultFileName),
+  };
+  jest.mock(
+    '../transfer/utils',
+    () => {
+      return mockUtils;
+    },
+    { virtual: true }
+  );
 
-const exit = jest.spyOn(process, 'exit').mockImplementation(() => {});
+  // other spies=
-jest.spyOn(console, 'error').mockImplementation(() => {});
+  jest.spyOn(console, 'log').mockImplementation(() => {});
+  jest.spyOn(console, 'warn').mockImplementation(() => {});
+  jest.spyOn(console, 'error').mockImplementation(() => {});
 
-jest.mock('../transfer/utils');
+  // Now that everything is mocked, import export command
+  const exportCommand = require('../transfer/export');
 
-const defaultFileName = 'defaultFilename';
+  const expectExit = async (code, fn) => {
-
+    const exit = jest.spyOn(process, 'exit').mockImplementation((number) => {
-describe('export', () => {
+      throw new Error(`process.exit: ${number}`);
-  beforeEach(() => {
-    jest.resetAllMocks();
     });
+    await expect(async () => {
+      await fn();
+    }).rejects.toThrow();
+    expect(exit).toHaveBeenCalledWith(code);
+    exit.mockRestore();
+  };
+
+  beforeEach(() => {});
 
   it('uses path provided by user', async () => {
-    const filename = 'testfile';
+    const filename = 'test';
 
+    await expectExit(1, async () => {
       await exportCommand({ file: filename });
+    });
 
     expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
       expect.objectContaining({
         file: { path: filename },
       })
     );
-    expect(utils.getDefaultExportName).not.toHaveBeenCalled();
+    expect(mockUtils.getDefaultExportName).not.toHaveBeenCalled();
-    expect(exit).toHaveBeenCalled();
   });
 
   it('uses default path if not provided by user', async () => {
-    utils.getDefaultExportName.mockReturnValue(defaultFileName);
+    await expectExit(1, async () => {
-
       await exportCommand({});
+    });
 
+    expect(mockUtils.getDefaultExportName).toHaveBeenCalledTimes(1);
     expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
       expect.objectContaining({
         file: { path: defaultFileName },
       })
     );
-
-    expect(utils.getDefaultExportName).toHaveBeenCalled();
-    expect(exit).toHaveBeenCalled();
   });
 
   it('encrypts the output file if specified', async () => {
     const encrypt = true;
+    await expectExit(1, async () => {
       await exportCommand({ encrypt });
+    });
+
     expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
       expect.objectContaining({
         encryption: { enabled: encrypt },
       })
     );
-    expect(exit).toHaveBeenCalled();
   });
 
   it('encrypts the output file with the given key', async () => {
     const key = 'secret-key';
     const encrypt = true;
-
+    await expectExit(1, async () => {
       await exportCommand({ encrypt, key });
+    });
+
     expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
       expect.objectContaining({
         encryption: { enabled: encrypt, key },
       })
     );
-    expect(exit).toHaveBeenCalled();
   });
 
-  it('compresses the output file if specified', async () => {
+  it('uses compress option', async () => {
-    const compress = true;
+    await expectExit(1, async () => {
-    await exportCommand({ compress });
+      await exportCommand({ compress: false });
+    });
+
     expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
       expect.objectContaining({
-        compression: { enabled: compress },
+        compression: { enabled: false },
+      })
+    );
+    await expectExit(1, async () => {
+      await exportCommand({ compress: true });
+    });
+    expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
+      expect.objectContaining({
+        compression: { enabled: true },
       })
     );
-    expect(exit).toHaveBeenCalled();
   });
 });

packages/core/strapi/lib/commands/transfer/export.js

@@ -74,12 +74,9 @@ module.exports = async (opts) => {
     },
   });
 
-  try {
-    logger.log(`Starting export...`);
-
   const progress = engine.progress.stream;
 
-    const telemetryPayload = (/* payload */) => {
+  const getTelemetryPayload = (/* payload */) => {
     return {
       eventProperties: {
         source: engine.sourceProvider.name,
@@ -88,18 +85,12 @@ module.exports = async (opts) => {
     };
   };
 
-    progress.on('transfer::start', (payload) => {
+  progress.on('transfer::start', async () => {
-      strapi.telemetry.send('didDEITSProcessStart', telemetryPayload(payload));
+    logger.log(`Starting export...`);
-    });
+    await strapi.telemetry.send('didDEITSProcessStart', getTelemetryPayload());
-
-    progress.on('transfer::finish', (payload) => {
-      strapi.telemetry.send('didDEITSProcessFinish', telemetryPayload(payload));
-    });
-
-    progress.on('transfer::error', (payload) => {
-      strapi.telemetry.send('didDEITSProcessFail', telemetryPayload(payload));
   });
 
+  try {
     const results = await engine.transfer();
     const outFile = results.destination.file.path;
 
@@ -113,11 +104,15 @@ module.exports = async (opts) => {
 
     logger.log(`${chalk.bold('Export process has been completed successfully!')}`);
     logger.log(`Export archive is in ${chalk.green(outFile)}`);
-    process.exit(0);
   } catch (e) {
+    await strapi.telemetry.send('didDEITSProcessFail', getTelemetryPayload());
     logger.error('Export process failed unexpectedly:', e.toString());
     process.exit(1);
   }
+
+  // Note: Telemetry can't be sent in a finish event, because it runs async after this block but we can't await it, so if process.exit is used it won't send
+  await strapi.telemetry.send('didDEITSProcessFinish', getTelemetryPayload());
+  process.exit(0);
 };
 
 /**

packages/core/strapi/lib/commands/transfer/import.js

@@ -77,13 +77,11 @@ module.exports = async (opts) => {
       ],
     },
   };
+
   const engine = createTransferEngine(source, destination, engineOptions);
 
-  try {
-    logger.info('Starting import...');
-
   const progress = engine.progress.stream;
-    const telemetryPayload = (/* payload */) => {
+  const getTelemetryPayload = () => {
     return {
       eventProperties: {
         source: engine.sourceProvider.name,
@@ -92,29 +90,27 @@ module.exports = async (opts) => {
     };
   };
 
-    progress.on('transfer::start', (payload) => {
+  progress.on('transfer::start', async () => {
-      strapiInstance.telemetry.send('didDEITSProcessStart', telemetryPayload(payload));
+    logger.info('Starting import...');
-    });
+    await strapiInstance.telemetry.send('didDEITSProcessStart', getTelemetryPayload());
-
-    progress.on('transfer::finish', (payload) => {
-      strapiInstance.telemetry.send('didDEITSProcessFinish', telemetryPayload(payload));
-    });
-
-    progress.on('transfer::error', (payload) => {
-      strapiInstance.telemetry.send('didDEITSProcessFail', telemetryPayload(payload));
   });
 
+  try {
     const results = await engine.transfer();
     const table = buildTransferTable(results.engine);
     logger.info(table.toString());
 
     logger.info('Import process has been completed successfully!');
-    process.exit(0);
   } catch (e) {
+    await strapiInstance.telemetry.send('didDEITSProcessFail', getTelemetryPayload());
     logger.error('Import process failed unexpectedly:');
     logger.error(e);
     process.exit(1);
   }
+
+  // Note: Telemetry can't be sent in a finish event, because it runs async after this block but we can't await it, so if process.exit is used it won't send
+  await strapi.telemetry.send('didDEITSProcessFinish', getTelemetryPayload());
+  process.exit(0);
 };
 
 /**