Merge branch 'features/deits' of github.com:strapi/strapi into features/deits

packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts

@@ -1,8 +1,8 @@
-import { createCipher } from '../encrypt';
+import { createEncryptionCipher } from '..';
 
 describe('Encryption', () => {
   test('encrypting data with default algorithm aes-128-ecb', () => {
-    const cipher = createCipher('password');
+    const cipher = createEncryptionCipher('password');
     const textToEncrypt = 'something ate an apple';
     const encryptedData = cipher.update(textToEncrypt);
 
@@ -12,7 +12,7 @@ describe('Encryption', () => {
   });
 
   test('encrypting data with aes128', () => {
-    const cipher = createCipher('password', 'aes128');
+    const cipher = createEncryptionCipher('password', 'aes128');
     const textToEncrypt = 'something ate an apple';
     const encryptedData = cipher.update(textToEncrypt);
 
@@ -22,7 +22,7 @@ describe('Encryption', () => {
   });
 
   test('encrypting data with aes192', () => {
-    const cipher = createCipher('password', 'aes192');
+    const cipher = createEncryptionCipher('password', 'aes192');
     const textToEncrypt = 'something ate an apple';
     const encryptedData = cipher.update(textToEncrypt);
 
@@ -32,7 +32,7 @@ describe('Encryption', () => {
   });
 
   test('encrypting data with aes256', () => {
-    const cipher = createCipher('password', 'aes256');
+    const cipher = createEncryptionCipher('password', 'aes256');
     const textToEncrypt = 'something ate an apple';
     const encryptedData = cipher.update(textToEncrypt);
 
@@ -42,9 +42,9 @@ describe('Encryption', () => {
   });
 
   test('data encrypted with different algorithms should have different results', () => {
-    const cipherAES256 = createCipher('password', 'aes256');
+    const cipherAES256 = createEncryptionCipher('password', 'aes256');
-    const cipherAES192 = createCipher('password', 'aes192');
+    const cipherAES192 = createEncryptionCipher('password', 'aes192');
-    const cipherDefault = createCipher('password');
+    const cipherDefault = createEncryptionCipher('password');
     const textToEncrypt = 'something ate an apple';
     const encryptedDataAES256 = cipherAES256.update(textToEncrypt).toString();
     const encryptedDataAES192 = cipherAES192.update(textToEncrypt).toString();
@@ -56,8 +56,8 @@ describe('Encryption', () => {
   });
 
   test('data encrypted with different key should be different', () => {
-    const cipher1 = createCipher('password');
+    const cipher1 = createEncryptionCipher('password');
-    const cipher2 = createCipher('differentpassword');
+    const cipher2 = createEncryptionCipher('differentpassword');
     const textToEncrypt = 'something ate an apple';
     const encryptedData1 = cipher1.update(textToEncrypt).toString();
     const encryptedData2 = cipher2.update(textToEncrypt).toString();

packages/core/data-transfer/lib/encryption/decrypt.ts

@@ -0,0 +1,41 @@
+import { Cipher, scryptSync, CipherKey, BinaryLike, createDecipheriv } from 'crypto';
+import { EncryptionStrategy, Strategies, Algorithm } from '../../types';
+
+// different key values depending on algorithm chosen
+const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
+  const strategies: Strategies = {
+    'aes-128-ecb': (key: string): Cipher => {
+      const hashedKey = scryptSync(key, '', 16);
+      const initVector: BinaryLike | null = null;
+      const securityKey: CipherKey = hashedKey;
+      return createDecipheriv(algorithm, securityKey, initVector);
+    },
+    aes128: (key: string): Cipher => {
+      const hashedKey = scryptSync(key, '', 32);
+      const initVector: BinaryLike | null = hashedKey.slice(16);
+      const securityKey: CipherKey = hashedKey.slice(0, 16);
+      return createDecipheriv(algorithm, securityKey, initVector);
+    },
+    aes192: (key: string): Cipher => {
+      const hashedKey = scryptSync(key, '', 40);
+      const initVector: BinaryLike | null = hashedKey.slice(24);
+      const securityKey: CipherKey = hashedKey.slice(0, 24);
+      return createDecipheriv(algorithm, securityKey, initVector);
+    },
+    aes256: (key: string): Cipher => {
+      const hashedKey = scryptSync(key, '', 48);
+      const initVector: BinaryLike | null = hashedKey.slice(32);
+      const securityKey: CipherKey = hashedKey.slice(0, 32);
+      return createDecipheriv(algorithm, securityKey, initVector);
+    },
+  };
+
+  return strategies[algorithm];
+};
+
+export const createDecryptionCipher = (
+  key: string,
+  algorithm: Algorithm = 'aes-128-ecb'
+): Cipher => {
+  return getDecryptionStrategy(algorithm)(key);
+};

packages/core/data-transfer/lib/encryption/encrypt.ts

@@ -33,6 +33,9 @@ const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
   return strategies[algorithm];
 };
 
-export const createCipher = (key: string, algorithm: Algorithm = 'aes-128-ecb'): Cipher => {
+export const createEncryptionCipher = (
+  key: string,
+  algorithm: Algorithm = 'aes-128-ecb'
+): Cipher => {
   return getEncryptionStrategy(algorithm)(key);
 };

packages/core/data-transfer/lib/encryption/index.ts

@@ -0,0 +1,2 @@
+export * from './encrypt';
+export * from './decrypt';

packages/core/data-transfer/lib/engine/index.ts

@@ -111,11 +111,7 @@ class TransferEngine implements ITransferEngine {
       }
 
       await this.transferSchemas();
-      await this.transferEntities()
+      await this.transferEntities();
-        // Temporary while we don't have the final API for streaming data from the database
-        .catch((e) => {
-          console.log(`Could not complete the entities transfer. ${e.message}`);
-        });
       await this.transferMedia();
       await this.transferLinks();
       await this.transferConfiguration();
@@ -133,8 +129,12 @@ class TransferEngine implements ITransferEngine {
     const inStream = await this.sourceProvider.streamSchemas?.();
     const outStream = await this.destinationProvider.getSchemasStream?.();
 
-    if (!inStream || !outStream) {
+    if (!inStream) {
-      throw new Error('Unable to transfer schemas, one of the streams is missing');
+      throw new Error('Unable to transfer schemas, source stream is missing');
+    }
+
+    if (!outStream) {
+      throw new Error('Unable to transfer schemas, destination stream is missing');
     }
 
     return new Promise((resolve, reject) => {
@@ -144,9 +144,7 @@ class TransferEngine implements ITransferEngine {
 
       outStream
         // Throw on error in the destination
-        .on('error', (e) => {
+        .on('error', reject)
-          reject(e);
-        })
         // Resolve the promise when the destination has finished reading all the data from the source
         .on('close', resolve);
 
@@ -161,6 +159,7 @@ class TransferEngine implements ITransferEngine {
     if (!inStream) {
       throw new Error('Unable to transfer entities, source stream is missing');
     }
+
     if (!outStream) {
       throw new Error('Unable to transfer entities, destination stream is missing');
     }
@@ -191,6 +190,7 @@ class TransferEngine implements ITransferEngine {
     if (!inStream) {
       throw new Error('Unable to transfer links, source stream is missing');
     }
+
     if (!outStream) {
       throw new Error('Unable to transfer links, destination stream is missing');
     }
@@ -211,7 +211,7 @@ class TransferEngine implements ITransferEngine {
   }
 
   async transferMedia(): Promise<void> {
-    console.log('transferMedia not yet implemented');
+    console.warn('transferMedia not yet implemented');
     return new Promise((resolve) => resolve());
   }
 
@@ -222,6 +222,7 @@ class TransferEngine implements ITransferEngine {
     if (!inStream) {
       throw new Error('Unable to transfer configuration, source stream is missing');
     }
+
     if (!outStream) {
       throw new Error('Unable to transfer configuration, destination stream is missing');
     }

packages/core/data-transfer/lib/providers/local-file-destination-provider.ts

@@ -6,7 +6,7 @@ import { chain } from 'stream-chain';
 import { stringer } from 'stream-json/jsonl/Stringer';
 
 import type { IDestinationProvider, ProviderType, Stream } from '../../types';
-import { createCipher } from '../encryption/encrypt';
+import { createEncryptionCipher } from '../encryption/encrypt';
 
 export interface ILocalFileDestinationProviderOptions {
   // Encryption
@@ -59,7 +59,9 @@ class LocalFileDestinationProvider implements IDestinationProvider {
       if (!this.options.encryption.key) {
         throw new Error("Can't encrypt without a key");
       }
-      const cipher = createCipher(this.options.encryption.key);
+
+      const cipher = createEncryptionCipher(this.options.encryption.key);
+
       transforms.push(cipher);
     }
 

packages/core/data-transfer/lib/providers/local-file-source-provider.ts

@@ -2,9 +2,11 @@ import fs from 'fs';
 import zip from 'zlib';
 import tar from 'tar';
 import { chain } from 'stream-chain';
-import { pipeline, Duplex } from 'stream';
+import { pipeline, PassThrough } from 'stream';
 import { parser } from 'stream-json/jsonl/Parser';
 
+import { createDecryptionCipher } from '../encryption';
+
 import { IMetadata, ISourceProvider, ProviderType } from '../../types';
 
 type StreamItemArray = Parameters<typeof chain>[0];
@@ -80,6 +82,10 @@ class LocalFileSourceProvider implements ISourceProvider {
     return this.#parseJSONFile<IMetadata>(backupStream, METADATA_FILE_PATH);
   }
 
+  streamSchemas(): NodeJS.ReadableStream {
+    return this.#streamJsonlDirectory('schemas');
+  }
+
   streamEntities(): NodeJS.ReadableStream {
     return this.#streamJsonlDirectory('entities');
   }
@@ -107,9 +113,10 @@ class LocalFileSourceProvider implements ISourceProvider {
   }
 
   #streamJsonlDirectory(directory: string) {
+    const options = this.options;
     const inStream = this.#getBackupStream();
 
-    const outStream = new Duplex();
+    const outStream = new PassThrough({ objectMode: true });
 
     pipeline(
       [
@@ -130,17 +137,26 @@ class LocalFileSourceProvider implements ISourceProvider {
           },
 
           onentry(entry) {
-            const transforms = chain([
+            const transforms = [];
-              // TODO: Add the decryption transform stream before parsing each line
+
+            if (options.encrypted) {
+              transforms.push(createDecryptionCipher(options.encryptionKey!));
+            }
+
+            if (options.compressed) {
+              transforms.push(zip.createGunzip());
+            }
+
+            transforms.push(
               // JSONL parser to read the data chunks one by one (line by line)
               parser(),
               // The JSONL parser returns each line as key/value
-              (line: { key: string; value: any }) => line.value,
+              (line: { key: string; value: any }) => line.value
-            ]);
+            );
 
             entry
               // Pipe transforms
-              .pipe(transforms)
+              .pipe(chain(transforms))
               // Pipe the out stream to the whole pipeline
               // DO NOT send the 'end' event when this entry has finished
               // emitting data, so that it doesn't close the out stream