diff --git a/packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts b/packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts index 33b69e312c..c9c3e50496 100644 --- a/packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts +++ b/packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts @@ -1,8 +1,8 @@ -import { createCipher } from '../encrypt'; +import { createEncryptionCipher } from '..'; describe('Encryption', () => { test('encrypting data with default algorithm aes-128-ecb', () => { - const cipher = createCipher('password'); + const cipher = createEncryptionCipher('password'); const textToEncrypt = 'something ate an apple'; const encryptedData = cipher.update(textToEncrypt); @@ -12,7 +12,7 @@ describe('Encryption', () => { }); test('encrypting data with aes128', () => { - const cipher = createCipher('password', 'aes128'); + const cipher = createEncryptionCipher('password', 'aes128'); const textToEncrypt = 'something ate an apple'; const encryptedData = cipher.update(textToEncrypt); @@ -22,7 +22,7 @@ describe('Encryption', () => { }); test('encrypting data with aes192', () => { - const cipher = createCipher('password', 'aes192'); + const cipher = createEncryptionCipher('password', 'aes192'); const textToEncrypt = 'something ate an apple'; const encryptedData = cipher.update(textToEncrypt); @@ -32,7 +32,7 @@ describe('Encryption', () => { }); test('encrypting data with aes256', () => { - const cipher = createCipher('password', 'aes256'); + const cipher = createEncryptionCipher('password', 'aes256'); const textToEncrypt = 'something ate an apple'; const encryptedData = cipher.update(textToEncrypt); @@ -42,9 +42,9 @@ describe('Encryption', () => { }); test('data encrypted with different algorithms should have different results', () => { - const cipherAES256 = createCipher('password', 'aes256'); - const cipherAES192 = createCipher('password', 'aes192'); - const cipherDefault = createCipher('password'); + const cipherAES256 = createEncryptionCipher('password', 'aes256'); + const cipherAES192 = createEncryptionCipher('password', 'aes192'); + const cipherDefault = createEncryptionCipher('password'); const textToEncrypt = 'something ate an apple'; const encryptedDataAES256 = cipherAES256.update(textToEncrypt).toString(); const encryptedDataAES192 = cipherAES192.update(textToEncrypt).toString(); @@ -56,8 +56,8 @@ describe('Encryption', () => { }); test('data encrypted with different key should be different', () => { - const cipher1 = createCipher('password'); - const cipher2 = createCipher('differentpassword'); + const cipher1 = createEncryptionCipher('password'); + const cipher2 = createEncryptionCipher('differentpassword'); const textToEncrypt = 'something ate an apple'; const encryptedData1 = cipher1.update(textToEncrypt).toString(); const encryptedData2 = cipher2.update(textToEncrypt).toString(); diff --git a/packages/core/data-transfer/lib/encryption/decrypt.ts b/packages/core/data-transfer/lib/encryption/decrypt.ts new file mode 100644 index 0000000000..2babd377e0 --- /dev/null +++ b/packages/core/data-transfer/lib/encryption/decrypt.ts @@ -0,0 +1,41 @@ +import { Cipher, scryptSync, CipherKey, BinaryLike, createDecipheriv } from 'crypto'; +import { EncryptionStrategy, Strategies, Algorithm } from '../../types'; + +// different key values depending on algorithm chosen +const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { + const strategies: Strategies = { + 'aes-128-ecb': (key: string): Cipher => { + const hashedKey = scryptSync(key, '', 16); + const initVector: BinaryLike | null = null; + const securityKey: CipherKey = hashedKey; + return createDecipheriv(algorithm, securityKey, initVector); + }, + aes128: (key: string): Cipher => { + const hashedKey = scryptSync(key, '', 32); + const initVector: BinaryLike | null = hashedKey.slice(16); + const securityKey: CipherKey = hashedKey.slice(0, 16); + return createDecipheriv(algorithm, securityKey, initVector); + }, + aes192: (key: string): Cipher => { + const hashedKey = scryptSync(key, '', 40); + const initVector: BinaryLike | null = hashedKey.slice(24); + const securityKey: CipherKey = hashedKey.slice(0, 24); + return createDecipheriv(algorithm, securityKey, initVector); + }, + aes256: (key: string): Cipher => { + const hashedKey = scryptSync(key, '', 48); + const initVector: BinaryLike | null = hashedKey.slice(32); + const securityKey: CipherKey = hashedKey.slice(0, 32); + return createDecipheriv(algorithm, securityKey, initVector); + }, + }; + + return strategies[algorithm]; +}; + +export const createDecryptionCipher = ( + key: string, + algorithm: Algorithm = 'aes-128-ecb' +): Cipher => { + return getDecryptionStrategy(algorithm)(key); +}; diff --git a/packages/core/data-transfer/lib/encryption/encrypt.ts b/packages/core/data-transfer/lib/encryption/encrypt.ts index 42049d6248..523fecf537 100644 --- a/packages/core/data-transfer/lib/encryption/encrypt.ts +++ b/packages/core/data-transfer/lib/encryption/encrypt.ts @@ -33,6 +33,9 @@ const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { return strategies[algorithm]; }; -export const createCipher = (key: string, algorithm: Algorithm = 'aes-128-ecb'): Cipher => { +export const createEncryptionCipher = ( + key: string, + algorithm: Algorithm = 'aes-128-ecb' +): Cipher => { return getEncryptionStrategy(algorithm)(key); }; diff --git a/packages/core/data-transfer/lib/encryption/index.ts b/packages/core/data-transfer/lib/encryption/index.ts new file mode 100644 index 0000000000..04a109a35c --- /dev/null +++ b/packages/core/data-transfer/lib/encryption/index.ts @@ -0,0 +1,2 @@ +export * from './encrypt'; +export * from './decrypt'; diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index 26cdd62d90..75511a5b56 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -111,11 +111,7 @@ class TransferEngine implements ITransferEngine { } await this.transferSchemas(); - await this.transferEntities() - // Temporary while we don't have the final API for streaming data from the database - .catch((e) => { - console.log(`Could not complete the entities transfer. ${e.message}`); - }); + await this.transferEntities(); await this.transferMedia(); await this.transferLinks(); await this.transferConfiguration(); @@ -133,8 +129,12 @@ class TransferEngine implements ITransferEngine { const inStream = await this.sourceProvider.streamSchemas?.(); const outStream = await this.destinationProvider.getSchemasStream?.(); - if (!inStream || !outStream) { - throw new Error('Unable to transfer schemas, one of the streams is missing'); + if (!inStream) { + throw new Error('Unable to transfer schemas, source stream is missing'); + } + + if (!outStream) { + throw new Error('Unable to transfer schemas, destination stream is missing'); } return new Promise((resolve, reject) => { @@ -144,9 +144,7 @@ class TransferEngine implements ITransferEngine { outStream // Throw on error in the destination - .on('error', (e) => { - reject(e); - }) + .on('error', reject) // Resolve the promise when the destination has finished reading all the data from the source .on('close', resolve); @@ -161,6 +159,7 @@ class TransferEngine implements ITransferEngine { if (!inStream) { throw new Error('Unable to transfer entities, source stream is missing'); } + if (!outStream) { throw new Error('Unable to transfer entities, destination stream is missing'); } @@ -191,6 +190,7 @@ class TransferEngine implements ITransferEngine { if (!inStream) { throw new Error('Unable to transfer links, source stream is missing'); } + if (!outStream) { throw new Error('Unable to transfer links, destination stream is missing'); } @@ -211,7 +211,7 @@ class TransferEngine implements ITransferEngine { } async transferMedia(): Promise { - console.log('transferMedia not yet implemented'); + console.warn('transferMedia not yet implemented'); return new Promise((resolve) => resolve()); } @@ -222,6 +222,7 @@ class TransferEngine implements ITransferEngine { if (!inStream) { throw new Error('Unable to transfer configuration, source stream is missing'); } + if (!outStream) { throw new Error('Unable to transfer configuration, destination stream is missing'); } diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts index 4b8b34f383..2f13bfbadd 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts @@ -6,7 +6,7 @@ import { chain } from 'stream-chain'; import { stringer } from 'stream-json/jsonl/Stringer'; import type { IDestinationProvider, ProviderType, Stream } from '../../types'; -import { createCipher } from '../encryption/encrypt'; +import { createEncryptionCipher } from '../encryption/encrypt'; export interface ILocalFileDestinationProviderOptions { // Encryption @@ -59,7 +59,9 @@ class LocalFileDestinationProvider implements IDestinationProvider { if (!this.options.encryption.key) { throw new Error("Can't encrypt without a key"); } - const cipher = createCipher(this.options.encryption.key); + + const cipher = createEncryptionCipher(this.options.encryption.key); + transforms.push(cipher); } diff --git a/packages/core/data-transfer/lib/providers/local-file-source-provider.ts b/packages/core/data-transfer/lib/providers/local-file-source-provider.ts index 666e4287a4..5d732e726c 100644 --- a/packages/core/data-transfer/lib/providers/local-file-source-provider.ts +++ b/packages/core/data-transfer/lib/providers/local-file-source-provider.ts @@ -2,9 +2,11 @@ import fs from 'fs'; import zip from 'zlib'; import tar from 'tar'; import { chain } from 'stream-chain'; -import { pipeline, Duplex } from 'stream'; +import { pipeline, PassThrough } from 'stream'; import { parser } from 'stream-json/jsonl/Parser'; +import { createDecryptionCipher } from '../encryption'; + import { IMetadata, ISourceProvider, ProviderType } from '../../types'; type StreamItemArray = Parameters[0]; @@ -80,6 +82,10 @@ class LocalFileSourceProvider implements ISourceProvider { return this.#parseJSONFile(backupStream, METADATA_FILE_PATH); } + streamSchemas(): NodeJS.ReadableStream { + return this.#streamJsonlDirectory('schemas'); + } + streamEntities(): NodeJS.ReadableStream { return this.#streamJsonlDirectory('entities'); } @@ -107,9 +113,10 @@ class LocalFileSourceProvider implements ISourceProvider { } #streamJsonlDirectory(directory: string) { + const options = this.options; const inStream = this.#getBackupStream(); - const outStream = new Duplex(); + const outStream = new PassThrough({ objectMode: true }); pipeline( [ @@ -130,17 +137,26 @@ class LocalFileSourceProvider implements ISourceProvider { }, onentry(entry) { - const transforms = chain([ - // TODO: Add the decryption transform stream before parsing each line + const transforms = []; + + if (options.encrypted) { + transforms.push(createDecryptionCipher(options.encryptionKey!)); + } + + if (options.compressed) { + transforms.push(zip.createGunzip()); + } + + transforms.push( // JSONL parser to read the data chunks one by one (line by line) parser(), // The JSONL parser returns each line as key/value - (line: { key: string; value: any }) => line.value, - ]); + (line: { key: string; value: any }) => line.value + ); entry // Pipe transforms - .pipe(transforms) + .pipe(chain(transforms)) // Pipe the out stream to the whole pipeline // DO NOT send the 'end' event when this entry has finished // emitting data, so that it doesn't close the out stream