diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index a0a14daa7d..ebdf90af24 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -110,6 +110,7 @@ class TransferEngine implements ITransferEngine { ); } + await this.transferSchemas(); await this.transferEntities() // Temporary while we don't have the final API for streaming data from the database .catch((e) => { @@ -128,6 +129,31 @@ class TransferEngine implements ITransferEngine { } } + async transferSchemas(): Promise { + const inStream = await this.sourceProvider.streamSchemas?.(); + const outStream = await this.destinationProvider.getSchemasStream?.(); + console.log(inStream); + if (!inStream || !outStream) { + throw new Error('Unable to transfer schemas, one of the streams is missing'); + } + + return new Promise((resolve, reject) => { + inStream + // Throw on error in the source + .on('error', reject); + + outStream + // Throw on error in the destination + .on('error', (e) => { + reject(e); + }) + // Resolve the promise when the destination has finished reading all the data from the source + .on('close', resolve); + + inStream.pipe(outStream); + }); + } + async transferEntities(): Promise { const inStream = await this.sourceProvider.streamEntities?.(); const outStream = await this.destinationProvider.getEntitiesStream?.(); diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts index 9eab55dfa6..4a18689490 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts @@ -4,7 +4,6 @@ import zip from 'zlib'; import { Duplex } from 'stream'; import { chain, Writable } from 'stream-chain'; import { stringer } from 'stream-json/jsonl/Stringer'; -import type { Cipher } from 'crypto'; import type { IDestinationProvider, ProviderType, Stream } from '../../types'; import { createCipher } from '../encryption/encrypt'; @@ -52,6 +51,7 @@ class LocalFileDestinationProvider implements IDestinationProvider { } fs.mkdirSync(rootDir, { recursive: true }); + fs.mkdirSync(path.join(rootDir, 'schemas')); fs.mkdirSync(path.join(rootDir, 'entities')); fs.mkdirSync(path.join(rootDir, 'links')); fs.mkdirSync(path.join(rootDir, 'media')); @@ -66,6 +66,39 @@ class LocalFileDestinationProvider implements IDestinationProvider { return null; } + getSchemasStream() { + const filePathFactory = (fileIndex: number = 0) => { + return path.join( + // Backup path + this.options.file.path, + // "schemas/" directory + 'schemas', + // "schemas_00000.jsonl" file + `schemas_${String(fileIndex).padStart(5, '0')}.jsonl` + ); + }; + + const streams: any[] = [ + // create jsonl strings from object entities + stringer(), + ]; + + // Compression + if (this.options.compression?.enabled) { + streams.push(zip.createGzip()); + } + + // Encryption; + if (this.options.encryption?.enabled) { + streams.push(createCipher(this.options.encryption.key)); + } + + // FS write stream + streams.push(createMultiFilesWriteStream(filePathFactory, this.options.file?.maxSize)); + + return chain(streams); + } + getEntitiesStream(): Duplex { const filePathFactory = (fileIndex: number = 0) => { return path.join( diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts index 7dd70a1556..94b5a495eb 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts @@ -120,4 +120,86 @@ describe('Local Strapi Source Provider', () => { }); }); }); + + describe('Streaming Schemas', () => { + test('Should successfully create a readable stream with all Schemas', async () => { + const contentTypes = { + foo: { uid: 'foo', attributes: { title: { type: 'string' } } }, + bar: { uid: 'bar', attributes: { age: { type: 'number' } } }, + }; + + const components ={ + 'basic.simple': { + collectionName: 'components_basic_simples', + info: { displayName: 'simple', icon: 'ambulance', description: '' }, + options: {}, + attributes: { name: {type:'string'} }, + uid: 'basic.simple', + category: 'basic', + modelType: 'component', + modelName: 'simple', + globalId: 'ComponentBasicSimple' + }, + 'blog.test-como': { + collectionName: 'components_blog_test_comos', + info: { + displayName: 'test comp', + icon: 'air-freshener', + description: '' + }, + options: {}, + attributes: { name: { type: 'string' } }, + uid: 'blog.test-como', + category: 'blog', + modelType: 'component', + modelName: 'test-como', + globalId: 'ComponentBlogTestComo' + }, + } + + const provider = createLocalStrapiSourceProvider({ getStrapi: getStrapiFactory({ + contentTypes, + components + }) }); + + await provider.bootstrap(); + + const schemasStream = provider.streamSchemas() as Readable; + const schemas = await collect(schemasStream); + + expect(schemasStream).toBeInstanceOf(Readable); + expect(schemas).toHaveLength(4); + + expect(schemas).toEqual([ + { uid: 'foo', attributes: { title: { type: 'string' } } }, + { uid: 'bar', attributes: { age: { type: 'number' } } }, + { + collectionName: 'components_basic_simples', + info: { displayName: 'simple', icon: 'ambulance', description: '' }, + options: {}, + attributes: { name: {type:'string'} }, + uid: 'basic.simple', + category: 'basic', + modelType: 'component', + modelName: 'simple', + globalId: 'ComponentBasicSimple' + }, + { + collectionName: 'components_blog_test_comos', + info: { + displayName: 'test comp', + icon: 'air-freshener', + description: '' + }, + options: {}, + attributes: { name: { type: 'string' } }, + uid: 'blog.test-como', + category: 'blog', + modelType: 'component', + modelName: 'test-como', + globalId: 'ComponentBlogTestComo' + } + ]) + }); + }) }); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts index 61afa58a3e..7777b99905 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts @@ -1,7 +1,7 @@ import type { ISourceProvider, ProviderType } from '../../../types'; import { chain } from 'stream-chain'; - +import { Readable } from 'stream'; import { createEntitiesStream, createEntitiesTransformStream } from './entities'; import { createLinksStream } from './links'; @@ -64,4 +64,16 @@ class LocalStrapiSourceProvider implements ISourceProvider { return createLinksStream(this.strapi); } + + getSchemas() { + if (!this.strapi) { + throw new Error('Not able to get Schemas. Strapi instance not found'); + } + + return [...Object.values(this.strapi.contentTypes), ...Object.values(this.strapi.components)]; + } + + streamSchemas(): NodeJS.ReadableStream { + return Readable.from(this.getSchemas()); + } } diff --git a/packages/core/data-transfer/types/providers.d.ts b/packages/core/data-transfer/types/providers.d.ts index ab4564bb30..b7b0813555 100644 --- a/packages/core/data-transfer/types/providers.d.ts +++ b/packages/core/data-transfer/types/providers.d.ts @@ -19,6 +19,8 @@ export interface ISourceProvider extends IProvider { streamLinks?(): NodeJS.ReadableStream | Promise; streamMedia?(): NodeJS.ReadableStream | Promise; streamConfiguration?(): NodeJS.ReadableStream | Promise; + getSchemas?(): any; + streamSchemas?(): NodeJS.ReadableStream; } export interface IDestinationProvider extends IProvider { @@ -32,4 +34,5 @@ export interface IDestinationProvider extends IProvider { getLinksStream?(): NodeJS.WritableStream | Promise; getMediaStream?(): NodeJS.WritableStream | Promise; getConfigurationStream?(): NodeJS.WritableStream | Promise; + getSchemasStream?(): NodeJS.WritableStream | Promise; } diff --git a/packages/core/data-transfer/types/transfer-engine.d.ts b/packages/core/data-transfer/types/transfer-engine.d.ts index c96c789b1d..4df639ebcd 100644 --- a/packages/core/data-transfer/types/transfer-engine.d.ts +++ b/packages/core/data-transfer/types/transfer-engine.d.ts @@ -48,6 +48,12 @@ export interface ITransferEngine { */ close(): Promise; + /** + * Start the schemas transfer by connecting the + * related source and destination providers streams + */ + transferSchemas(): Promise; + /** * Start the entities transfer by connecting the * related source and destination providers streams