From dba3f9ba9c53aa7dd9a95f40ed12d4355cf83ced Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 21 Nov 2022 17:03:15 +0100 Subject: [PATCH] Fix the gzip compression --- .../local-file-destination-provider.ts | 169 +++++++++++------- .../strapi/lib/commands/transfer/export.js | 2 +- 2 files changed, 101 insertions(+), 70 deletions(-) diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts index 0749a53744..e684e53304 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider.ts @@ -3,6 +3,7 @@ import type { IDestinationProviderTransferResults, IMetadata, ProviderType, + Stream, } from '../../types'; import fs from 'fs-extra'; @@ -55,70 +56,12 @@ class LocalFileDestinationProvider implements IDestinationProvider { results: ILocalFileDestinationProviderTransferResults = {}; #providersMetadata: { source?: IMetadata; destination?: IMetadata } = {}; - #archive?: tar.Pack; + #archive: { stream?: tar.Pack } = {}; constructor(options: ILocalFileDestinationProviderOptions) { this.options = options; } - setMetadata(target: ProviderType, metadata: IMetadata): IDestinationProvider { - this.#providersMetadata[target] = metadata; - - return this; - } - - #getDataTransformers(options: { jsonl?: boolean } = {}) { - const { jsonl = true } = options; - const transforms = []; - - if (jsonl) { - // Convert to stringified JSON lines - transforms.push(stringer()); - } - - return transforms; - } - - bootstrap(): void | Promise { - const { compression, encryption } = this.options; - - if (encryption.enabled && !encryption.key) { - throw new Error("Can't encrypt without a key"); - } - - this.#archive = tar.pack(); - - const outStream = fs.createWriteStream(this.#archivePath); - - const archiveTransforms = []; - - if (compression.enabled) { - archiveTransforms.push(zlib.createGzip()); - } - - if (encryption.enabled && encryption.key) { - archiveTransforms.push(createEncryptionCipher(encryption.key)); - } - - chain([this.#archive, ...archiveTransforms, outStream]); - - this.results.file = { path: this.#archivePath }; - } - - async close() { - await this.#writeMetadata(); - this.#archive?.finalize(); - } - - async rollback(): Promise { - await this.close(); - fs.rmSync(this.#archivePath, { force: true }); - } - - getMetadata() { - return null; - } - get #archivePath() { const { encryption, compression, file } = this.options; @@ -135,6 +78,77 @@ class LocalFileDestinationProvider implements IDestinationProvider { return path; } + setMetadata(target: ProviderType, metadata: IMetadata): IDestinationProvider { + this.#providersMetadata[target] = metadata; + + return this; + } + + #getDataTransformers(options: { jsonl?: boolean } = {}) { + const { jsonl = true } = options; + const transforms: Stream[] = []; + + if (jsonl) { + // Convert to stringified JSON lines + transforms.push(stringer()); + } + + return transforms; + } + + bootstrap(): void | Promise { + const { compression, encryption } = this.options; + + if (encryption.enabled && !encryption.key) { + throw new Error("Can't encrypt without a key"); + } + + this.#archive.stream = tar.pack(); + + const outStream = fs.createWriteStream(this.#archivePath); + + const archiveTransforms: Stream[] = []; + + if (compression.enabled) { + archiveTransforms.push(zlib.createGzip()); + } + + if (encryption.enabled && encryption.key) { + archiveTransforms.push(createEncryptionCipher(encryption.key)); + } + + chain([this.#archive.stream, ...archiveTransforms, outStream]); + + this.results.file = { path: this.#archivePath }; + } + + async close() { + const { stream } = this.#archive; + + if (!stream) { + return; + } + + await this.#writeMetadata(); + + stream.finalize(); + + if (!stream.closed) { + await new Promise((resolve, reject) => { + stream.on('close', resolve).on('error', reject); + }); + } + } + + async rollback(): Promise { + await this.close(); + fs.rmSync(this.#archivePath, { force: true }); + } + + getMetadata() { + return null; + } + async #writeMetadata(): Promise { const metadata = this.#providersMetadata.source; @@ -149,15 +163,24 @@ class LocalFileDestinationProvider implements IDestinationProvider { } #getMetadataStream() { - return createTarEntryStream(this.#archive!, () => 'metadata.json'); + const { stream } = this.#archive; + + if (!stream) { + throw new Error('Archive stream is unavailable'); + } + + return createTarEntryStream(stream, () => 'metadata.json'); } getSchemasStream() { + if (!this.#archive.stream) { + throw new Error('Archive stream is unavailable'); + } + const filePathFactory = createFilePathFactory('schemas'); - // FS write stream const entryStream = createTarEntryStream( - this.#archive!, + this.#archive.stream, filePathFactory, this.options.file.maxSize ); @@ -166,12 +189,14 @@ class LocalFileDestinationProvider implements IDestinationProvider { } getEntitiesStream(): NodeJS.WritableStream { - // const filePathFactory = createFilePathFactory(this.options.file.path, 'entities'); + if (!this.#archive.stream) { + throw new Error('Archive stream is unavailable'); + } + const filePathFactory = createFilePathFactory('entities'); - // FS write stream const entryStream = createTarEntryStream( - this.#archive!, + this.#archive.stream, filePathFactory, this.options.file.maxSize ); @@ -180,11 +205,14 @@ class LocalFileDestinationProvider implements IDestinationProvider { } getLinksStream(): NodeJS.WritableStream { + if (!this.#archive.stream) { + throw new Error('Archive stream is unavailable'); + } + const filePathFactory = createFilePathFactory('links'); - // FS write stream const entryStream = createTarEntryStream( - this.#archive!, + this.#archive.stream, filePathFactory, this.options.file.maxSize ); @@ -193,11 +221,14 @@ class LocalFileDestinationProvider implements IDestinationProvider { } getConfigurationStream(): NodeJS.WritableStream { + if (!this.#archive.stream) { + throw new Error('Archive stream is unavailable'); + } + const filePathFactory = createFilePathFactory('configuration'); - // FS write stream const entryStream = createTarEntryStream( - this.#archive!, + this.#archive.stream, filePathFactory, this.options.file.maxSize ); diff --git a/packages/core/strapi/lib/commands/transfer/export.js b/packages/core/strapi/lib/commands/transfer/export.js index 6ac7b40f71..d869db36e8 100644 --- a/packages/core/strapi/lib/commands/transfer/export.js +++ b/packages/core/strapi/lib/commands/transfer/export.js @@ -147,7 +147,7 @@ module.exports = async (filename, opts) => { logger.log(table.toString()); // TODO: once archiving is implemented, we need to check file extensions - if (!fs.pathExistsSync(file)) { + if (!fs.pathExistsSync(results.destination.file.path)) { logger.log(file); throw new Error('Export file not created'); }