diff --git a/packages/core/data-transfer/src/engine/diagnostic.ts b/packages/core/data-transfer/src/engine/diagnostic.ts new file mode 100644 index 0000000000..7f303d9ab2 --- /dev/null +++ b/packages/core/data-transfer/src/engine/diagnostic.ts @@ -0,0 +1,130 @@ +import { EventEmitter } from 'events'; + +export interface IDiagnosticReporterOptions { + stackSize?: number; +} + +export type GenericDiagnostic = { + kind: K; + details: { + message: string; + createdAt: Date; + } & T; +}; + +export type DiagnosticKind = 'error' | 'warning' | 'info'; + +export type DiagnosticListener = ( + diagnostic: { kind: T } & Diagnostic extends infer U ? U : 'foo' +) => void | Promise; + +export type DiagnosticEvent = 'diagnostic' | `diagnostic.${DiagnosticKind}`; + +export type GetEventListener = E extends 'diagnostic' + ? DiagnosticListener + : E extends `diagnostic.${infer K}` + ? K extends DiagnosticKind + ? DiagnosticListener + : never + : never; + +export type Diagnostic = ErrorDiagnostic | WarningDiagnostic | InfoDiagnostic; + +export type ErrorDiagnosticSeverity = 'fatal' | 'error' | 'silly'; + +export type ErrorDiagnostic = GenericDiagnostic< + 'error', + { + name: string; + severity: ErrorDiagnosticSeverity; + error: Error; + } +>; + +export type WarningDiagnostic = GenericDiagnostic< + 'warning', + { + origin?: string; + } +>; + +export type InfoDiagnostic = GenericDiagnostic< + 'info', + { + params?: T; + } +>; + +export interface IDiagnosticReporter { + stack: { + readonly size: number; + readonly items: Diagnostic[]; + }; + + report(diagnostic: Diagnostic): IDiagnosticReporter; + onDiagnostic(listener: DiagnosticListener): IDiagnosticReporter; + on(kind: T, listener: DiagnosticListener): IDiagnosticReporter; +} + +const createDiagnosticReporter = ( + options: IDiagnosticReporterOptions = {} +): IDiagnosticReporter => { + const { stackSize = -1 } = options; + + const emitter = new EventEmitter(); + const stack: Diagnostic[] = []; + + const addListener = (event: T, listener: GetEventListener) => { + emitter.on(event, listener); + }; + + const isDiagnosticValid = (diagnostic: Diagnostic) => { + if (!diagnostic.kind || !diagnostic.details || !diagnostic.details.message) { + return false; + } + return true; + }; + + return { + stack: { + get size() { + return stack.length; + }, + + get items() { + return stack; + }, + }, + + report(diagnostic: Diagnostic) { + if (!isDiagnosticValid(diagnostic)) { + return this; + } + + emitter.emit('diagnostic', diagnostic); + emitter.emit(`diagnostic.${diagnostic.kind}`, diagnostic); + + if (stackSize !== -1 && stack.length >= stackSize) { + stack.shift(); + } + + stack.push(diagnostic); + + return this; + }, + + onDiagnostic(listener: DiagnosticListener) { + addListener('diagnostic', listener); + + return this; + }, + + on(kind: T, listener: DiagnosticListener) { + addListener(`diagnostic.${kind}`, listener as never); + + return this; + }, + }; +}; + +export { createDiagnosticReporter }; diff --git a/packages/core/data-transfer/src/engine/errors.ts b/packages/core/data-transfer/src/engine/errors.ts new file mode 100644 index 0000000000..799dbc63d4 --- /dev/null +++ b/packages/core/data-transfer/src/engine/errors.ts @@ -0,0 +1,46 @@ +import { DataTransferError, Severity, SeverityKind } from '../errors'; + +type TransferEngineStep = 'initialization' | 'validation' | 'transfer'; + +type TransferEngineErrorDetails

= { + step: P; +} & ([U] extends [never] ? unknown : { details?: U }); + +class TransferEngineError< + P extends TransferEngineStep = TransferEngineStep, + U = never, + T extends TransferEngineErrorDetails = TransferEngineErrorDetails +> extends DataTransferError { + constructor(severity: Severity, message?: string, details?: T | null) { + super('engine', severity, message, details); + } +} + +class TransferEngineInitializationError extends TransferEngineError<'initialization'> { + constructor(message?: string) { + super(SeverityKind.FATAL, message, { step: 'initialization' }); + } +} + +class TransferEngineValidationError< + T extends { check: string } = { check: string } +> extends TransferEngineError<'validation', T> { + constructor(message?: string, details?: T) { + super(SeverityKind.FATAL, message, { step: 'validation', details }); + } +} + +class TransferEngineTransferError< + T extends { check: string } = { check: string } +> extends TransferEngineError<'transfer', T> { + constructor(message?: string, details?: T) { + super(SeverityKind.FATAL, message, { step: 'transfer', details }); + } +} + +export { + TransferEngineError, + TransferEngineInitializationError, + TransferEngineValidationError, + TransferEngineTransferError, +}; diff --git a/packages/core/data-transfer/src/engine/index.ts b/packages/core/data-transfer/src/engine/index.ts index 088134d431..5d04e6f932 100644 --- a/packages/core/data-transfer/src/engine/index.ts +++ b/packages/core/data-transfer/src/engine/index.ts @@ -1,6 +1,7 @@ import { PassThrough, Transform, Readable, Writable, Stream } from 'stream'; import { extname } from 'path'; -import { isEmpty, uniq } from 'lodash/fp'; +import { EOL } from 'os'; +import { isEmpty, uniq, last } from 'lodash/fp'; import { diff as semverDiff } from 'semver'; import type { Schema } from '@strapi/strapi'; @@ -16,14 +17,23 @@ import type { ITransferResults, TransferStage, TransferTransform, + IProvider, TransferFilters, TransferFilterPreset, } from '../../types'; import type { Diff } from '../utils/json'; -import { compareSchemas } from './validation/schemas'; +import { compareSchemas, validateProvider } from './validation'; import { filter, map } from '../utils/stream'; +import { TransferEngineValidationError } from './errors'; +import { + createDiagnosticReporter, + IDiagnosticReporter, + ErrorDiagnosticSeverity, +} from './diagnostic'; +import { DataTransferError } from '../errors'; + export const TRANSFER_STAGES: ReadonlyArray = Object.freeze([ 'entities', 'links', @@ -83,17 +93,14 @@ class TransferEngine< stream: PassThrough; }; - constructor( - sourceProvider: ISourceProvider, - destinationProvider: IDestinationProvider, - options: ITransferEngineOptions - ) { - if (sourceProvider.type !== 'source') { - throw new Error("SourceProvider does not have type 'source'"); - } - if (destinationProvider.type !== 'destination') { - throw new Error("DestinationProvider does not have type 'destination'"); - } + diagnostics: IDiagnosticReporter; + + constructor(sourceProvider: S, destinationProvider: D, options: ITransferEngineOptions) { + this.diagnostics = createDiagnosticReporter(); + + validateProvider('source', sourceProvider); + validateProvider('destination', destinationProvider); + this.sourceProvider = sourceProvider; this.destinationProvider = destinationProvider; this.options = options; @@ -101,6 +108,56 @@ class TransferEngine< this.progress = { data: {}, stream: new PassThrough({ objectMode: true }) }; } + /** + * Report a fatal error and throw it + */ + #panic(error: Error) { + this.#reportError(error, 'fatal'); + + throw error; + } + + /** + * Report an error diagnostic + */ + #reportError(error: Error, severity: ErrorDiagnosticSeverity) { + this.diagnostics.report({ + kind: 'error', + details: { + severity, + createdAt: new Date(), + name: error.name, + message: error.message, + error, + }, + }); + } + + /** + * Report a warning diagnostic + */ + #reportWarning(message: string, origin?: string) { + this.diagnostics.report({ + kind: 'warning', + details: { createdAt: new Date(), message, origin }, + }); + } + + /** + * Report an info diagnostic + */ + #reportInfo(message: string, params?: unknown) { + this.diagnostics.report({ + kind: 'info', + details: { createdAt: new Date(), message, params }, + }); + } + + /** + * Create and return a transform stream based on the given stage and options. + * + * Allowed transformations includes 'filter' and 'map'. + */ #createStageTransformStream( key: T, options: { includeGlobal?: boolean } = {} @@ -131,6 +188,11 @@ class TransferEngine< return stream; } + /** + * Update the Engine's transfer progress data for a given stage. + * + * Providing aggregate options enable custom computation to get the size (bytes) or the aggregate key associated with the data + */ #updateTransferProgress( stage: TransferStage, data: T, @@ -172,6 +234,11 @@ class TransferEngine< } } + /** + * Create and return a PassThrough stream. + * + * Upon writing data into it, it'll update the Engine's transfer progress data and trigger stage update events. + */ #progressTracker( stage: TransferStage, aggregate?: { @@ -189,10 +256,16 @@ class TransferEngine< }); } + /** + * Shorthand method used to trigger transfer update events to every listeners + */ #emitTransferUpdate(type: 'init' | 'start' | 'finish' | 'error', payload?: object) { this.progress.stream.emit(`transfer::${type}`, payload); } + /** + * Shorthand method used to trigger stage update events to every listeners + */ #emitStageUpdate(type: 'start' | 'finish' | 'progress' | 'skip', transferStage: TransferStage) { this.progress.stream.emit(`stage::${type}`, { data: this.progress.data, @@ -200,9 +273,25 @@ class TransferEngine< }); } + /** + * Run a version check between two strapi version (source and destination) using the strategy given to the engine during initialization. + * + * If there is a mismatch, throws a validation error. + */ #assertStrapiVersionIntegrity(sourceVersion?: string, destinationVersion?: string) { const strategy = this.options.versionStrategy || DEFAULT_VERSION_STRATEGY; + const reject = () => { + throw new TransferEngineValidationError( + `The source and destination provide are targeting incompatible Strapi versions (using the "${strategy}" strategy). The source (${this.sourceProvider.name}) version is ${sourceVersion} and the destination (${this.destinationProvider.name}) version is ${destinationVersion}`, + { + check: 'strapi.version', + strategy, + versions: { source: sourceVersion, destination: destinationVersion }, + } + ); + }; + if ( !sourceVersion || !destinationVersion || @@ -215,11 +304,10 @@ class TransferEngine< let diff; try { diff = semverDiff(sourceVersion, destinationVersion); - } catch (e: unknown) { - throw new Error( - `Strapi versions doesn't match (${strategy} check): ${sourceVersion} does not match with ${destinationVersion}` - ); + } catch { + reject(); } + if (!diff) { return; } @@ -237,13 +325,17 @@ class TransferEngine< return; } - throw new Error( - `Strapi versions doesn't match (${strategy} check): ${sourceVersion} does not match with ${destinationVersion}` - ); + reject(); } + /** + * Run a check between two set of schemas (source and destination) using the strategy given to the engine during initialization. + * + * If there are differences and/or incompatibilities between source and destination schemas, then throw a validation error. + */ #assertSchemasMatching(sourceSchemas: SchemaMap, destinationSchemas: SchemaMap) { const strategy = this.options.schemaStrategy || DEFAULT_SCHEMA_STRATEGY; + if (strategy === 'ignore') { return; } @@ -254,7 +346,7 @@ class TransferEngine< keys.forEach((key) => { const sourceSchema = sourceSchemas[key]; const destinationSchema = destinationSchemas[key]; - const schemaDiffs = compareSchemas(sourceSchema, destinationSchema, strategy); + const schemaDiffs = compareSchemas(destinationSchema, sourceSchema, strategy); if (schemaDiffs.length) { diffs[key] = schemaDiffs; @@ -262,10 +354,45 @@ class TransferEngine< }); if (!isEmpty(diffs)) { - throw new Error( - `Import process failed because the project doesn't have a matching data structure - ${JSON.stringify(diffs, null, 2)} - ` + const formattedDiffs = Object.entries(diffs) + .map(([uid, ctDiffs]) => { + let msg = `- ${uid}:${EOL}`; + + msg += ctDiffs + .sort((a, b) => (a.kind > b.kind ? -1 : 1)) + .map((diff) => { + const path = diff.path.join('.'); + + if (diff.kind === 'added') { + return `Added "${path}": "${diff.value}" (${diff.type})`; + } + + if (diff.kind === 'deleted') { + return `Removed "${path}"`; + } + + if (diff.kind === 'modified') { + return `Modified "${path}": "${diff.values[0]}" (${diff.types[0]}) => "${diff.values[1]}" (${diff.types[1]})`; + } + + throw new TransferEngineValidationError(`Invalid diff found for "${uid}"`, { + check: `schema on ${uid}`, + }); + }) + .map((line) => ` - ${line}`) + .join(EOL); + + return msg; + }) + .join(EOL); + + throw new TransferEngineValidationError( + `Invalid schema changes detected during integrity checks (using the ${strategy} strategy). Please find a summary of the changes below:\n${formattedDiffs}`, + { + check: 'schema.changes', + strategy, + diffs, + } ); } } @@ -308,7 +435,7 @@ class TransferEngine< if (!source || !destination || this.shouldSkipStage(stage)) { // Wait until source and destination are closed - await Promise.allSettled( + const results = await Promise.allSettled( [source, destination].map((stream) => { // if stream is undefined or already closed, resolve immediately if (!stream || stream.destroyed) { @@ -322,6 +449,12 @@ class TransferEngine< }) ); + results.forEach((state) => { + if (state.status === 'rejected') { + this.#reportWarning(state.reason, `transfer(${stage})`); + } + }); + this.#emitStageUpdate('skip', stage); return; @@ -340,7 +473,15 @@ class TransferEngine< stream = stream.pipe(tracker); } - stream.pipe(destination).on('error', reject).on('close', resolve); + stream + .pipe(destination) + .on('error', (e) => { + // TODO ? + // this.#reportError(e, 'error'); + // destination.destroy(e); + reject(e); + }) + .on('close', resolve); }); this.#emitStageUpdate('finish', stage); @@ -359,12 +500,36 @@ class TransferEngine< } } + /** + * Run the bootstrap method in both source and destination providers + */ async bootstrap(): Promise { - await Promise.all([this.sourceProvider.bootstrap?.(), this.destinationProvider.bootstrap?.()]); + const results = await Promise.allSettled([ + this.sourceProvider.bootstrap?.(), + this.destinationProvider.bootstrap?.(), + ]); + + results.forEach((result) => { + if (result.status === 'rejected') { + this.#panic(result.reason); + } + }); } + /** + * Run the close method in both source and destination providers + */ async close(): Promise { - await Promise.all([this.sourceProvider.close?.(), this.destinationProvider.close?.()]); + const results = await Promise.allSettled([ + this.sourceProvider.close?.(), + this.destinationProvider.close?.(), + ]); + + results.forEach((result) => { + if (result.status === 'rejected') { + this.#panic(result.reason); + } + }); } async #resolveProviderResource() { @@ -380,7 +545,7 @@ class TransferEngine< } } - async integrityCheck(): Promise { + async integrityCheck() { try { const sourceMetadata = await this.sourceProvider.getMetadata(); const destinationMetadata = await this.destinationProvider.getMetadata(); @@ -398,10 +563,12 @@ class TransferEngine< if (sourceSchemas && destinationSchemas) { this.#assertSchemasMatching(sourceSchemas, destinationSchemas); } - - return true; } catch (error) { - return false; + if (error instanceof Error) { + this.#panic(error); + } + + throw error; } } @@ -413,17 +580,13 @@ class TransferEngine< this.#emitTransferUpdate('init'); await this.bootstrap(); await this.init(); - const isValidTransfer = await this.integrityCheck(); - if (!isValidTransfer) { - // TODO: provide the log from the integrity check - throw new Error( - `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` - ); - } + + await this.integrityCheck(); this.#emitTransferUpdate('start'); await this.beforeTransfer(); + // Run the transfer stages await this.transferSchemas(); await this.transferEntities(); @@ -437,9 +600,19 @@ class TransferEngine< } catch (e: unknown) { this.#emitTransferUpdate('error', { error: e }); + const lastDiagnostic = last(this.diagnostics.stack.items); + // Do not report an error diagnostic if the last one reported the same error + if ( + e instanceof Error && + (!lastDiagnostic || lastDiagnostic.kind !== 'error' || lastDiagnostic.details.error !== e) + ) { + this.#reportError(e, (e as DataTransferError).severity || 'fatal'); + } + // Rollback the destination provider if an exception is thrown during the transfer // Note: This will be configurable in the future await this.destinationProvider.rollback?.(e as Error); + throw e; } @@ -451,8 +624,23 @@ class TransferEngine< } async beforeTransfer(): Promise { - await this.sourceProvider.beforeTransfer?.(); - await this.destinationProvider.beforeTransfer?.(); + const runWithDiagnostic = async (provider: IProvider) => { + try { + await provider.beforeTransfer?.(); + } catch (error) { + // Error happening during the before transfer step should be considered fatal errors + if (error instanceof Error) { + this.#panic(error); + } else { + this.#panic( + new Error(`Unknwon error when executing "beforeTransfer" on the ${origin} provider`) + ); + } + } + }; + + await runWithDiagnostic(this.sourceProvider); + await runWithDiagnostic(this.destinationProvider); } async transferSchemas(): Promise { @@ -500,7 +688,7 @@ class TransferEngine< const transform = this.#createStageTransformStream(stage); const tracker = this.#progressTracker(stage, { size: (value: IAsset) => value.stats.size, - key: (value: IAsset) => extname(value.filename), + key: (value: IAsset) => extname(value.filename) ?? 'NA', }); await this.#transferStage({ stage, source, destination, transform, tracker }); @@ -519,10 +707,7 @@ class TransferEngine< } } -export const createTransferEngine = < - S extends ISourceProvider = ISourceProvider, - D extends IDestinationProvider = IDestinationProvider ->( +export const createTransferEngine = ( sourceProvider: S, destinationProvider: D, options: ITransferEngineOptions diff --git a/packages/core/data-transfer/src/engine/validation/index.ts b/packages/core/data-transfer/src/engine/validation/index.ts index 197f1788ef..9fc174d463 100644 --- a/packages/core/data-transfer/src/engine/validation/index.ts +++ b/packages/core/data-transfer/src/engine/validation/index.ts @@ -1 +1,2 @@ -export * as schemas from './schemas'; +export * from './schemas'; +export * from './provider'; diff --git a/packages/core/data-transfer/src/engine/validation/provider.ts b/packages/core/data-transfer/src/engine/validation/provider.ts new file mode 100644 index 0000000000..b5d95eeece --- /dev/null +++ b/packages/core/data-transfer/src/engine/validation/provider.ts @@ -0,0 +1,27 @@ +import { capitalize } from 'lodash/fp'; + +import type { IDestinationProvider, ISourceProvider, ProviderType } from '../../../types'; +import { TransferEngineValidationError } from '../errors'; + +const reject = (reason: string): never => { + throw new TransferEngineValidationError(`Invalid provider supplied. ${reason}`); +}; + +const validateProvider = ( + type: ProviderType, + provider?: ([T] extends ['source'] ? ISourceProvider : IDestinationProvider) | null +) => { + if (!provider) { + return reject( + `Expected an instance of "${capitalize(type)}Provider", but got "${typeof provider}" instead.` + ); + } + + if (provider.type !== type) { + return reject( + `Expected the provider to be of type "${type}" but got "${provider.type}" instead.` + ); + } +}; + +export { validateProvider }; diff --git a/packages/core/data-transfer/src/errors/base.ts b/packages/core/data-transfer/src/errors/base.ts new file mode 100644 index 0000000000..1d3987a39f --- /dev/null +++ b/packages/core/data-transfer/src/errors/base.ts @@ -0,0 +1,19 @@ +import { Severity } from './constants'; + +class DataTransferError extends Error { + origin: string; + + severity: Severity; + + details: T | null; + + constructor(origin: string, severity: Severity, message?: string, details?: T | null) { + super(message); + + this.origin = origin; + this.severity = severity; + this.details = details ?? null; + } +} + +export { DataTransferError }; diff --git a/packages/core/data-transfer/src/errors/constants.ts b/packages/core/data-transfer/src/errors/constants.ts new file mode 100644 index 0000000000..cae73c6e9a --- /dev/null +++ b/packages/core/data-transfer/src/errors/constants.ts @@ -0,0 +1,8 @@ +import { ErrorDiagnosticSeverity } from '../engine/diagnostic'; + +export const SeverityKind: Record = { + FATAL: 'fatal', + ERROR: 'error', + SILLY: 'silly', +} as const; +export type Severity = typeof SeverityKind[keyof typeof SeverityKind]; diff --git a/packages/core/data-transfer/src/errors/index.ts b/packages/core/data-transfer/src/errors/index.ts new file mode 100644 index 0000000000..41bf5eaffb --- /dev/null +++ b/packages/core/data-transfer/src/errors/index.ts @@ -0,0 +1,2 @@ +export * from './constants'; +export * from './base'; diff --git a/packages/core/data-transfer/src/errors/providers.ts b/packages/core/data-transfer/src/errors/providers.ts new file mode 100644 index 0000000000..d84b85083e --- /dev/null +++ b/packages/core/data-transfer/src/errors/providers.ts @@ -0,0 +1,40 @@ +import { DataTransferError } from './base'; +import { Severity, SeverityKind } from './constants'; + +type ProviderStep = 'initialization' | 'validation' | 'transfer'; + +type ProviderErrorDetails

= { + step: P; +} & ([U] extends [never] ? unknown : { details?: U }); + +export class ProviderError< + P extends ProviderStep = ProviderStep, + U = never, + T extends ProviderErrorDetails = ProviderErrorDetails +> extends DataTransferError { + constructor(severity: Severity, message?: string, details?: T | null) { + super('provider', severity, message, details); + } +} + +export class ProviderInitializationError extends ProviderError<'initialization'> { + constructor(message?: string) { + super(SeverityKind.FATAL, message, { step: 'initialization' }); + } +} + +// TODO: these types are not working correctly, ProviderTransferError() is accepting any details object rather than requiring T +export class ProviderValidationError extends ProviderError< + 'validation', + T +> { + constructor(message?: string, details?: T) { + super(SeverityKind.SILLY, message, { step: 'validation', details }); + } +} +// TODO: these types are not working correctly, ProviderTransferError() is accepting any details object rather than requiring T +export class ProviderTransferError extends ProviderError<'transfer', T> { + constructor(message?: string, details?: T) { + super(SeverityKind.FATAL, message, { step: 'transfer', details }); + } +} diff --git a/packages/core/data-transfer/src/strapi/__tests__/register.test.ts b/packages/core/data-transfer/src/strapi/__tests__/register.test.ts index 9063b60c19..fc5902845a 100644 --- a/packages/core/data-transfer/src/strapi/__tests__/register.test.ts +++ b/packages/core/data-transfer/src/strapi/__tests__/register.test.ts @@ -2,7 +2,7 @@ import { getStrapiFactory } from '../../__tests__/test-utils'; import { createTransferHandler } from '../remote/handlers'; import register from '../register'; -import { TRANSFER_PATH } from '../../../lib/strapi/remote/constants'; +import { TRANSFER_PATH } from '../remote/constants'; afterEach(() => { jest.clearAllMocks(); diff --git a/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts index 5db7765384..820451686f 100644 --- a/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts @@ -11,6 +11,8 @@ import type { import { restore } from './strategies'; import * as utils from '../../../utils'; +import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers'; +import { assertValidStrapi } from '../../../utils/providers'; export const VALID_CONFLICT_STRATEGIES = ['restore', 'merge']; export const DEFAULT_CONFLICT_STRATEGY = 'restore'; @@ -62,15 +64,16 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { #validateOptions() { if (!VALID_CONFLICT_STRATEGIES.includes(this.options.strategy)) { - throw new Error(`Invalid stategy ${this.options.strategy}`); + throw new ProviderValidationError(`Invalid strategy ${this.options.strategy}`, { + check: 'strategy', + strategy: this.options.strategy, + validStrategies: VALID_CONFLICT_STRATEGIES, + }); } } async #deleteAll() { - if (!this.strapi) { - throw new Error('Strapi instance not found'); - } - + assertValidStrapi(this.strapi); return restore.deleteRecords(this.strapi, this.options.restore); } @@ -114,10 +117,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { } getSchemas() { - if (!this.strapi) { - throw new Error('Not able to get Schemas. Strapi instance not found'); - } - + assertValidStrapi(this.strapi, 'Not able to get Schemas'); const schemas = { ...this.strapi.contentTypes, ...this.strapi.components, @@ -127,10 +127,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { } createEntitiesWriteStream(): Writable { - if (!this.strapi) { - throw new Error('Not able to import entities. Strapi instance not found'); - } - + assertValidStrapi(this.strapi, 'Not able to import entities'); const { strategy } = this.options; const updateMappingTable = (type: string, oldID: number, newID: number) => { @@ -149,14 +146,16 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { }); } - throw new Error(`Invalid strategy supplied: "${strategy}"`); + throw new ProviderValidationError(`Invalid strategy ${this.options.strategy}`, { + check: 'strategy', + strategy: this.options.strategy, + validStrategies: VALID_CONFLICT_STRATEGIES, + }); } // TODO: Move this logic to the restore strategy async createAssetsWriteStream(): Promise { - if (!this.strapi) { - throw new Error('Not able to stream Assets. Strapi instance not found'); - } + assertValidStrapi(this.strapi, 'Not able to stream Assets'); const assetsDirectory = path.join(this.strapi.dirs.static.public, 'uploads'); const backupDirectory = path.join( @@ -185,12 +184,12 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { await fse.rm(assetsDirectory, { recursive: true, force: true }); await fse.rename(backupDirectory, assetsDirectory); this.destroy( - new Error( + new ProviderTransferError( `There was an error during the transfer process. The original files have been restored to ${assetsDirectory}` ) ); } catch (err) { - throw new Error( + throw new ProviderTransferError( `There was an error doing the rollback process. The original files are in ${backupDirectory}, but we failed to restore them to ${assetsDirectory}` ); } finally { @@ -202,9 +201,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { } async createConfigurationWriteStream(): Promise { - if (!this.strapi) { - throw new Error('Not able to stream Configurations. Strapi instance not found'); - } + assertValidStrapi(this.strapi, 'Not able to stream Configurations'); const { strategy } = this.options; @@ -212,7 +209,11 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { return restore.createConfigurationWriteStream(this.strapi, this.transaction); } - throw new Error(`Invalid strategy supplied: "${strategy}"`); + throw new ProviderValidationError(`Invalid strategy ${strategy}`, { + check: 'strategy', + strategy, + validStrategies: VALID_CONFLICT_STRATEGIES, + }); } async createLinksWriteStream(): Promise { @@ -227,7 +228,11 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { return restore.createLinksWriteStream(mapID, this.strapi, this.transaction); } - throw new Error(`Invalid strategy supplied: "${strategy}"`); + throw new ProviderValidationError(`Invalid strategy ${strategy}`, { + check: 'strategy', + strategy, + validStrategies: VALID_CONFLICT_STRATEGIES, + }); } } diff --git a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts index 0961b834d5..dde7750d0b 100644 --- a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts @@ -1,6 +1,7 @@ import { omit } from 'lodash/fp'; import { Writable } from 'stream'; import chalk from 'chalk'; +import { ProviderTransferError } from '../../../../../errors/providers'; import { IConfiguration, Transaction } from '../../../../../../types'; const omitInvalidCreationAttributes = omit(['id']); @@ -45,7 +46,13 @@ export const createConfigurationWriteStream = async ( try { await restoreConfigs(strapi, config); } catch (error) { - return callback(new Error(`Failed to import ${config.type} ${error}`)); + return callback( + new ProviderTransferError( + `Failed to import ${chalk.yellowBright(config.type)} (${chalk.greenBright( + config.value.id + )}` + ) + ); } callback(); }); diff --git a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts index ea12cdad16..700f632cd8 100644 --- a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts @@ -3,6 +3,7 @@ import type { SchemaUID } from '@strapi/strapi/lib/types/utils'; import { get, last } from 'lodash/fp'; import { Writable } from 'stream'; +import { ProviderTransferError } from '../../../../../errors/providers'; import type { IEntity, Transaction } from '../../../../../../types'; import { json } from '../../../../../utils'; import * as queries from '../../../../queries'; @@ -92,7 +93,7 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { return callback(e); } - return callback(new Error(`Failed to create "${type}" (${id})`)); + return callback(new ProviderTransferError(`Failed to create "${type}" (${id})`)); } return callback(null); diff --git a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts index 90f9505fdb..3d4808a17c 100644 --- a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts @@ -1,4 +1,5 @@ import type { ContentTypeSchema } from '@strapi/strapi'; +import { ProviderTransferError } from '../../../../../errors/providers'; import * as queries from '../../../../queries'; export interface IRestoreOptions { @@ -116,7 +117,7 @@ const useResults = ( const update = (count: number, key?: string) => { if (key) { if (!(key in results.aggregate)) { - throw new Error(`Unknown key "${key}" provided in results update`); + throw new ProviderTransferError(`Unknown key "${key}" provided in results update`); } results.aggregate[key].count += count; diff --git a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts index 2d33ff083d..e7db4dcd25 100644 --- a/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts @@ -1,4 +1,5 @@ import { Writable } from 'stream'; +import { ProviderTransferError } from '../../../../../errors/providers'; import { ILink, Transaction } from '../../../../../../types'; import { createLinkQuery } from '../../../../queries/link'; @@ -26,9 +27,12 @@ export const createLinksWriteStream = ( } return callback( - new Error(`An error happened while trying to import a ${left.type} link. ${e}`) + new ProviderTransferError( + `An error happened while trying to import a ${left.type} link.` + ) ); } + callback(null); }); }, diff --git a/packages/core/data-transfer/src/strapi/providers/local-source/index.ts b/packages/core/data-transfer/src/strapi/providers/local-source/index.ts index cc3e07dd8f..b1401c967f 100644 --- a/packages/core/data-transfer/src/strapi/providers/local-source/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/index.ts @@ -7,6 +7,7 @@ import { createLinksStream } from './links'; import { createConfigurationStream } from './configuration'; import { createAssetsStream } from './assets'; import * as utils from '../../../utils'; +import { assertValidStrapi } from '../../../utils/providers'; export interface ILocalStrapiSourceProviderOptions { getStrapi(): Strapi.Strapi | Promise; @@ -64,9 +65,7 @@ class LocalStrapiSourceProvider implements ISourceProvider { } async createEntitiesReadStream(): Promise { - if (!this.strapi) { - throw new Error('Not able to stream entities. Strapi instance not found'); - } + assertValidStrapi(this.strapi, 'Not able to stream entities'); return chain([ // Entities stream @@ -78,25 +77,19 @@ class LocalStrapiSourceProvider implements ISourceProvider { } createLinksReadStream(): Readable { - if (!this.strapi) { - throw new Error('Not able to stream links. Strapi instance not found'); - } + assertValidStrapi(this.strapi, 'Not able to stream links'); return createLinksStream(this.strapi); } createConfigurationReadStream(): Readable { - if (!this.strapi) { - throw new Error('Not able to stream configuration. Strapi instance not found'); - } + assertValidStrapi(this.strapi, 'Not able to stream configuration'); return createConfigurationStream(strapi); } getSchemas() { - if (!this.strapi) { - throw new Error('Not able to get Schemas. Strapi instance not found'); - } + assertValidStrapi(this.strapi, 'Not able to get Schemas'); const schemas = { ...this.strapi.contentTypes, @@ -111,9 +104,7 @@ class LocalStrapiSourceProvider implements ISourceProvider { } createAssetsReadStream(): Readable { - if (!this.strapi) { - throw new Error('Not able to stream assets. Strapi instance not found'); - } + assertValidStrapi(this.strapi, 'Not able to stream assets'); return createAssetsStream(this.strapi); } diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts index 4f60192a56..613d88af31 100644 --- a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts @@ -1,5 +1,5 @@ import { WebSocket } from 'ws'; -import { TRANSFER_PATH } from '../../../../../lib/strapi/remote/constants'; +import { TRANSFER_PATH } from '../../../remote/constants'; import { CommandMessage } from '../../../../../types/remote/protocol/client'; import { createDispatcher } from '../utils'; diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts index a83b27c6be..8987a650e6 100644 --- a/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts @@ -16,6 +16,7 @@ import type { import type { client, server } from '../../../../types/remote/protocol'; import type { ILocalStrapiDestinationProviderOptions } from '../local-destination'; import { TRANSFER_PATH } from '../../remote/constants'; +import { ProviderTransferError, ProviderValidationError } from '../../../errors/providers'; interface ITokenAuth { type: 'token'; @@ -66,7 +67,9 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { const res = (await query) as server.Payload; if (!res?.transferID) { - return reject(new Error('Init failed, invalid response from the server')); + return reject( + new ProviderTransferError('Init failed, invalid response from the server') + ); } resolve(res.transferID); @@ -87,10 +90,10 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { } if (typeof e === 'string') { - return new Error(e); + return new ProviderTransferError(e); } - return new Error('Unexpected error'); + return new ProviderTransferError('Unexpected error'); } return null; @@ -98,14 +101,22 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { async bootstrap(): Promise { const { url, auth } = this.options; + const validProtocols = ['https:', 'http:']; let ws: WebSocket; - if (!['https:', 'http:'].includes(url.protocol)) { - throw new Error(`Invalid protocol "${url.protocol}"`); + if (!validProtocols.includes(url.protocol)) { + throw new ProviderValidationError(`Invalid protocol "${url.protocol}"`, { + check: 'url', + details: { + protocol: url.protocol, + validProtocols, + }, + }); } const wsProtocol = url.protocol === 'https:' ? 'wss:' : 'ws:'; const wsUrl = `${wsProtocol}//${url.host}${url.pathname}${TRANSFER_PATH}`; + const validAuthMethods = ['token']; // No auth defined, trying public access for transfer if (!auth) { @@ -120,7 +131,13 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { // Invalid auth method provided else { - throw new Error('Auth method not implemented'); + throw new ProviderValidationError('Auth method not implemented', { + check: 'auth.type', + details: { + auth: auth.type, + validAuthMethods, + }, + }); } this.ws = ws; diff --git a/packages/core/data-transfer/src/strapi/remote/constants.ts b/packages/core/data-transfer/src/strapi/remote/constants.ts index 536be055a2..4b8400ece4 100644 --- a/packages/core/data-transfer/src/strapi/remote/constants.ts +++ b/packages/core/data-transfer/src/strapi/remote/constants.ts @@ -1 +1,2 @@ export const TRANSFER_PATH = '/transfer'; +export const TRANSFER_METHODS = ['push', 'pull']; diff --git a/packages/core/data-transfer/src/strapi/remote/handlers.ts b/packages/core/data-transfer/src/strapi/remote/handlers.ts index f6c9315b21..f0d91dc144 100644 --- a/packages/core/data-transfer/src/strapi/remote/handlers.ts +++ b/packages/core/data-transfer/src/strapi/remote/handlers.ts @@ -8,6 +8,8 @@ import type { IPushController } from './controllers/push'; import createPushController from './controllers/push'; import type { client, server } from '../../../types/remote/protocol'; +import { ProviderTransferError, ProviderInitializationError } from '../../errors/providers'; +import { TRANSFER_METHODS } from './constants'; interface ITransferState { transfer?: { id: string; kind: client.TransferKind }; @@ -68,9 +70,13 @@ export const createTransferHandler = if (e instanceof Error) { callback(e); } else if (typeof e === 'string') { - callback(new Error(e)); + callback(new ProviderTransferError(e)); } else { - callback(new Error('Unexpected error')); + callback( + new ProviderTransferError('Unexpected error', { + error: e, + }) + ); } } }; @@ -83,8 +89,9 @@ export const createTransferHandler = }; const init = (msg: client.InitCommand): server.Payload => { + // TODO: this only checks for this instance of node: we should consider a database lock if (state.controller) { - throw new Error('Transfer already in progres'); + throw new ProviderInitializationError('Transfer already in progres'); } const { transfer } = msg.params; @@ -102,7 +109,10 @@ export const createTransferHandler = // Pull or any other string else { - throw new Error(`Transfer not implemented: "${transfer}"`); + throw new ProviderTransferError(`Transfer type not implemented: "${transfer}"`, { + transfer, + validTransfers: TRANSFER_METHODS, + }); } state.transfer = { id: randomUUID(), kind: transfer }; @@ -125,7 +135,12 @@ export const createTransferHandler = } if (command === 'status') { - await callback(new Error('Command not implemented: "status"')); + await callback( + new ProviderTransferError('Command not implemented: "status"', { + command, + validCommands: ['init', 'end', 'status'], + }) + ); } }; @@ -137,15 +152,15 @@ export const createTransferHandler = // It shouldn't be possible to strart a pull transfer for now, so reaching // this code should be impossible too, but this has been added by security if (state.transfer?.kind === 'pull') { - return callback(new Error('Pull transfer not implemented')); + return callback(new ProviderTransferError('Pull transfer not implemented')); } if (!controller) { - return callback(new Error("The transfer hasn't been initialized")); + return callback(new ProviderTransferError("The transfer hasn't been initialized")); } if (!transferID) { - return callback(new Error('Missing transfer ID')); + return callback(new ProviderTransferError('Missing transfer ID')); } // Action @@ -153,7 +168,12 @@ export const createTransferHandler = const { action } = msg; if (!(action in controller.actions)) { - return callback(new Error(`Invalid action provided: "${action}"`)); + return callback( + new ProviderTransferError(`Invalid action provided: "${action}"`, { + action, + validActions: Object.keys(controller.actions), + }) + ); } await answer(() => controller.actions[action as keyof typeof controller.actions]()); @@ -187,6 +207,7 @@ export const createTransferHandler = ws.on('error', (e) => { teardown(); + // TODO: is logging a console error to the running instance of Strapi ok to do? Should we check for an existing strapi.logger to use? console.error(e); }); @@ -194,7 +215,8 @@ export const createTransferHandler = const msg: client.Message = JSON.parse(raw.toString()); if (!msg.uuid) { - throw new Error('Missing uuid in message'); + await callback(new ProviderTransferError('Missing uuid in message')); + return; } uuid = msg.uuid; @@ -211,7 +233,7 @@ export const createTransferHandler = // Invalid messages else { - await callback(new Error('Bad request')); + await callback(new ProviderTransferError('Bad request')); } }); }); diff --git a/packages/core/data-transfer/src/utils/providers.ts b/packages/core/data-transfer/src/utils/providers.ts new file mode 100644 index 0000000000..a180d74aa2 --- /dev/null +++ b/packages/core/data-transfer/src/utils/providers.ts @@ -0,0 +1,12 @@ +import { ProviderInitializationError } from '../errors/providers'; + +export type ValidStrapiAssertion = ( + strapi: unknown, + msg?: string +) => asserts strapi is Strapi.Strapi; + +export const assertValidStrapi: ValidStrapiAssertion = (strapi?: unknown, msg = '') => { + if (!strapi) { + throw new ProviderInitializationError(`${msg}. Strapi instance not found.`); + } +}; diff --git a/packages/core/data-transfer/types/transfer-engine.d.ts b/packages/core/data-transfer/types/transfer-engine.d.ts index 61f7a70928..9ffea8ff4a 100644 --- a/packages/core/data-transfer/types/transfer-engine.d.ts +++ b/packages/core/data-transfer/types/transfer-engine.d.ts @@ -1,7 +1,12 @@ +import type { PassThrough } from 'stream'; import type { IAsset, IEntity, ILink } from './common-entities'; import type { ITransferResults, TransferTransform, TransferTransforms } from './utils'; import type { ISourceProvider, IDestinationProvider } from './providers'; import type { Schema } from '@strapi/strapi'; +import type { ITransferResults, TransferTransform, TransferProgress } from './utils'; +import type { ISourceProvider, IDestinationProvider } from './providers'; +import type { Severity } from '../src/errors'; +import type { DiagnosticReporter } from '../src/engine/diagnostic'; export type TransferFilterPreset = 'content' | 'files' | 'config'; @@ -24,6 +29,18 @@ export interface ITransferEngine< * The options used to customize the behavio of the transfer engine */ options: ITransferEngineOptions; + /** + * A diagnostic reporter instance used to gather information about + * errors, warnings and information emitted by the engine + */ + diagnostics: DiagnosticReporter; + /** + * Utilities used to retrieve transfer progress data + */ + progress: { + data: TransferProgress; + stream: PassThrough; + }; /** * Runs the integrity check which will make sure it's possible @@ -31,7 +48,7 @@ export interface ITransferEngine< * * Note: It requires to read the content of the source & destination metadata files */ - integrityCheck(): Promise; + integrityCheck(): Promise; /** * Start streaming selected data from the source to the destination diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index 4a02b2c315..42119a1c44 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -28,6 +28,10 @@ describe('Export', () => { }, sourceProvider: { name: 'testSource' }, destinationProvider: { name: 'testDestination' }, + diagnostics: { + on: jest.fn().mockReturnThis(), + onDiagnostic: jest.fn().mockReturnThis(), + }, }; }, }, @@ -39,6 +43,7 @@ describe('Export', () => { // mock utils const mockUtils = { + formatDiagnostic: jest.fn(), createStrapiInstance() { return { telemetry: { diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/import.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/import.test.js index 7e2c466537..80471d071c 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/import.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/import.test.js @@ -23,6 +23,10 @@ const createTransferEngine = jest.fn(() => { type: 'destination', getMetadata: jest.fn(), }, + diagnostics: { + on: jest.fn().mockReturnThis(), + onDiagnostic: jest.fn().mockReturnThis(), + }, }; }); @@ -53,6 +57,7 @@ describe('Import', () => { // mock utils const mockUtils = { + formatDiagnostic: jest.fn(), createStrapiInstance: jest.fn().mockReturnValue({ telemetry: { send: jest.fn(), @@ -70,10 +75,10 @@ describe('Import', () => { ); // other spies - jest.spyOn(console, 'log').mockImplementation(() => {}); - jest.spyOn(console, 'warn').mockImplementation(() => {}); - jest.spyOn(console, 'info').mockImplementation(() => {}); - jest.spyOn(console, 'error').mockImplementation(() => {}); + // jest.spyOn(console, 'log').mockImplementation(() => {}); + // jest.spyOn(console, 'warn').mockImplementation(() => {}); + // jest.spyOn(console, 'info').mockImplementation(() => {}); + // jest.spyOn(console, 'error').mockImplementation(() => {}); // Now that everything is mocked, load the 'import' command const importCommand = require('../../transfer/import'); diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/shared/transfer.test.utils.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/shared/transfer.test.utils.js index 575bba6f2a..3b91627f6a 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/shared/transfer.test.utils.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/shared/transfer.test.utils.js @@ -6,7 +6,7 @@ const expectExit = async (code, fn) => { }); await expect(async () => { await fn(); - }).rejects.toThrow(); + }).rejects.toThrow('process.exit'); expect(exit).toHaveBeenCalledWith(code); exit.mockRestore(); }; diff --git a/packages/core/strapi/lib/commands/transfer/export.js b/packages/core/strapi/lib/commands/transfer/export.js index ce55e881c9..6197a8e062 100644 --- a/packages/core/strapi/lib/commands/transfer/export.js +++ b/packages/core/strapi/lib/commands/transfer/export.js @@ -11,11 +11,13 @@ const { isObject, isString, isFinite, toNumber } = require('lodash/fp'); const fs = require('fs-extra'); const chalk = require('chalk'); +const { TransferEngineTransferError } = require('@strapi/data-transfer/lib/engine/errors'); const { getDefaultExportName, buildTransferTable, DEFAULT_IGNORED_CONTENT_TYPES, createStrapiInstance, + formatDiagnostic, } = require('./utils'); /** @@ -76,6 +78,8 @@ module.exports = async (opts) => { }, }); + engine.diagnostics.onDiagnostic(formatDiagnostic('export')); + const progress = engine.progress.stream; const getTelemetryPayload = (/* payload */) => { @@ -101,14 +105,14 @@ module.exports = async (opts) => { const outFileExists = await fs.pathExists(outFile); if (!outFileExists) { - throw new Error(`Export file not created "${outFile}"`); + throw new TransferEngineTransferError(`Export file not created "${outFile}"`); } logger.log(`${chalk.bold('Export process has been completed successfully!')}`); logger.log(`Export archive is in ${chalk.green(outFile)}`); - } catch (e) { + } catch { await strapi.telemetry.send('didDEITSProcessFail', getTelemetryPayload()); - logger.error('Export process failed unexpectedly:', e.toString()); + logger.error('Export process failed'); process.exit(1); } diff --git a/packages/core/strapi/lib/commands/transfer/import.js b/packages/core/strapi/lib/commands/transfer/import.js index 4e1f68870a..b2dbc96bf1 100644 --- a/packages/core/strapi/lib/commands/transfer/import.js +++ b/packages/core/strapi/lib/commands/transfer/import.js @@ -18,6 +18,7 @@ const { buildTransferTable, DEFAULT_IGNORED_CONTENT_TYPES, createStrapiInstance, + formatDiagnostic, } = require('./utils'); /** @@ -86,6 +87,8 @@ module.exports = async (opts) => { const engine = createTransferEngine(source, destination, engineOptions); + engine.diagnostics.onDiagnostic(formatDiagnostic('import')); + const progress = engine.progress.stream; const getTelemetryPayload = () => { return { diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index d97a6b9829..2c79057e66 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -15,6 +15,7 @@ const { buildTransferTable, createStrapiInstance, DEFAULT_IGNORED_CONTENT_TYPES, + formatDiagnostic, } = require('./utils'); const logger = console; @@ -109,6 +110,8 @@ module.exports = async (opts) => { }, }); + engine.diagnostics.onDiagnostic(formatDiagnostic('transfer')); + try { logger.log(`Starting transfer...`); diff --git a/packages/core/strapi/lib/commands/transfer/utils.js b/packages/core/strapi/lib/commands/transfer/utils.js index 64e13f6217..f0a8dee0fa 100644 --- a/packages/core/strapi/lib/commands/transfer/utils.js +++ b/packages/core/strapi/lib/commands/transfer/utils.js @@ -4,6 +4,11 @@ const chalk = require('chalk'); const Table = require('cli-table3'); const { Option } = require('commander'); const { TransferGroupPresets } = require('@strapi/data-transfer/lib/engine'); + +const { + configs: { createOutputFileConfiguration }, + createLogger, +} = require('@strapi/logger'); const { readableBytes, exitWith } = require('../utils/helpers'); const strapi = require('../../index'); const { getParseListWithChoices } = require('../utils/commander'); @@ -121,6 +126,55 @@ const validateExcludeOnly = (command) => { } }; +const errorColors = { + fatal: chalk.red, + error: chalk.red, + silly: chalk.yellow, +}; + +const formatDiagnostic = + (operation) => + ({ details, kind }) => { + const logger = createLogger( + createOutputFileConfiguration(`${operation}_error_log_${Date.now()}.log`) + ); + try { + if (kind === 'error') { + const { message, severity = 'fatal', error, details: moreDetails } = details; + + const detailsInfo = error ?? moreDetails; + let errorMessage = errorColors[severity](`[${severity.toUpperCase()}] ${message}`); + if (detailsInfo && detailsInfo.details) { + const { + origin, + details: { step, details: stepDetails, ...moreInfo }, + } = detailsInfo; + errorMessage = `${errorMessage}. Thrown at ${origin} during ${step}.\n`; + if (stepDetails || moreInfo) { + const { check, ...info } = stepDetails ?? moreInfo; + errorMessage = `${errorMessage} Check ${check ?? ''}: ${JSON.stringify(info, null, 2)}`; + } + } + + logger.error(new Error(errorMessage, error)); + } + if (kind === 'info') { + const { message, params } = details; + + const msg = `${message}\n${params ? JSON.stringify(params, null, 2) : ''}`; + + logger.info(msg); + } + if (kind === 'warning') { + const { origin, message } = details; + + logger.warn(`(${origin ?? 'transfer'}) ${message}`); + } + } catch (err) { + logger.error(err); + } + }; + module.exports = { buildTransferTable, getDefaultExportName, @@ -129,4 +183,5 @@ module.exports = { excludeOption, onlyOption, validateExcludeOnly, + formatDiagnostic, }; diff --git a/packages/utils/logger/lib/default-configuration.js b/packages/utils/logger/lib/configs/default-configuration.js similarity index 73% rename from packages/utils/logger/lib/default-configuration.js rename to packages/utils/logger/lib/configs/default-configuration.js index 39c7f91047..b2005d24f4 100644 --- a/packages/utils/logger/lib/default-configuration.js +++ b/packages/utils/logger/lib/configs/default-configuration.js @@ -1,8 +1,8 @@ 'use strict'; const { transports } = require('winston'); -const { LEVEL_LABEL, LEVELS } = require('./constants'); -const { prettyPrint } = require('./formats'); +const { LEVEL_LABEL, LEVELS } = require('../constants'); +const { prettyPrint } = require('../formats'); const createDefaultConfiguration = () => { return { diff --git a/packages/utils/logger/lib/configs/index.js b/packages/utils/logger/lib/configs/index.js new file mode 100644 index 0000000000..5490935f05 --- /dev/null +++ b/packages/utils/logger/lib/configs/index.js @@ -0,0 +1,9 @@ +'use strict'; + +const createDefaultConfiguration = require('./default-configuration'); +const createOutputFileConfiguration = require('./output-file-configuration'); + +module.exports = { + createDefaultConfiguration, + createOutputFileConfiguration, +}; diff --git a/packages/utils/logger/lib/configs/output-file-configuration.js b/packages/utils/logger/lib/configs/output-file-configuration.js new file mode 100644 index 0000000000..e78a6939f7 --- /dev/null +++ b/packages/utils/logger/lib/configs/output-file-configuration.js @@ -0,0 +1,20 @@ +'use strict'; + +const { transports } = require('winston'); +const { LEVEL_LABEL, LEVELS } = require('../constants'); +const { prettyPrint } = require('../formats'); +const { excludeColors } = require('../formats'); + +const createOutputFileConfiguration = (filename) => { + return { + level: LEVEL_LABEL, + levels: LEVELS, + format: prettyPrint(), + transports: [ + new transports.Console(), + new transports.File({ level: 'error', filename, format: excludeColors }), + ], + }; +}; + +module.exports = createOutputFileConfiguration; diff --git a/packages/utils/logger/lib/formats/exclude-colors.js b/packages/utils/logger/lib/formats/exclude-colors.js new file mode 100644 index 0000000000..70ba41ae48 --- /dev/null +++ b/packages/utils/logger/lib/formats/exclude-colors.js @@ -0,0 +1,17 @@ +'use strict'; + +const { format } = require('winston'); + +/** + * This will remove the chalk color codes from the message provided. + * It's used to log plain text in the log file + */ +const excludeColors = format.printf(({ message }) => { + return message.replace( + // eslint-disable-next-line no-control-regex + /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g, + '' + ); +}); + +module.exports = excludeColors; diff --git a/packages/utils/logger/lib/formats/index.js b/packages/utils/logger/lib/formats/index.js index e23b1ef747..6a2c9ddde3 100644 --- a/packages/utils/logger/lib/formats/index.js +++ b/packages/utils/logger/lib/formats/index.js @@ -2,8 +2,10 @@ const prettyPrint = require('./pretty-print'); const levelFilter = require('./level-filter'); +const excludeColors = require('./exclude-colors'); module.exports = { prettyPrint, levelFilter, + excludeColors, }; diff --git a/packages/utils/logger/lib/index.js b/packages/utils/logger/lib/index.js index fec80cf5ba..f7de8d88bf 100644 --- a/packages/utils/logger/lib/index.js +++ b/packages/utils/logger/lib/index.js @@ -3,14 +3,19 @@ const winston = require('winston'); const formats = require('./formats'); -const createDefaultConfiguration = require('./default-configuration'); +const configs = require('./configs'); const createLogger = (userConfiguration = {}) => { - const configuration = createDefaultConfiguration(); + const configuration = configs.createDefaultConfiguration(); Object.assign(configuration, userConfiguration); return winston.createLogger(configuration); }; -module.exports = { createLogger, winston, formats }; +module.exports = { + createLogger, + winston, + formats, + configs, +};