From 4b2b00a3dd05144b1967df6ba70b01d4c0db88db Mon Sep 17 00:00:00 2001 From: Convly Date: Thu, 22 Dec 2022 12:30:45 +0100 Subject: [PATCH 01/44] Init push transfer capabilities Co-authored-by: Christian Co-authored-by: Ben Irvin --- packages/core/admin/package.json | 1 + packages/core/admin/server/register.js | 4 + .../core/data-transfer/lib/engine/index.ts | 15 + packages/core/data-transfer/lib/index.ts | 2 + .../core/data-transfer/lib/providers/index.ts | 2 + .../index.ts | 12 +- .../configuration.ts | 31 +- .../local-strapi-source-provider/entities.ts | 12 +- .../index.ts | 201 ++++++++++++ packages/core/data-transfer/lib/register.ts | 295 ++++++++++++++++++ packages/core/data-transfer/package.json | 6 +- packages/core/strapi/bin/strapi.js | 9 + .../strapi/lib/commands/transfer/transfer.js | 119 +++++++ yarn.lock | 44 +++ 14 files changed, 729 insertions(+), 24 deletions(-) create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts create mode 100644 packages/core/data-transfer/lib/register.ts create mode 100644 packages/core/strapi/lib/commands/transfer/transfer.js diff --git a/packages/core/admin/package.json b/packages/core/admin/package.json index e256921cd9..099d60ceb1 100644 --- a/packages/core/admin/package.json +++ b/packages/core/admin/package.json @@ -52,6 +52,7 @@ "@fortawesome/react-fontawesome": "^0.2.0", "@pmmmwh/react-refresh-webpack-plugin": "0.5.7", "@strapi/babel-plugin-switch-ee-ce": "4.5.4", + "@strapi/data-transfer": "4.5.4", "@strapi/design-system": "1.4.0", "@strapi/helper-plugin": "4.5.4", "@strapi/icons": "1.4.0", diff --git a/packages/core/admin/server/register.js b/packages/core/admin/server/register.js index 99d33710d9..1789abf77e 100644 --- a/packages/core/admin/server/register.js +++ b/packages/core/admin/server/register.js @@ -1,5 +1,7 @@ 'use strict'; +const { register: registerDataTransfer } = require('@strapi/data-transfer'); + const registerAdminPanelRoute = require('./routes/serve-admin-panel'); const adminAuthStrategy = require('./strategies/admin'); const apiTokenAuthStrategy = require('./strategies/api-token'); @@ -14,4 +16,6 @@ module.exports = ({ strapi }) => { if (strapi.config.serveAdminPanel) { registerAdminPanelRoute({ strapi }); } + + registerDataTransfer(strapi); }; diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index b68826422f..779f2d3fd7 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -239,6 +239,21 @@ class TransferEngine< const { stage, source, destination, transform, tracker } = options; if (!source || !destination) { + // Wait until source and destination are closed + await Promise.allSettled( + [source, destination].map((stream) => { + // if stream is undefined or already closed, resolve immediately + if (!stream || stream.destroyed) { + return Promise.resolve(); + } + + // Wait until the close event is produced and then destroy the stream and resolve + return new Promise((resolve, reject) => { + stream.on('close', resolve).on('error', reject).destroy(); + }); + }) + ); + return; } diff --git a/packages/core/data-transfer/lib/index.ts b/packages/core/data-transfer/lib/index.ts index 95411a583a..eba8ce7d72 100644 --- a/packages/core/data-transfer/lib/index.ts +++ b/packages/core/data-transfer/lib/index.ts @@ -1,2 +1,4 @@ export * from './engine'; export * from './providers'; + +export { default as register } from './register'; diff --git a/packages/core/data-transfer/lib/providers/index.ts b/packages/core/data-transfer/lib/providers/index.ts index 77df6a4bf7..04397d7349 100644 --- a/packages/core/data-transfer/lib/providers/index.ts +++ b/packages/core/data-transfer/lib/providers/index.ts @@ -5,3 +5,5 @@ export * from './local-strapi-source-provider'; // destination providers export * from './local-file-destination-provider'; export * from './local-strapi-destination-provider'; + +export * from './remote-strapi-destination-provider'; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts index 78fe4f26f3..b89e8001de 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts @@ -9,8 +9,9 @@ import * as utils from '../../utils'; export const VALID_CONFLICT_STRATEGIES = ['restore', 'merge']; export const DEFAULT_CONFLICT_STRATEGY = 'restore'; -interface ILocalStrapiDestinationProviderOptions { +export interface ILocalStrapiDestinationProviderOptions { getStrapi(): Strapi.Strapi | Promise; + autoDestroy?: boolean; restore?: restore.IRestoreOptions; strategy: 'restore' | 'merge'; } @@ -37,7 +38,12 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { } async close(): Promise { - await this.strapi?.destroy?.(); + const { autoDestroy } = this.options; + + // Basically `!== false` but more deterministic + if (autoDestroy === undefined || autoDestroy === true) { + await this.strapi?.destroy(); + } } #validateOptions() { @@ -60,7 +66,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { } } - getMetadata(): IMetadata | Promise { + getMetadata(): IMetadata { const strapiVersion = strapi.config.get('info.strapi'); const createdAt = new Date().toISOString(); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts index af9e3161b7..c956887398 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts @@ -8,24 +8,23 @@ import type { IConfiguration } from '../../../types'; * Create a readable stream that export the Strapi app configuration */ export const createConfigurationStream = (strapi: Strapi.Strapi): Readable => { - // Core Store - const coreStoreStream = chain([ - strapi.db.queryBuilder('strapi::core-store').stream(), - (data) => set('value', JSON.parse(data.value), data), - wrapConfigurationItem('core-store'), - ]); - - // Webhook - const webhooksStream = chain([ - strapi.db.queryBuilder('webhook').stream(), - wrapConfigurationItem('webhook'), - ]); - - const streams = [coreStoreStream, webhooksStream]; - - // Readable configuration stream return Readable.from( (async function* configurationGenerator(): AsyncGenerator { + // Core Store + const coreStoreStream = chain([ + strapi.db.queryBuilder('strapi::core-store').stream(), + (data) => set('value', JSON.parse(data.value), data), + wrapConfigurationItem('core-store'), + ]); + + // Webhook + const webhooksStream = chain([ + strapi.db.queryBuilder('webhook').stream(), + wrapConfigurationItem('webhook'), + ]); + + const streams = [coreStoreStream, webhooksStream]; + for (const stream of streams) { for await (const item of stream) { yield item; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts index 215efa628f..05e7bf9b4f 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts @@ -32,11 +32,15 @@ export const createEntitiesStream = (strapi: Strapi.Strapi): Readable => { contentType: ContentTypeSchema; }> { for await (const { stream, contentType } of contentTypeStreamGenerator()) { - for await (const entity of stream) { - yield { entity, contentType }; + try { + for await (const entity of stream) { + yield { entity, contentType }; + } + } catch { + // ignore + } finally { + stream.destroy(); } - - stream.destroy(); } })() ); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts new file mode 100644 index 0000000000..df2585706f --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -0,0 +1,201 @@ +import { WebSocket } from 'ws'; +import { v4 } from 'uuid'; +import { Writable } from 'stream'; + +import type { + IDestinationProvider, + IEntity, + ILink, + IMetadata, + ProviderType, + IConfiguration, + TransferStage, +} from '../../../types'; +import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; + +interface ITokenAuth { + type: 'token'; + token: string; +} + +interface ICredentialsAuth { + type: 'credentials'; + email: string; + password: string; +} + +interface IRemoteStrapiDestinationProvider { + url: string; + auth?: ITokenAuth | ICredentialsAuth; + strategy: ILocalStrapiDestinationProviderOptions['strategy']; +} + +type Actions = 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; + +export const createRemoteStrapiDestinationProvider = ( + options: IRemoteStrapiDestinationProvider +) => { + return new RemoteStrapiDestinationProvider(options); +}; + +class RemoteStrapiDestinationProvider implements IDestinationProvider { + name = 'destination::remote-strapi'; + + type: ProviderType = 'destination'; + + options: IRemoteStrapiDestinationProvider; + + ws: WebSocket | null; + + constructor(options: IRemoteStrapiDestinationProvider) { + this.options = options; + this.ws = null; + } + + async #dispatch(message: T): Promise { + const { ws } = this; + + if (!ws) { + throw new Error('No ws connection found'); + } + + return new Promise((resolve, reject) => { + const uuid = v4(); + const payload = JSON.stringify({ ...message, uuid }); + + ws.send(payload, (error) => { + if (error) { + reject(error); + } + }); + + ws.once('message', (raw) => { + const response: { uuid: string; data: U; error: string | null } = JSON.parse( + raw.toString() + ); + + if (response.error) { + return reject(new Error(response.error)); + } + + if (response.uuid === uuid) { + return resolve(response.data); + } + }); + }); + } + + async #dispatchAction(action: Actions) { + return this.#dispatch({ type: 'action', action }); + } + + async #dispatchTransfer(stage: TransferStage, data: T) { + try { + await this.#dispatch({ type: 'transfer', stage, data }); + } catch (e) { + if (e instanceof Error) { + return e; + } + + return new Error('Unexected error'); + } + + return null; + } + + async bootstrap(): Promise { + const { url, auth, strategy } = this.options; + + let ws: WebSocket; + + // No auth defined, trying public access for transfer + if (!auth) { + ws = new WebSocket(url); + } + + // Common token auth, this should be the main auth method + else if (auth.type === 'token') { + const headers = { Authentication: `Bearer ${auth.token}` }; + + ws = new WebSocket(this.options.url, { headers }); + } + + // Invalid auth method provided + else { + throw new Error('Auth method not implemented'); + } + + this.ws = ws; + + // Wait for the connection to be made to the server, then init the transfer + await new Promise((resolve, reject) => { + ws.once('open', async () => { + await this.#dispatch({ type: 'init', kind: 'push', data: { strategy } }); + resolve(); + }).once('error', reject); + }); + + // Run the bootstrap + await this.#dispatchAction('bootstrap'); + } + + async close() { + await this.#dispatchAction('close'); + + await new Promise((resolve) => { + const { ws } = this; + + if (!ws || ws.CLOSED) { + resolve(); + return; + } + + ws.on('close', () => resolve()).close(); + }); + } + + getMetadata() { + return this.#dispatchAction('getMetadata'); + } + + async beforeTransfer() { + await this.#dispatchAction('beforeTransfer'); + } + + getSchemas(): Promise { + return this.#dispatchAction('getSchemas'); + } + + getEntitiesStream(): Writable { + return new Writable({ + objectMode: true, + write: async (entity: IEntity, _encoding, callback) => { + const e = await this.#dispatchTransfer('entities', entity); + + callback(e); + }, + }); + } + + getLinksStream(): Writable { + return new Writable({ + objectMode: true, + write: async (link: ILink, _encoding, callback) => { + const e = await this.#dispatchTransfer('links', link); + + callback(e); + }, + }); + } + + getConfigurationStream(): Writable { + return new Writable({ + objectMode: true, + write: async (configuration: IConfiguration, _encoding, callback) => { + const e = await this.#dispatchTransfer('configuration', configuration); + + callback(e); + }, + }); + } +} diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts new file mode 100644 index 0000000000..0e1a876856 --- /dev/null +++ b/packages/core/data-transfer/lib/register.ts @@ -0,0 +1,295 @@ +import type { Context } from 'koa'; +import type { ServerOptions } from 'ws'; +import { WebSocket } from 'ws'; +import { Writable } from 'stream'; +import { IAsset, IConfiguration, IEntity, ILink, IMetadata, TransferStage } from '../types'; +import { + createLocalStrapiDestinationProvider, + ILocalStrapiDestinationProviderOptions, +} from './providers'; + +type PushTransferStage = Exclude; +type MessageKind = 'push' | 'pull'; + +type Message = { uuid: string } & (InitMessage | TransferMessage | ActionMessage | TeardownMessage); + +// init + +type InitMessage = { type: 'init' } & (IPushInitMessage | IPullInitMessage); + +interface IPushInitMessage { + type: 'init'; + kind: 'push'; + data: { strategy: ILocalStrapiDestinationProviderOptions['strategy'] }; +} + +interface IPullInitMessage { + type: 'init'; + kind: 'pull'; +} + +// teardown + +type TeardownMessage = { type: 'teardown' }; + +// action + +type ActionMessage = { + type: 'action'; + action: 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; +}; + +// transfer + +type TransferMessage = PushTransferMessage; + +type PushTransferMessage = { type: 'transfer' } & ( + | PushEntityMessage + | PushLinkMessage + | PushAssetMessage + | PushConfigurationMessage +); + +type PushEntityMessage = { stage: 'entities'; data: IEntity }; +type PushLinkMessage = { stage: 'links'; data: ILink }; +type PushAssetMessage = { stage: 'assets'; data: IAsset }; +type PushConfigurationMessage = { stage: 'configuration'; data: IConfiguration }; + +// Internal state + +interface ITransferState { + kind?: MessageKind; + controller?: IPushController; +} + +// Controllers + +interface IPushController { + actions: { + getMetadata(): Promise; + getSchemas(): Strapi.Schemas; + bootstrap(): Promise; + close(): Promise; + beforeTransfer(): Promise; + }; + transfer: { + entities(entity: IEntity): Promise | void; + links(link: ILink): Promise | void; + configuration(configuration: IConfiguration): Promise | void; + assets(asset: IAsset): Promise | void; + }; +} + +const createPushController = ( + ws: WebSocket, + options: ILocalStrapiDestinationProviderOptions +): IPushController => { + const provider = createLocalStrapiDestinationProvider(options); + + const streams: { [stage in PushTransferStage]?: Writable } = {}; + + const writeAsync = (stream: Writable, data: T) => { + return new Promise((resolve, reject) => { + stream.write(data, (error) => { + if (error) { + reject(error); + } + + resolve(); + }); + }); + }; + + return { + actions: { + async getSchemas() { + return provider.getSchemas(); + }, + + async getMetadata() { + return provider.getMetadata(); + }, + + async bootstrap() { + return provider.bootstrap(); + }, + + async close() { + return provider.close(); + }, + + async beforeTransfer() { + return provider.beforeTransfer(); + }, + }, + + transfer: { + async entities(entity) { + if (!streams.entities) { + streams.entities = provider.getEntitiesStream(); + } + + await writeAsync(streams.entities, entity); + }, + + async links(link) { + if (!streams.links) { + streams.links = await provider.getLinksStream(); + } + + await writeAsync(streams.links, link); + }, + + async configuration(config) { + if (!streams.configuration) { + streams.configuration = await provider.getConfigurationStream(); + } + + await writeAsync(streams.configuration, config); + }, + + async assets(asset) { + if (!streams.assets) { + streams.assets = await provider.getAssetsStream(); + } + await writeAsync(streams.assets, asset); + }, + }, + }; +}; + +const createTransferController = + (options: ServerOptions = {}) => + async (ctx: Context) => { + const upgradeHeader = (ctx.request.headers.upgrade || '') + .split(',') + .map((s) => s.trim().toLowerCase()); + + // Create the websocket server + const wss = new WebSocket.Server({ ...options, noServer: true }); + + if (upgradeHeader.includes('websocket')) { + wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => { + // Create a connection between the client & the server + wss.emit('connection', ws, ctx.req); + + const state: ITransferState = {}; + let uuid: string | undefined; + + const callback = (e: Error | null = null, data?: T) => { + return new Promise((resolve, reject) => { + if (!uuid) { + reject(new Error('Missing uuid for this message')); + return; + } + + const payload = JSON.stringify({ + uuid, + data: data ?? {}, + error: e, + }); + + ws.send(payload, (error) => (error ? reject(error) : resolve())); + }); + }; + + const answer = async (fn: () => T) => { + try { + const response = await fn(); + callback(null, response); + } catch (e) { + if (e instanceof Error) { + callback(e); + } else if (typeof e === 'string') { + callback(new Error(e)); + } else { + callback(new Error('Unexpected error')); + } + } + }; + + const teardown = () => { + delete state.kind; + delete state.controller; + + return { ok: true }; + }; + + const init = (kind: MessageKind, data: unknown = {}) => { + if (state.controller) { + throw new Error('Transfer already in progres'); + } + + if (kind === 'push') { + const { strategy } = data as Partial; + + if (!strategy) { + throw new Error('Tried to initiate a push transfer without a strategy'); + } + + state.controller = createPushController(ws, { + strategy, + autoDestroy: false, + getStrapi() { + return strapi; + }, + }); + } + + return { ok: true }; + }; + + ws.on('close', () => { + teardown(); + }); + + ws.on('error', (e) => { + teardown(); + console.error(e); + }); + + ws.on('message', async (raw) => { + const msg: Message = JSON.parse(raw.toString()); + + if (!msg.uuid) { + throw new Error('Missing uuid in message'); + } + + uuid = msg.uuid; + + if (msg.type === 'init') { + await answer(() => init(msg.kind, (msg as any)?.data)); + } + + if (msg.type === 'teardown') { + await answer(teardown); + } + + if (msg.type === 'action') { + await answer(() => state.controller?.actions[msg.action]?.()); + } + + if (msg.type === 'transfer') { + await answer(() => state.controller?.transfer[msg.stage]?.(msg.data as any)); + } + }); + }); + + ctx.respond = false; + } + }; + +const registerTransferRoute = (strapi: any) => { + strapi.admin.routes.push({ + method: 'GET', + path: '/transfer', + handler: createTransferController(), + config: { auth: false }, + }); +}; + +const register = (strapi: any) => { + registerTransferRoute(strapi); +}; + +export default register; diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index 0c4f3e093e..53afe6e46a 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -47,7 +47,9 @@ "stream-chain": "2.2.5", "stream-json": "1.7.4", "tar": "6.1.12", - "tar-stream": "2.2.0" + "tar-stream": "2.2.0", + "uuid": "9.0.0", + "ws": "8.11.0" }, "devDependencies": { "@tsconfig/node16": "1.0.3", @@ -58,6 +60,8 @@ "@types/stream-json": "1.7.2", "@types/tar": "6.1.3", "@types/tar-stream": "2.2.2", + "@types/uuid": "9.0.0", + "koa": "2.14.1", "rimraf": "3.0.2", "typescript": "4.6.2" }, diff --git a/packages/core/strapi/bin/strapi.js b/packages/core/strapi/bin/strapi.js index aaae25cfaa..471be826f6 100755 --- a/packages/core/strapi/bin/strapi.js +++ b/packages/core/strapi/bin/strapi.js @@ -258,6 +258,15 @@ program .option('-s, --silent', `Run the generation silently, without any output`, false) .action(getLocalScript('ts/generate-types')); +// `$ strapi transfer` +program + .command('transfer') + .description('Transfer data from one source to another') + .addOption(new Option('--from ', `Source of your data`).default('local')) + .addOption(new Option('--to ', `Destination of your data`).default('remote')) + .allowExcessArguments(false) + .action(getLocalScript('transfer/transfer')); + // `$ strapi export` program .command('export') diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js new file mode 100644 index 0000000000..952a5d5f50 --- /dev/null +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -0,0 +1,119 @@ +'use strict'; + +const { + createRemoteStrapiDestinationProvider, + createLocalStrapiSourceProvider, + createTransferEngine, + // TODO: we need to solve this issue with typescript modules + // eslint-disable-next-line import/no-unresolved, node/no-missing-require +} = require('@strapi/data-transfer'); +const { isObject } = require('lodash/fp'); +const chalk = require('chalk'); + +const { + buildTransferTable, + createStrapiInstance, + DEFAULT_IGNORED_CONTENT_TYPES, +} = require('./utils'); + +/** + * @typedef ImportCommandOptions Options given to the CLI import command + * + * @property {string} [file] The file path to import + * @property {boolean} [encrypt] Used to encrypt the final archive + * @property {string} [key] Encryption key, only useful when encryption is enabled + * @property {boolean} [compress] Used to compress the final archive + */ + +const logger = console; + +/** + * Import command. + * + * It transfers data from a local file to a local strapi instance + * + * @param {ImportCommandOptions} opts + */ +module.exports = async (opts) => { + // Validate inputs from Commander + if (!isObject(opts)) { + logger.error('Could not parse command arguments'); + process.exit(1); + } + + const strapi = await createStrapiInstance(); + + let source; + let destination; + if (opts.from === 'local') { + source = createSourceProvider(strapi); + } + if (opts.to) { + destination = createDestinationProvider({ + url: opts.to, + auth: false, + strategy: 'restore', + }); + } + if (!source || !destination) { + logger.error("Couldn't create providers"); + process.exit(1); + } + + const engine = createTransferEngine(source, destination, { + versionStrategy: 'ignore', // for an export to file, versionStrategy will always be skipped + schemaStrategy: 'ignore', // for an export to file, schemaStrategy will always be skipped + transforms: { + links: [ + { + filter(link) { + return ( + !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) && + !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type) + ); + }, + }, + ], + entities: [ + { + filter(entity) { + return !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type); + }, + }, + ], + }, + }); + + try { + logger.log(`Starting export...`); + + const results = await engine.transfer(); + + const table = buildTransferTable(results.engine); + logger.log(table.toString()); + + logger.log(`${chalk.bold('Transfer process has been completed successfully!')}`); + process.exit(0); + } catch (e) { + logger.error('Transfer process failed unexpectedly:', e); + process.exit(1); + } +}; + +/** + * It creates a local strapi destination provider + */ +const createSourceProvider = (strapi) => { + return createLocalStrapiSourceProvider({ + async getStrapi() { + return strapi; + }, + }); +}; + +/** + * It creates a remote strapi destination provider based on the given options + */ +const createDestinationProvider = (opts) => { + return createRemoteStrapiDestinationProvider(opts); +}; diff --git a/yarn.lock b/yarn.lock index d2221c3966..a74e93a3b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6713,6 +6713,11 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== +"@types/uuid@9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.0.tgz#53ef263e5239728b56096b0a869595135b7952d2" + integrity sha512-kr90f+ERiQtKWMz5rP32ltJ/BtULDI5RVO0uavn1HQUOwjx0R1h0rnDYNL0CepF1zL5bSY6FISAfd9tOdDhU5Q== + "@types/webpack-env@^1.16.0": version "1.18.0" resolved "https://registry.yarnpkg.com/@types/webpack-env/-/webpack-env-1.18.0.tgz#ed6ecaa8e5ed5dfe8b2b3d00181702c9925f13fb" @@ -15557,6 +15562,35 @@ koa@2.13.4, koa@^2.13.4: type-is "^1.6.16" vary "^1.1.2" +koa@2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/koa/-/koa-2.14.1.tgz#defb9589297d8eb1859936e777f3feecfc26925c" + integrity sha512-USJFyZgi2l0wDgqkfD27gL4YGno7TfUkcmOe6UOLFOVuN+J7FwnNu4Dydl4CUQzraM1lBAiGed0M9OVJoT0Kqw== + dependencies: + accepts "^1.3.5" + cache-content-type "^1.0.0" + content-disposition "~0.5.2" + content-type "^1.0.4" + cookies "~0.8.0" + debug "^4.3.2" + delegates "^1.0.0" + depd "^2.0.0" + destroy "^1.0.4" + encodeurl "^1.0.2" + escape-html "^1.0.3" + fresh "~0.5.2" + http-assert "^1.3.0" + http-errors "^1.6.3" + is-generator-function "^1.0.7" + koa-compose "^4.1.0" + koa-convert "^2.0.0" + on-finished "^2.3.0" + only "~0.0.2" + parseurl "^1.3.2" + statuses "^1.5.0" + type-is "^1.6.16" + vary "^1.1.2" + kuler@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" @@ -22716,6 +22750,11 @@ uuid@8.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== +uuid@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" + integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== + uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -23421,6 +23460,11 @@ write-pkg@^4.0.0: type-fest "^0.4.1" write-json-file "^3.2.0" +ws@8.11.0: + version "8.11.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" + integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== + "ws@^5.2.0 || ^6.0.0 || ^7.0.0", ws@^7.3.1: version "7.5.9" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" From 8ba5278b0daa62e4d181fe062c9b8de84b40dcaf Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 22 Dec 2022 15:03:09 +0100 Subject: [PATCH 02/44] Add restore option to the destination provider --- .../remote-strapi-destination-provider/index.ts | 9 +++++---- packages/core/data-transfer/lib/register.ts | 10 ++-------- packages/core/strapi/lib/commands/transfer/transfer.js | 3 +++ 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index df2585706f..8cdbd9a62d 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -2,6 +2,7 @@ import { WebSocket } from 'ws'; import { v4 } from 'uuid'; import { Writable } from 'stream'; +import type { restore } from '../local-strapi-destination-provider/strategies'; import type { IDestinationProvider, IEntity, @@ -24,10 +25,10 @@ interface ICredentialsAuth { password: string; } -interface IRemoteStrapiDestinationProvider { +interface IRemoteStrapiDestinationProvider + extends Pick { url: string; auth?: ITokenAuth | ICredentialsAuth; - strategy: ILocalStrapiDestinationProviderOptions['strategy']; } type Actions = 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; @@ -104,7 +105,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { } async bootstrap(): Promise { - const { url, auth, strategy } = this.options; + const { url, auth, strategy, restore } = this.options; let ws: WebSocket; @@ -130,7 +131,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { // Wait for the connection to be made to the server, then init the transfer await new Promise((resolve, reject) => { ws.once('open', async () => { - await this.#dispatch({ type: 'init', kind: 'push', data: { strategy } }); + await this.#dispatch({ type: 'init', kind: 'push', data: { strategy, restore } }); resolve(); }).once('error', reject); }); diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 0e1a876856..4e1d94a08f 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -20,7 +20,7 @@ type InitMessage = { type: 'init' } & (IPushInitMessage | IPullInitMessage); interface IPushInitMessage { type: 'init'; kind: 'push'; - data: { strategy: ILocalStrapiDestinationProviderOptions['strategy'] }; + data: Pick; } interface IPullInitMessage { @@ -221,14 +221,8 @@ const createTransferController = } if (kind === 'push') { - const { strategy } = data as Partial; - - if (!strategy) { - throw new Error('Tried to initiate a push transfer without a strategy'); - } - state.controller = createPushController(ws, { - strategy, + ...(data as IPushInitMessage['data']), autoDestroy: false, getStrapi() { return strapi; diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index 952a5d5f50..e91e175763 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -53,6 +53,9 @@ module.exports = async (opts) => { url: opts.to, auth: false, strategy: 'restore', + restore: { + entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES }, + }, }); } if (!source || !destination) { From 63d58433fcb7abb1eb4a05b0758ee9fefdb85f04 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 22 Dec 2022 18:48:23 +0100 Subject: [PATCH 03/44] Add typings for transfer --- .../local-file-destination-provider/index.ts | 1 + .../index.ts | 37 +++- packages/core/data-transfer/lib/register.ts | 171 ++++++++++-------- packages/core/data-transfer/types/index.d.ts | 1 + packages/core/data-transfer/types/remote.d.ts | 90 +++++++++ 5 files changed, 227 insertions(+), 73 deletions(-) create mode 100644 packages/core/data-transfer/types/remote.d.ts diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts index 081ebef70b..3799915c74 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts @@ -255,6 +255,7 @@ class LocalFileDestinationProvider implements IDestinationProvider { entry .on('finish', () => { + console.log('FINISH WRITING ALREADY'); callback(null); }) .on('error', (error) => { diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 8cdbd9a62d..1d9661e55f 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -2,7 +2,6 @@ import { WebSocket } from 'ws'; import { v4 } from 'uuid'; import { Writable } from 'stream'; -import type { restore } from '../local-strapi-destination-provider/strategies'; import type { IDestinationProvider, IEntity, @@ -11,6 +10,7 @@ import type { ProviderType, IConfiguration, TransferStage, + IAsset, } from '../../../types'; import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; @@ -199,4 +199,39 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { }, }); } + + getAssetsStream(): Writable | Promise { + return new Writable({ + objectMode: true, + final: async (callback) => { + const e = await this.#dispatchTransfer('assets', null); + callback(e); + }, + write: async (asset: IAsset, _encoding, callback) => { + const { filename, filepath, stats, stream } = asset; + const assetID = v4(); + + await this.#dispatchTransfer('assets', { + step: 'start', + assetID, + data: { filename, filepath, stats }, + }); + + for await (const chunk of stream) { + await this.#dispatchTransfer('assets', { + step: 'stream', + assetID, + data: { chunk }, + }); + } + + await this.#dispatchTransfer('assets', { + step: 'end', + assetID, + }); + + callback(); + }, + }); + } } diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4e1d94a08f..4f358ab65f 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -1,70 +1,35 @@ import type { Context } from 'koa'; import type { ServerOptions } from 'ws'; import { WebSocket } from 'ws'; -import { Writable } from 'stream'; -import { IAsset, IConfiguration, IEntity, ILink, IMetadata, TransferStage } from '../types'; +import { Writable, PassThrough } from 'stream'; +import { v4 } from 'uuid'; +import { + IAsset, + IConfiguration, + Message, + ILink, + IMetadata, + PushTransferMessage, + PushEntitiesTransferMessage, + TransferKind, + InitMessage, + PushTransferStage, +} from '../types'; import { - createLocalStrapiDestinationProvider, ILocalStrapiDestinationProviderOptions, + createLocalStrapiDestinationProvider, } from './providers'; -type PushTransferStage = Exclude; -type MessageKind = 'push' | 'pull'; - -type Message = { uuid: string } & (InitMessage | TransferMessage | ActionMessage | TeardownMessage); - -// init - -type InitMessage = { type: 'init' } & (IPushInitMessage | IPullInitMessage); - -interface IPushInitMessage { - type: 'init'; - kind: 'push'; - data: Pick; -} - -interface IPullInitMessage { - type: 'init'; - kind: 'pull'; -} - -// teardown - -type TeardownMessage = { type: 'teardown' }; - -// action - -type ActionMessage = { - type: 'action'; - action: 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; -}; - -// transfer - -type TransferMessage = PushTransferMessage; - -type PushTransferMessage = { type: 'transfer' } & ( - | PushEntityMessage - | PushLinkMessage - | PushAssetMessage - | PushConfigurationMessage -); - -type PushEntityMessage = { stage: 'entities'; data: IEntity }; -type PushLinkMessage = { stage: 'links'; data: ILink }; -type PushAssetMessage = { stage: 'assets'; data: IAsset }; -type PushConfigurationMessage = { stage: 'configuration'; data: IConfiguration }; - -// Internal state - interface ITransferState { - kind?: MessageKind; + kind?: TransferKind; + transferID?: string; controller?: IPushController; } // Controllers interface IPushController { + streams: { [stage in PushTransferStage]?: Writable }; actions: { getMetadata(): Promise; getSchemas(): Strapi.Schemas; @@ -73,20 +38,17 @@ interface IPushController { beforeTransfer(): Promise; }; transfer: { - entities(entity: IEntity): Promise | void; - links(link: ILink): Promise | void; - configuration(configuration: IConfiguration): Promise | void; - assets(asset: IAsset): Promise | void; + [key in PushTransferStage]: ( + value: T extends { stage: P; data: infer U } ? U : never + ) => Promise; }; } -const createPushController = ( - ws: WebSocket, - options: ILocalStrapiDestinationProviderOptions -): IPushController => { +const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { const provider = createLocalStrapiDestinationProvider(options); const streams: { [stage in PushTransferStage]?: Writable } = {}; + const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; const writeAsync = (stream: Writable, data: T) => { return new Promise((resolve, reject) => { @@ -101,6 +63,8 @@ const createPushController = ( }; return { + streams, + actions: { async getSchemas() { return provider.getSchemas(); @@ -129,7 +93,7 @@ const createPushController = ( streams.entities = provider.getEntitiesStream(); } - await writeAsync(streams.entities, entity); + await writeAsync(streams.entities!, entity); }, async links(link) { @@ -137,7 +101,7 @@ const createPushController = ( streams.links = await provider.getLinksStream(); } - await writeAsync(streams.links, link); + await writeAsync(streams.links!, link); }, async configuration(config) { @@ -145,14 +109,46 @@ const createPushController = ( streams.configuration = await provider.getConfigurationStream(); } - await writeAsync(streams.configuration, config); + await writeAsync(streams.configuration!, config); }, - async assets(asset) { + async assets(asset: any) { + if (asset === null) { + streams.assets?.end(); + return; + } + + const { step, assetID } = asset; + if (!streams.assets) { streams.assets = await provider.getAssetsStream(); } - await writeAsync(streams.assets, asset); + + // on init, we create a passthrough stream for the asset chunks + // send to the assets destination stream the metadata for the current asset + // + the stream that we just created for the asset + if (step === 'start') { + assets[assetID] = { ...asset.data, stream: new PassThrough() }; + writeAsync(streams.assets!, assets[assetID]); + } + + // propagate the chunk + if (step === 'stream') { + await writeAsync(assets[assetID].stream, Buffer.from(asset.data.chunk)); + } + + // on end, we indicate that all the chunks have been sent + if (step === 'end') { + await new Promise((resolve, reject) => { + assets[assetID].stream + .on('close', () => { + delete assets[assetID]; + resolve(); + }) + .on('error', reject) + .end(); + }); + } }, }, }; @@ -211,18 +207,21 @@ const createTransferController = const teardown = () => { delete state.kind; delete state.controller; + delete state.transferID; return { ok: true }; }; - const init = (kind: MessageKind, data: unknown = {}) => { + const init = (msg: InitMessage) => { + const { kind, options: controllerOptions } = msg; + if (state.controller) { throw new Error('Transfer already in progres'); } if (kind === 'push') { - state.controller = createPushController(ws, { - ...(data as IPushInitMessage['data']), + state.controller = createPushController({ + ...controllerOptions, autoDestroy: false, getStrapi() { return strapi; @@ -230,7 +229,14 @@ const createTransferController = }); } - return { ok: true }; + // Pull or others + else { + throw new Error(`${kind} transfer not implemented`); + } + + state.transferID = v4(); + + return { transferID: state.transferID }; }; ws.on('close', () => { @@ -252,7 +258,7 @@ const createTransferController = uuid = msg.uuid; if (msg.type === 'init') { - await answer(() => init(msg.kind, (msg as any)?.data)); + await answer(() => init(msg)); } if (msg.type === 'teardown') { @@ -264,7 +270,13 @@ const createTransferController = } if (msg.type === 'transfer') { - await answer(() => state.controller?.transfer[msg.stage]?.(msg.data as any)); + await answer(() => { + const fn = state.controller?.transfer[msg.stage]; + + type Msg = typeof msg; + + fn?.(msg.data); + }); } }); }); @@ -287,3 +299,18 @@ const register = (strapi: any) => { }; export default register; + +/** + * entities:start + * entities:transfer + * entities:end + * + * + * assets:start + * + * assets:transfer:start + * assets:transfer:stream + * assets:transfer:end + * + * assets:end + */ diff --git a/packages/core/data-transfer/types/index.d.ts b/packages/core/data-transfer/types/index.d.ts index 73d649f130..51f3087de4 100644 --- a/packages/core/data-transfer/types/index.d.ts +++ b/packages/core/data-transfer/types/index.d.ts @@ -3,3 +3,4 @@ export * from './providers'; export * from './transfer-engine'; export * from './utils'; export * from './encryption'; +export * from './remote'; diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts new file mode 100644 index 0000000000..cb64d420c1 --- /dev/null +++ b/packages/core/data-transfer/types/remote.d.ts @@ -0,0 +1,90 @@ +import type { ILocalStrapiDestinationProviderOptions } from '../lib'; +import type { IAsset, IConfiguration, IEntity, ILink } from './common-entities'; + +/** + * Utils + */ + +type EmptyObject = Record; + +/** + * Messages + */ + +export type Message = { uuid: string | null | undefined } & ( + | InitMessage + | ActionMessage + | PushTransferMessage + | TeardownMessage +); + +export type MessageType = Message['type']; +export type TransferKind = InitMessage['kind']; +export type PushTransferStage = PushTransferMessage['stage']; + +/** + * Init + */ + +// init should return a transfer ID used in the teardown +export type InitMessage = { type: 'init' } & ( + | { kind: 'pull'; options: EmptyObject } + | { kind: 'push'; options: Pick } +); + +/** + * Action + */ + +export type ActionMessage = { type: 'action' } & ( + | { action: 'getMetadata'; options: EmptyObject } + | { action: 'getSchemas'; options: EmptyObject } + | { action: 'bootstrap'; options: EmptyObject } + | { action: 'close'; options: EmptyObject } + | { action: 'beforeTransfer'; options: EmptyObject } +); + +/** + * Transfer + */ + +export type PushTransferMessage = { + type: 'transfer'; +} & ( + | PushEntitiesTransferMessage + | PushLinksTransferMessage + | PushConfigurationTransferMessage + | PushAssetTransferMessage +); + +export type PushEntitiesTransferMessage = { + stage: 'entities'; + data: IEntity | null; +}; + +export type PushLinksTransferMessage = { stage: 'links'; data: ILink | null }; + +export type PushConfigurationTransferMessage = { + stage: 'configuration'; + data: IConfiguration | null; +}; + +export type PushAssetTransferMessage = { + stage: 'assets'; + data: + | ({ assetID: string } & ( + | { step: 'start'; data: Omit } + | { step: 'stream'; data: { chunk: { type: 'Buffer'; data: number[] } } } + | { step: 'end'; data: EmptyObject } + )) + | null; +}; + +/** + * Teardown + */ + +export type TeardownMessage = { + type: 'teardown'; + transferID: string; +}; From 9cdb5c1096d36cefdbfbedfd5f80b40026a6eee4 Mon Sep 17 00:00:00 2001 From: Convly Date: Fri, 23 Dec 2022 10:19:14 +0100 Subject: [PATCH 04/44] Add better typing for the remote protocol --- .../index.ts | 37 ++++- packages/core/data-transfer/lib/register.ts | 143 +++++++++--------- packages/core/data-transfer/types/index.d.ts | 1 + packages/core/data-transfer/types/remote.d.ts | 90 +++++++++++ 4 files changed, 200 insertions(+), 71 deletions(-) create mode 100644 packages/core/data-transfer/types/remote.d.ts diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 8cdbd9a62d..1d9661e55f 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -2,7 +2,6 @@ import { WebSocket } from 'ws'; import { v4 } from 'uuid'; import { Writable } from 'stream'; -import type { restore } from '../local-strapi-destination-provider/strategies'; import type { IDestinationProvider, IEntity, @@ -11,6 +10,7 @@ import type { ProviderType, IConfiguration, TransferStage, + IAsset, } from '../../../types'; import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; @@ -199,4 +199,39 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { }, }); } + + getAssetsStream(): Writable | Promise { + return new Writable({ + objectMode: true, + final: async (callback) => { + const e = await this.#dispatchTransfer('assets', null); + callback(e); + }, + write: async (asset: IAsset, _encoding, callback) => { + const { filename, filepath, stats, stream } = asset; + const assetID = v4(); + + await this.#dispatchTransfer('assets', { + step: 'start', + assetID, + data: { filename, filepath, stats }, + }); + + for await (const chunk of stream) { + await this.#dispatchTransfer('assets', { + step: 'stream', + assetID, + data: { chunk }, + }); + } + + await this.#dispatchTransfer('assets', { + step: 'end', + assetID, + }); + + callback(); + }, + }); + } } diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4e1d94a08f..41c8d64fb8 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -1,64 +1,25 @@ import type { Context } from 'koa'; import type { ServerOptions } from 'ws'; import { WebSocket } from 'ws'; -import { Writable } from 'stream'; -import { IAsset, IConfiguration, IEntity, ILink, IMetadata, TransferStage } from '../types'; +import { Writable, PassThrough } from 'stream'; +import { v4 } from 'uuid'; +import { + IAsset, + Message, + IMetadata, + PushTransferMessage, + TransferKind, + InitMessage, + PushTransferStage, +} from '../types'; import { - createLocalStrapiDestinationProvider, ILocalStrapiDestinationProviderOptions, + createLocalStrapiDestinationProvider, } from './providers'; -type PushTransferStage = Exclude; -type MessageKind = 'push' | 'pull'; - -type Message = { uuid: string } & (InitMessage | TransferMessage | ActionMessage | TeardownMessage); - -// init - -type InitMessage = { type: 'init' } & (IPushInitMessage | IPullInitMessage); - -interface IPushInitMessage { - type: 'init'; - kind: 'push'; - data: Pick; -} - -interface IPullInitMessage { - type: 'init'; - kind: 'pull'; -} - -// teardown - -type TeardownMessage = { type: 'teardown' }; - -// action - -type ActionMessage = { - type: 'action'; - action: 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; -}; - -// transfer - -type TransferMessage = PushTransferMessage; - -type PushTransferMessage = { type: 'transfer' } & ( - | PushEntityMessage - | PushLinkMessage - | PushAssetMessage - | PushConfigurationMessage -); - -type PushEntityMessage = { stage: 'entities'; data: IEntity }; -type PushLinkMessage = { stage: 'links'; data: ILink }; -type PushAssetMessage = { stage: 'assets'; data: IAsset }; -type PushConfigurationMessage = { stage: 'configuration'; data: IConfiguration }; - -// Internal state - interface ITransferState { - kind?: MessageKind; + kind?: TransferKind; + transferID?: string; controller?: IPushController; } @@ -73,20 +34,17 @@ interface IPushController { beforeTransfer(): Promise; }; transfer: { - entities(entity: IEntity): Promise | void; - links(link: ILink): Promise | void; - configuration(configuration: IConfiguration): Promise | void; - assets(asset: IAsset): Promise | void; + [key in PushTransferStage]: ( + value: T extends { stage: key; data: infer U } ? U : never + ) => Promise; }; } -const createPushController = ( - ws: WebSocket, - options: ILocalStrapiDestinationProviderOptions -): IPushController => { +const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { const provider = createLocalStrapiDestinationProvider(options); const streams: { [stage in PushTransferStage]?: Writable } = {}; + const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; const writeAsync = (stream: Writable, data: T) => { return new Promise((resolve, reject) => { @@ -102,7 +60,7 @@ const createPushController = ( return { actions: { - async getSchemas() { + async getSchemas(): Promise { return provider.getSchemas(); }, @@ -148,11 +106,42 @@ const createPushController = ( await writeAsync(streams.configuration, config); }, - async assets(asset) { + async assets(payload) { + if (payload === null) { + streams.assets?.end(); + return; + } + + const { step, assetID } = payload; + if (!streams.assets) { streams.assets = await provider.getAssetsStream(); } - await writeAsync(streams.assets, asset); + + if (step === 'start') { + assets[assetID] = { ...payload.data, stream: new PassThrough() }; + writeAsync(streams.assets, assets[assetID]); + } + + if (step === 'stream') { + const chunk = Buffer.from(payload.data.chunk.data); + + await writeAsync(assets[assetID].stream, chunk); + } + + if (step === 'end') { + await new Promise((resolve, reject) => { + const { stream } = assets[assetID]; + + stream + .on('close', () => { + delete assets[assetID]; + resolve(); + }) + .on('error', reject) + .end(); + }); + } }, }, }; @@ -211,18 +200,21 @@ const createTransferController = const teardown = () => { delete state.kind; delete state.controller; + delete state.transferID; return { ok: true }; }; - const init = (kind: MessageKind, data: unknown = {}) => { + const init = (msg: InitMessage) => { + const { kind, options: controllerOptions } = msg; + if (state.controller) { throw new Error('Transfer already in progres'); } if (kind === 'push') { - state.controller = createPushController(ws, { - ...(data as IPushInitMessage['data']), + state.controller = createPushController({ + ...controllerOptions, autoDestroy: false, getStrapi() { return strapi; @@ -230,7 +222,14 @@ const createTransferController = }); } - return { ok: true }; + // Pull or others + else { + throw new Error(`${kind} transfer not implemented`); + } + + state.transferID = v4(); + + return { transferID: state.transferID }; }; ws.on('close', () => { @@ -252,7 +251,7 @@ const createTransferController = uuid = msg.uuid; if (msg.type === 'init') { - await answer(() => init(msg.kind, (msg as any)?.data)); + await answer(() => init(msg)); } if (msg.type === 'teardown') { @@ -264,7 +263,11 @@ const createTransferController = } if (msg.type === 'transfer') { - await answer(() => state.controller?.transfer[msg.stage]?.(msg.data as any)); + await answer(() => { + const { stage, data } = msg; + + return state.controller?.transfer[stage](data as never); + }); } }); }); diff --git a/packages/core/data-transfer/types/index.d.ts b/packages/core/data-transfer/types/index.d.ts index 73d649f130..51f3087de4 100644 --- a/packages/core/data-transfer/types/index.d.ts +++ b/packages/core/data-transfer/types/index.d.ts @@ -3,3 +3,4 @@ export * from './providers'; export * from './transfer-engine'; export * from './utils'; export * from './encryption'; +export * from './remote'; diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts new file mode 100644 index 0000000000..cb64d420c1 --- /dev/null +++ b/packages/core/data-transfer/types/remote.d.ts @@ -0,0 +1,90 @@ +import type { ILocalStrapiDestinationProviderOptions } from '../lib'; +import type { IAsset, IConfiguration, IEntity, ILink } from './common-entities'; + +/** + * Utils + */ + +type EmptyObject = Record; + +/** + * Messages + */ + +export type Message = { uuid: string | null | undefined } & ( + | InitMessage + | ActionMessage + | PushTransferMessage + | TeardownMessage +); + +export type MessageType = Message['type']; +export type TransferKind = InitMessage['kind']; +export type PushTransferStage = PushTransferMessage['stage']; + +/** + * Init + */ + +// init should return a transfer ID used in the teardown +export type InitMessage = { type: 'init' } & ( + | { kind: 'pull'; options: EmptyObject } + | { kind: 'push'; options: Pick } +); + +/** + * Action + */ + +export type ActionMessage = { type: 'action' } & ( + | { action: 'getMetadata'; options: EmptyObject } + | { action: 'getSchemas'; options: EmptyObject } + | { action: 'bootstrap'; options: EmptyObject } + | { action: 'close'; options: EmptyObject } + | { action: 'beforeTransfer'; options: EmptyObject } +); + +/** + * Transfer + */ + +export type PushTransferMessage = { + type: 'transfer'; +} & ( + | PushEntitiesTransferMessage + | PushLinksTransferMessage + | PushConfigurationTransferMessage + | PushAssetTransferMessage +); + +export type PushEntitiesTransferMessage = { + stage: 'entities'; + data: IEntity | null; +}; + +export type PushLinksTransferMessage = { stage: 'links'; data: ILink | null }; + +export type PushConfigurationTransferMessage = { + stage: 'configuration'; + data: IConfiguration | null; +}; + +export type PushAssetTransferMessage = { + stage: 'assets'; + data: + | ({ assetID: string } & ( + | { step: 'start'; data: Omit } + | { step: 'stream'; data: { chunk: { type: 'Buffer'; data: number[] } } } + | { step: 'end'; data: EmptyObject } + )) + | null; +}; + +/** + * Teardown + */ + +export type TeardownMessage = { + type: 'teardown'; + transferID: string; +}; From 4190687975f0f045af7f6b4f014d4b23fe44e45d Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Tue, 27 Dec 2022 09:55:26 +0100 Subject: [PATCH 05/44] Add tests and refactor --- .../core/data-transfer/lib/engine/index.ts | 35 +++---- .../__tests__/index.test.ts | 48 +++++++++ .../__tests__/utils.test.ts | 43 ++++++++ .../index.ts | 53 +++------- .../utils.ts | 34 +++++++ packages/core/data-transfer/lib/register.ts | 54 ++++------ .../{ => data-transfer}/export.test.js | 6 +- .../__tests__/data-transfer/transfer.test.js | 98 +++++++++++++++++++ .../strapi/lib/commands/transfer/transfer.js | 10 +- 9 files changed, 274 insertions(+), 107 deletions(-) create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts rename packages/core/strapi/lib/commands/__tests__/{ => data-transfer}/export.test.js (94%) create mode 100644 packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index 779f2d3fd7..489ee01c9b 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -338,28 +338,23 @@ class TransferEngine< async transfer(): Promise> { try { await this.bootstrap(); - await this.init(); - - const isValidTransfer = await this.integrityCheck(); - - if (!isValidTransfer) { - // TODO: provide the log from the integrity check - throw new Error( - `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` - ); - } - - await this.beforeTransfer(); - + // await this.init(); + // const isValidTransfer = await this.integrityCheck(); + // if (!isValidTransfer) { + // // TODO: provide the log from the integrity check + // throw new Error( + // `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` + // ); + // } + // await this.beforeTransfer(); // Run the transfer stages - await this.transferSchemas(); - await this.transferEntities(); - await this.transferAssets(); - await this.transferLinks(); - await this.transferConfiguration(); - + // await this.transferSchemas(); + // await this.transferEntities(); + // await this.transferAssets(); + // await this.transferLinks(); + // await this.transferConfiguration(); // Gracefully close the providers - await this.close(); + // await this.close(); } catch (e: unknown) { // Rollback the destination provider if an exception is thrown during the transfer // Note: This will be configurable in the future diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts new file mode 100644 index 0000000000..8a7a744af4 --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts @@ -0,0 +1,48 @@ +import type { IRemoteStrapiDestinationProviderOptions } from '..'; + +import { createRemoteStrapiDestinationProvider } from '..'; + +const defaultOptions: IRemoteStrapiDestinationProviderOptions = { + strategy: 'restore', + url: 'ws://test.com/admin/transfer', +}; + +jest.mock('../utils', () => ({ + dispatch: jest.fn(), +})); + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn((type, callback) => { + callback(); + return { + once: jest.fn((t, c) => c), + }; + }), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination', () => { + describe('Bootstrap', () => { + test('Should not have a defined websocket connection if bootstrap has not been called', () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + + expect(provider.ws).toBeNull(); + }); + + test('Should have a defined websocket connection if bootstrap has been called', async () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + await provider.bootstrap(); + + expect(provider.ws).not.toBeNull(); + }); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts new file mode 100644 index 0000000000..1cd4148b7a --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts @@ -0,0 +1,43 @@ +import { WebSocket } from 'ws'; +import { dispatch } from '../utils'; + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn(), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination Utils', () => { + test('Dispatch method sends payload', () => { + const ws = new WebSocket('ws://test/admin/transfer'); + const message = { + test: 'hello', + }; + + dispatch(ws, message); + + expect.extend({ + toContain(receivedString, expected) { + const jsonReceived = JSON.parse(receivedString); + const pass = Object.keys(expected).every((key) => jsonReceived[key] === expected[key]); + + return { + message: () => + `Expected ${jsonReceived} ${!pass && 'not'} to contain properties ${expected}`, + pass, + }; + }, + }); + + // @ts-ignore + expect(ws.send).toHaveBeenCalledWith(expect.toContain(message), expect.anything()); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 1d9661e55f..03fcefa284 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -13,6 +13,7 @@ import type { IAsset, } from '../../../types'; import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; +import { dispatch } from './utils'; interface ITokenAuth { type: 'token'; @@ -25,7 +26,7 @@ interface ICredentialsAuth { password: string; } -interface IRemoteStrapiDestinationProvider +export interface IRemoteStrapiDestinationProviderOptions extends Pick { url: string; auth?: ITokenAuth | ICredentialsAuth; @@ -34,7 +35,7 @@ interface IRemoteStrapiDestinationProvider type Actions = 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; export const createRemoteStrapiDestinationProvider = ( - options: IRemoteStrapiDestinationProvider + options: IRemoteStrapiDestinationProviderOptions ) => { return new RemoteStrapiDestinationProvider(options); }; @@ -44,61 +45,28 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { type: ProviderType = 'destination'; - options: IRemoteStrapiDestinationProvider; + options: IRemoteStrapiDestinationProviderOptions; ws: WebSocket | null; - constructor(options: IRemoteStrapiDestinationProvider) { + constructor(options: IRemoteStrapiDestinationProviderOptions) { this.options = options; this.ws = null; } - async #dispatch(message: T): Promise { - const { ws } = this; - - if (!ws) { - throw new Error('No ws connection found'); - } - - return new Promise((resolve, reject) => { - const uuid = v4(); - const payload = JSON.stringify({ ...message, uuid }); - - ws.send(payload, (error) => { - if (error) { - reject(error); - } - }); - - ws.once('message', (raw) => { - const response: { uuid: string; data: U; error: string | null } = JSON.parse( - raw.toString() - ); - - if (response.error) { - return reject(new Error(response.error)); - } - - if (response.uuid === uuid) { - return resolve(response.data); - } - }); - }); - } - async #dispatchAction(action: Actions) { - return this.#dispatch({ type: 'action', action }); + return dispatch(this.ws, { type: 'action', action }); } async #dispatchTransfer(stage: TransferStage, data: T) { try { - await this.#dispatch({ type: 'transfer', stage, data }); + await dispatch(this.ws, { type: 'transfer', stage, data }); } catch (e) { if (e instanceof Error) { return e; } - return new Error('Unexected error'); + return new Error('Unexpected error'); } return null; @@ -131,7 +99,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { // Wait for the connection to be made to the server, then init the transfer await new Promise((resolve, reject) => { ws.once('open', async () => { - await this.#dispatch({ type: 'init', kind: 'push', data: { strategy, restore } }); + await dispatch(this.ws, { type: 'init', kind: 'push', data: { strategy, restore } }); resolve(); }).once('error', reject); }); @@ -204,6 +172,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, final: async (callback) => { + console.log('FINAL'); const e = await this.#dispatchTransfer('assets', null); callback(e); }, @@ -217,6 +186,8 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { data: { filename, filepath, stats }, }); + console.log('is writing'); + for await (const chunk of stream) { await this.#dispatchTransfer('assets', { step: 'stream', diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts new file mode 100644 index 0000000000..b79fc64818 --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts @@ -0,0 +1,34 @@ +import { v4 } from 'uuid'; +import { WebSocket } from 'ws'; + +export async function dispatch( + ws: WebSocket | null, + message: T +): Promise { + if (!ws) { + throw new Error('No websocket connection found'); + } + + return new Promise((resolve, reject) => { + const uuid = v4(); + const payload = JSON.stringify({ ...message, uuid }); + + ws.send(payload, (error) => { + if (error) { + reject(error); + } + }); + + ws.once('message', (raw) => { + const response: { uuid: string; data: U; error: string | null } = JSON.parse(raw.toString()); + + if (response.error) { + return reject(new Error(response.error)); + } + + if (response.uuid === uuid) { + return resolve(response.data); + } + }); + }); +} diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4f358ab65f..4a6e75015a 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -5,12 +5,9 @@ import { Writable, PassThrough } from 'stream'; import { v4 } from 'uuid'; import { IAsset, - IConfiguration, Message, - ILink, IMetadata, PushTransferMessage, - PushEntitiesTransferMessage, TransferKind, InitMessage, PushTransferStage, @@ -38,8 +35,8 @@ interface IPushController { beforeTransfer(): Promise; }; transfer: { - [key in PushTransferStage]: ( - value: T extends { stage: P; data: infer U } ? U : never + [key in PushTransferStage]: ( + value: T extends { stage: key; data: infer U } ? U : never ) => Promise; }; } @@ -66,7 +63,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): streams, actions: { - async getSchemas() { + async getSchemas(): Promise { return provider.getSchemas(); }, @@ -112,35 +109,35 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): await writeAsync(streams.configuration!, config); }, - async assets(asset: any) { - if (asset === null) { + async assets(payload) { + console.log('llega'); + if (payload === null) { streams.assets?.end(); return; } - const { step, assetID } = asset; + const { step, assetID } = payload; if (!streams.assets) { streams.assets = await provider.getAssetsStream(); } - // on init, we create a passthrough stream for the asset chunks - // send to the assets destination stream the metadata for the current asset - // + the stream that we just created for the asset if (step === 'start') { - assets[assetID] = { ...asset.data, stream: new PassThrough() }; - writeAsync(streams.assets!, assets[assetID]); + assets[assetID] = { ...payload.data, stream: new PassThrough() }; + writeAsync(streams.assets, assets[assetID]); } - // propagate the chunk if (step === 'stream') { - await writeAsync(assets[assetID].stream, Buffer.from(asset.data.chunk)); + const chunk = Buffer.from(payload.data.chunk.data); + + await writeAsync(assets[assetID].stream, chunk); } - // on end, we indicate that all the chunks have been sent if (step === 'end') { await new Promise((resolve, reject) => { - assets[assetID].stream + const { stream } = assets[assetID]; + + stream .on('close', () => { delete assets[assetID]; resolve(); @@ -271,11 +268,9 @@ const createTransferController = if (msg.type === 'transfer') { await answer(() => { - const fn = state.controller?.transfer[msg.stage]; + const { stage, data } = msg; - type Msg = typeof msg; - - fn?.(msg.data); + return state.controller?.transfer[stage](data as never); }); } }); @@ -299,18 +294,3 @@ const register = (strapi: any) => { }; export default register; - -/** - * entities:start - * entities:transfer - * entities:end - * - * - * assets:start - * - * assets:transfer:start - * assets:transfer:stream - * assets:transfer:end - * - * assets:end - */ diff --git a/packages/core/strapi/lib/commands/__tests__/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js similarity index 94% rename from packages/core/strapi/lib/commands/__tests__/export.test.js rename to packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index bf30ebeeda..5d81b7aa58 100644 --- a/packages/core/strapi/lib/commands/__tests__/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -1,6 +1,6 @@ 'use strict'; -const utils = require('../transfer/utils'); +const utils = require('../../transfer/utils'); const mockDataTransfer = { createLocalFileDestinationProvider: jest.fn(), @@ -18,12 +18,12 @@ jest.mock( { virtual: true } ); -const exportCommand = require('../transfer/export'); +const exportCommand = require('../../transfer/export'); const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); jest.spyOn(console, 'error').mockImplementation(() => {}); -jest.mock('../transfer/utils'); +jest.mock('../../transfer/utils'); const defaultFileName = 'defaultFilename'; diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js new file mode 100644 index 0000000000..2c58f96c58 --- /dev/null +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -0,0 +1,98 @@ +'use strict'; + +const utils = require('../../transfer/utils'); + +const mockDataTransfer = { + createRemoteStrapiDestinationProvider: jest.fn(), + createLocalStrapiSourceProvider: jest.fn(), + createTransferEngine: jest.fn().mockReturnValue({ + transfer: jest.fn().mockReturnValue(Promise.resolve({})), + }), +}; + +jest.mock( + '@strapi/data-transfer', + () => { + return mockDataTransfer; + }, + { virtual: true } +); + +const transferCommand = require('../../transfer/transfer'); + +const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); +jest.spyOn(console, 'error').mockImplementation(() => {}); + +jest.mock('../../transfer/utils'); + +const destinationUrl = 'ws://strapi.com'; + +describe('transfer', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('uses destination url provided by user without authentication', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + + expect(exit).toHaveBeenCalled(); + }); + + it('uses destination url provided by user with authentication', async () => { + // TODO when authentication is implemented + }); + + it('uses restore as the default strategy', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + it('uses destination url provided by user without authentication', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + + expect(exit).toHaveBeenCalled(); + }); + + it('uses destination url provided by user with authentication', async () => { + // TODO when authentication is implemented + }); + + it('uses restore as the default strategy', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + + it('uses local strapi instance when local specified', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createLocalStrapiSourceProvider).toHaveBeenCalled(); + expect(utils.createStrapiInstance).toHaveBeenCalled(); + + expect(exit).toHaveBeenCalled(); + }); + + it('creates the transfer engine successfully', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); +}); diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index e91e175763..e3f2db936f 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -17,12 +17,10 @@ const { } = require('./utils'); /** - * @typedef ImportCommandOptions Options given to the CLI import command + * @typedef TransferCommandOptions Options given to the CLI import command * - * @property {string} [file] The file path to import - * @property {boolean} [encrypt] Used to encrypt the final archive - * @property {string} [key] Encryption key, only useful when encryption is enabled - * @property {boolean} [compress] Used to compress the final archive + * @property {string} [from] The source strapi project + * @property {string} [to] The destination strapi project */ const logger = console; @@ -32,7 +30,7 @@ const logger = console; * * It transfers data from a local file to a local strapi instance * - * @param {ImportCommandOptions} opts + * @param {TransferCommandOptions} opts */ module.exports = async (opts) => { // Validate inputs from Commander From 777f76a0df8c8536395bfcdffda6b132f70d7aff Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 11:12:34 +0100 Subject: [PATCH 06/44] fix --- packages/core/data-transfer/lib/register.ts | 4 ++-- packages/core/data-transfer/types/remote.d.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4a6e75015a..98030dcf59 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -179,7 +179,7 @@ const createTransferController = const payload = JSON.stringify({ uuid, data: data ?? {}, - error: e, + error: JSON.stringify(e?.message), }); ws.send(payload, (error) => (error ? reject(error) : resolve())); @@ -210,7 +210,7 @@ const createTransferController = }; const init = (msg: InitMessage) => { - const { kind, options: controllerOptions } = msg; + const { kind, data: controllerOptions } = msg; if (state.controller) { throw new Error('Transfer already in progres'); diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts index cb64d420c1..c651667953 100644 --- a/packages/core/data-transfer/types/remote.d.ts +++ b/packages/core/data-transfer/types/remote.d.ts @@ -28,8 +28,8 @@ export type PushTransferStage = PushTransferMessage['stage']; // init should return a transfer ID used in the teardown export type InitMessage = { type: 'init' } & ( - | { kind: 'pull'; options: EmptyObject } - | { kind: 'push'; options: Pick } + | { kind: 'pull'; data: EmptyObject } + | { kind: 'push'; data: Pick } ); /** From e3277b75743d4c4cf565c800c81968f90f573915 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 11:30:01 +0100 Subject: [PATCH 07/44] use options instead of data --- .../lib/providers/remote-strapi-destination-provider/index.ts | 2 +- packages/core/data-transfer/lib/register.ts | 2 +- packages/core/data-transfer/types/remote.d.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 03fcefa284..adddd27741 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -99,7 +99,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { // Wait for the connection to be made to the server, then init the transfer await new Promise((resolve, reject) => { ws.once('open', async () => { - await dispatch(this.ws, { type: 'init', kind: 'push', data: { strategy, restore } }); + await dispatch(this.ws, { type: 'init', kind: 'push', options: { strategy, restore } }); resolve(); }).once('error', reject); }); diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 98030dcf59..70b7b3ba6c 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -210,7 +210,7 @@ const createTransferController = }; const init = (msg: InitMessage) => { - const { kind, data: controllerOptions } = msg; + const { kind, options: controllerOptions } = msg; if (state.controller) { throw new Error('Transfer already in progres'); diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts index c651667953..cb64d420c1 100644 --- a/packages/core/data-transfer/types/remote.d.ts +++ b/packages/core/data-transfer/types/remote.d.ts @@ -28,8 +28,8 @@ export type PushTransferStage = PushTransferMessage['stage']; // init should return a transfer ID used in the teardown export type InitMessage = { type: 'init' } & ( - | { kind: 'pull'; data: EmptyObject } - | { kind: 'push'; data: Pick } + | { kind: 'pull'; options: EmptyObject } + | { kind: 'push'; options: Pick } ); /** From 218ea114de02ed0c62581bf65ee242eb18a8a125 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 11:36:33 +0100 Subject: [PATCH 08/44] fix error response --- packages/core/data-transfer/lib/register.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 70b7b3ba6c..d114f58152 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -179,7 +179,7 @@ const createTransferController = const payload = JSON.stringify({ uuid, data: data ?? {}, - error: JSON.stringify(e?.message), + error: e?.message || 'Unknown error', }); ws.send(payload, (error) => (error ? reject(error) : resolve())); From 536e70912a555659cb6084993194e513e5906fc8 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 18:00:48 +0100 Subject: [PATCH 09/44] clean up transfer command --- .../strapi/lib/commands/transfer/transfer.js | 59 ++++++++++--------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index e91e175763..932267389f 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -4,6 +4,7 @@ const { createRemoteStrapiDestinationProvider, createLocalStrapiSourceProvider, createTransferEngine, + createLocalStrapiDestinationProvider, // TODO: we need to solve this issue with typescript modules // eslint-disable-next-line import/no-unresolved, node/no-missing-require } = require('@strapi/data-transfer'); @@ -16,17 +17,15 @@ const { DEFAULT_IGNORED_CONTENT_TYPES, } = require('./utils'); +const logger = console; + /** * @typedef ImportCommandOptions Options given to the CLI import command * - * @property {string} [file] The file path to import - * @property {boolean} [encrypt] Used to encrypt the final archive - * @property {string} [key] Encryption key, only useful when encryption is enabled - * @property {boolean} [compress] Used to compress the final archive + * @property {string} [to] The destination provider to use ("local" or the url of a remote Strapi) + * @property {string} [from] The source provider to use ("local" or the url of a remote Strapi) */ -const logger = console; - /** * Import command. * @@ -45,11 +44,27 @@ module.exports = async (opts) => { let source; let destination; + if (opts.from === 'local') { - source = createSourceProvider(strapi); + source = createLocalStrapiSourceProvider({ + getStrapi: () => strapi, + }); + } else { + logger.error(`Cannot transfer from provider '${opts.from}'`); + process.exit(1); } - if (opts.to) { - destination = createDestinationProvider({ + + if (opts.to === 'local') { + if (opts.from === 'local') { + logger.error('Source and destination cannot both be local Strapi instances.'); + process.exit(1); + } + + destination = createLocalStrapiDestinationProvider({ + getStrapi: () => strapi, + }); + } else if (opts.to) { + destination = createRemoteStrapiDestinationProvider({ url: opts.to, auth: false, strategy: 'restore', @@ -57,9 +72,13 @@ module.exports = async (opts) => { entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES }, }, }); + } else { + logger.error(`Cannot transfer from provider '${opts.from}'`); + process.exit(1); } + if (!source || !destination) { - logger.error("Couldn't create providers"); + logger.error('Could not create providers'); process.exit(1); } @@ -88,7 +107,7 @@ module.exports = async (opts) => { }); try { - logger.log(`Starting export...`); + logger.log(`Starting transfer...`); const results = await engine.transfer(); @@ -102,21 +121,3 @@ module.exports = async (opts) => { process.exit(1); } }; - -/** - * It creates a local strapi destination provider - */ -const createSourceProvider = (strapi) => { - return createLocalStrapiSourceProvider({ - async getStrapi() { - return strapi; - }, - }); -}; - -/** - * It creates a remote strapi destination provider based on the given options - */ -const createDestinationProvider = (opts) => { - return createRemoteStrapiDestinationProvider(opts); -}; From cc9a87d092941e6a6b6c2f10a8419c92f0524524 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Wed, 28 Dec 2022 12:00:03 +0100 Subject: [PATCH 10/44] clean up transfer --- packages/core/strapi/bin/strapi.js | 12 ++++++++++-- .../strapi/lib/commands/transfer/transfer.js | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/packages/core/strapi/bin/strapi.js b/packages/core/strapi/bin/strapi.js index 471be826f6..63ceeee992 100755 --- a/packages/core/strapi/bin/strapi.js +++ b/packages/core/strapi/bin/strapi.js @@ -262,8 +262,16 @@ program program .command('transfer') .description('Transfer data from one source to another') - .addOption(new Option('--from ', `Source of your data`).default('local')) - .addOption(new Option('--to ', `Destination of your data`).default('remote')) + .addOption(new Option('--from ', `URL of remote Strapi instance to get data from.`)) + .addOption(new Option('--to ', `URL of remote Strapi instance to send data to`)) + .hook('preAction', async (thisCommand) => { + const opts = thisCommand.opts(); + + if (!opts.from && !opts.to) { + console.error('At least one source (from) or destination (to) option must be provided'); + process.exit(1); + } + }) .allowExcessArguments(false) .action(getLocalScript('transfer/transfer')); diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index 932267389f..e462c837ad 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -45,7 +45,8 @@ module.exports = async (opts) => { let source; let destination; - if (opts.from === 'local') { + // if no URL provided, use local Strapi + if (!opts.from) { source = createLocalStrapiSourceProvider({ getStrapi: () => strapi, }); @@ -54,16 +55,18 @@ module.exports = async (opts) => { process.exit(1); } - if (opts.to === 'local') { - if (opts.from === 'local') { - logger.error('Source and destination cannot both be local Strapi instances.'); + // if no URL provided, use local Strapi + if (!opts.to) { + // at least one of the two must be remote, however + if (!opts.from) { + logger.error(`At least 'to' or 'from' must be provided.`); process.exit(1); } destination = createLocalStrapiDestinationProvider({ getStrapi: () => strapi, }); - } else if (opts.to) { + } else { destination = createRemoteStrapiDestinationProvider({ url: opts.to, auth: false, @@ -72,9 +75,6 @@ module.exports = async (opts) => { entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES }, }, }); - } else { - logger.error(`Cannot transfer from provider '${opts.from}'`); - process.exit(1); } if (!source || !destination) { @@ -117,7 +117,8 @@ module.exports = async (opts) => { logger.log(`${chalk.bold('Transfer process has been completed successfully!')}`); process.exit(0); } catch (e) { - logger.error('Transfer process failed unexpectedly:', e); + logger.error('Transfer process failed unexpectedly'); + logger.error(e); process.exit(1); } }; From e6e20044b9659af6fd0681c224e71fee1191daac Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Wed, 28 Dec 2022 13:01:35 +0100 Subject: [PATCH 11/44] Add tests and refactor --- .../lib/bootstrap/controllers/index.ts | 2 + .../lib/bootstrap/controllers/push.ts | 134 +++++++++ .../lib/bootstrap/controllers/transfer.ts | 147 +++++++++ .../core/data-transfer/lib/engine/index.ts | 30 +- .../lib/providers/__tests__/register.test.ts | 32 ++ .../index.ts | 2 - packages/core/data-transfer/lib/register.ts | 282 +----------------- .../__tests__/data-transfer/transfer.test.js | 12 +- 8 files changed, 336 insertions(+), 305 deletions(-) create mode 100644 packages/core/data-transfer/lib/bootstrap/controllers/index.ts create mode 100644 packages/core/data-transfer/lib/bootstrap/controllers/push.ts create mode 100644 packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts create mode 100644 packages/core/data-transfer/lib/providers/__tests__/register.test.ts diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/index.ts b/packages/core/data-transfer/lib/bootstrap/controllers/index.ts new file mode 100644 index 0000000000..314b26e75b --- /dev/null +++ b/packages/core/data-transfer/lib/bootstrap/controllers/index.ts @@ -0,0 +1,2 @@ +export * from './push'; +export { default as createTransferController } from './transfer'; diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/push.ts b/packages/core/data-transfer/lib/bootstrap/controllers/push.ts new file mode 100644 index 0000000000..06a6e1981b --- /dev/null +++ b/packages/core/data-transfer/lib/bootstrap/controllers/push.ts @@ -0,0 +1,134 @@ +import { PassThrough, Writable } from 'stream-chain'; + +import { IAsset, IMetadata, PushTransferMessage, PushTransferStage } from '../../../types'; +import { + createLocalStrapiDestinationProvider, + ILocalStrapiDestinationProviderOptions, +} from '../../providers'; + +export interface IPushController { + streams: { [stage in PushTransferStage]?: Writable }; + actions: { + getMetadata(): Promise; + getSchemas(): Strapi.Schemas; + bootstrap(): Promise; + close(): Promise; + beforeTransfer(): Promise; + }; + transfer: { + [key in PushTransferStage]: ( + value: T extends { stage: key; data: infer U } ? U : never + ) => Promise; + }; +} + +const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { + const provider = createLocalStrapiDestinationProvider(options); + + const streams: { [stage in PushTransferStage]?: Writable } = {}; + const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; + + const writeAsync = (stream: Writable, data: T) => { + return new Promise((resolve, reject) => { + stream.write(data, (error) => { + if (error) { + reject(error); + } + + resolve(); + }); + }); + }; + + return { + streams, + + actions: { + async getSchemas(): Promise { + return provider.getSchemas(); + }, + + async getMetadata() { + return provider.getMetadata(); + }, + + async bootstrap() { + return provider.bootstrap(); + }, + + async close() { + return provider.close(); + }, + + async beforeTransfer() { + return provider.beforeTransfer(); + }, + }, + + transfer: { + async entities(entity) { + if (!streams.entities) { + streams.entities = provider.getEntitiesStream(); + } + + await writeAsync(streams.entities!, entity); + }, + + async links(link) { + if (!streams.links) { + streams.links = await provider.getLinksStream(); + } + + await writeAsync(streams.links!, link); + }, + + async configuration(config) { + if (!streams.configuration) { + streams.configuration = await provider.getConfigurationStream(); + } + + await writeAsync(streams.configuration!, config); + }, + + async assets(payload) { + if (payload === null) { + streams.assets?.end(); + return; + } + + const { step, assetID } = payload; + + if (!streams.assets) { + streams.assets = await provider.getAssetsStream(); + } + + if (step === 'start') { + assets[assetID] = { ...payload.data, stream: new PassThrough() }; + writeAsync(streams.assets, assets[assetID]); + } + + if (step === 'stream') { + const chunk = Buffer.from(payload.data.chunk.data); + + await writeAsync(assets[assetID].stream, chunk); + } + + if (step === 'end') { + await new Promise((resolve, reject) => { + const { stream } = assets[assetID]; + + stream + .on('close', () => { + delete assets[assetID]; + resolve(); + }) + .on('error', reject) + .end(); + }); + } + }, + }, + }; +}; + +export default createPushController; diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts new file mode 100644 index 0000000000..d52a848a57 --- /dev/null +++ b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts @@ -0,0 +1,147 @@ +import type { Context } from 'koa'; +import type { ServerOptions } from 'ws'; + +import { v4 } from 'uuid'; +import { WebSocket } from 'ws'; + +import type { IPushController } from './push'; + +import { InitMessage, Message, TransferKind } from '../../../types'; +import createPushController from './push'; + +interface ITransferState { + kind?: TransferKind; + transferID?: string; + controller?: IPushController; +} + +const createTransferController = + (options: ServerOptions = {}) => + async (ctx: Context) => { + const upgradeHeader = (ctx.request.headers.upgrade || '') + .split(',') + .map((s) => s.trim().toLowerCase()); + + // Create the websocket server + const wss = new WebSocket.Server({ ...options, noServer: true }); + + if (upgradeHeader.includes('websocket')) { + wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => { + // Create a connection between the client & the server + wss.emit('connection', ws, ctx.req); + + const state: ITransferState = {}; + let uuid: string | undefined; + + const callback = (e: Error | null = null, data?: T) => { + return new Promise((resolve, reject) => { + if (!uuid) { + reject(new Error('Missing uuid for this message')); + return; + } + + const payload = JSON.stringify({ + uuid, + data: data ?? {}, + error: e?.message || 'Unknown error', + }); + + ws.send(payload, (error) => (error ? reject(error) : resolve())); + }); + }; + + const answer = async (fn: () => T) => { + try { + const response = await fn(); + callback(null, response); + } catch (e) { + if (e instanceof Error) { + callback(e); + } else if (typeof e === 'string') { + callback(new Error(e)); + } else { + callback(new Error('Unexpected error')); + } + } + }; + + const teardown = () => { + delete state.kind; + delete state.controller; + delete state.transferID; + + return { ok: true }; + }; + + const init = (msg: InitMessage) => { + const { kind, options: controllerOptions } = msg; + + if (state.controller) { + throw new Error('Transfer already in progres'); + } + + if (kind === 'push') { + state.controller = createPushController({ + ...controllerOptions, + autoDestroy: false, + getStrapi() { + return strapi; + }, + }); + } + + // Pull or others + else { + throw new Error(`${kind} transfer not implemented`); + } + + state.transferID = v4(); + + return { transferID: state.transferID }; + }; + + ws.on('close', () => { + teardown(); + }); + + ws.on('error', (e) => { + teardown(); + console.error(e); + }); + + ws.on('message', async (raw) => { + const msg: Message = JSON.parse(raw.toString()); + + if (!msg.uuid) { + throw new Error('Missing uuid in message'); + } + + uuid = msg.uuid; + + if (msg.type === 'init') { + await answer(() => init(msg)); + } + + if (msg.type === 'teardown') { + await answer(teardown); + } + + if (msg.type === 'action') { + await answer(() => state.controller?.actions[msg.action]?.()); + } + + if (msg.type === 'transfer') { + await answer(() => { + const { stage, data } = msg; + + return state.controller?.transfer[stage](data as never); + }); + } + }); + }); + + ctx.respond = false; + } + }; + +export default createTransferController; diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index 489ee01c9b..d6b7027ae8 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -338,23 +338,23 @@ class TransferEngine< async transfer(): Promise> { try { await this.bootstrap(); - // await this.init(); - // const isValidTransfer = await this.integrityCheck(); - // if (!isValidTransfer) { - // // TODO: provide the log from the integrity check - // throw new Error( - // `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` - // ); - // } - // await this.beforeTransfer(); + await this.init(); + const isValidTransfer = await this.integrityCheck(); + if (!isValidTransfer) { + // TODO: provide the log from the integrity check + throw new Error( + `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` + ); + } + await this.beforeTransfer(); // Run the transfer stages - // await this.transferSchemas(); - // await this.transferEntities(); - // await this.transferAssets(); - // await this.transferLinks(); - // await this.transferConfiguration(); + await this.transferSchemas(); + await this.transferEntities(); + await this.transferAssets(); + await this.transferLinks(); + await this.transferConfiguration(); // Gracefully close the providers - // await this.close(); + await this.close(); } catch (e: unknown) { // Rollback the destination provider if an exception is thrown during the transfer // Note: This will be configurable in the future diff --git a/packages/core/data-transfer/lib/providers/__tests__/register.test.ts b/packages/core/data-transfer/lib/providers/__tests__/register.test.ts new file mode 100644 index 0000000000..56ed87157b --- /dev/null +++ b/packages/core/data-transfer/lib/providers/__tests__/register.test.ts @@ -0,0 +1,32 @@ +import { createTransferController } from '../../bootstrap/controllers'; +import register from '../../register'; + +afterEach(() => { + jest.clearAllMocks(); +}); + +const strapiMock = { + admin: { + routes: { + push: jest.fn(), + }, + }, +}; + +jest.mock('../../bootstrap/controllers', () => ({ + createTransferController: jest.fn(), +})); + +describe('Register the Transfer route', () => { + test('registers the /transfer route', () => { + register(strapiMock); + expect(strapiMock.admin.routes.push).toHaveBeenCalledWith({ + method: 'GET', + path: '/transfer', + handler: createTransferController(), + config: { + auth: false, + }, + }); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index adddd27741..3424cb2559 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -186,8 +186,6 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { data: { filename, filepath, stats }, }); - console.log('is writing'); - for await (const chunk of stream) { await this.#dispatchTransfer('assets', { step: 'stream', diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index d114f58152..c971fbce77 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -1,284 +1,4 @@ -import type { Context } from 'koa'; -import type { ServerOptions } from 'ws'; -import { WebSocket } from 'ws'; -import { Writable, PassThrough } from 'stream'; -import { v4 } from 'uuid'; -import { - IAsset, - Message, - IMetadata, - PushTransferMessage, - TransferKind, - InitMessage, - PushTransferStage, -} from '../types'; -import { - ILocalStrapiDestinationProviderOptions, - createLocalStrapiDestinationProvider, -} from './providers'; - -interface ITransferState { - kind?: TransferKind; - transferID?: string; - controller?: IPushController; -} - -// Controllers - -interface IPushController { - streams: { [stage in PushTransferStage]?: Writable }; - actions: { - getMetadata(): Promise; - getSchemas(): Strapi.Schemas; - bootstrap(): Promise; - close(): Promise; - beforeTransfer(): Promise; - }; - transfer: { - [key in PushTransferStage]: ( - value: T extends { stage: key; data: infer U } ? U : never - ) => Promise; - }; -} - -const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { - const provider = createLocalStrapiDestinationProvider(options); - - const streams: { [stage in PushTransferStage]?: Writable } = {}; - const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; - - const writeAsync = (stream: Writable, data: T) => { - return new Promise((resolve, reject) => { - stream.write(data, (error) => { - if (error) { - reject(error); - } - - resolve(); - }); - }); - }; - - return { - streams, - - actions: { - async getSchemas(): Promise { - return provider.getSchemas(); - }, - - async getMetadata() { - return provider.getMetadata(); - }, - - async bootstrap() { - return provider.bootstrap(); - }, - - async close() { - return provider.close(); - }, - - async beforeTransfer() { - return provider.beforeTransfer(); - }, - }, - - transfer: { - async entities(entity) { - if (!streams.entities) { - streams.entities = provider.getEntitiesStream(); - } - - await writeAsync(streams.entities!, entity); - }, - - async links(link) { - if (!streams.links) { - streams.links = await provider.getLinksStream(); - } - - await writeAsync(streams.links!, link); - }, - - async configuration(config) { - if (!streams.configuration) { - streams.configuration = await provider.getConfigurationStream(); - } - - await writeAsync(streams.configuration!, config); - }, - - async assets(payload) { - console.log('llega'); - if (payload === null) { - streams.assets?.end(); - return; - } - - const { step, assetID } = payload; - - if (!streams.assets) { - streams.assets = await provider.getAssetsStream(); - } - - if (step === 'start') { - assets[assetID] = { ...payload.data, stream: new PassThrough() }; - writeAsync(streams.assets, assets[assetID]); - } - - if (step === 'stream') { - const chunk = Buffer.from(payload.data.chunk.data); - - await writeAsync(assets[assetID].stream, chunk); - } - - if (step === 'end') { - await new Promise((resolve, reject) => { - const { stream } = assets[assetID]; - - stream - .on('close', () => { - delete assets[assetID]; - resolve(); - }) - .on('error', reject) - .end(); - }); - } - }, - }, - }; -}; - -const createTransferController = - (options: ServerOptions = {}) => - async (ctx: Context) => { - const upgradeHeader = (ctx.request.headers.upgrade || '') - .split(',') - .map((s) => s.trim().toLowerCase()); - - // Create the websocket server - const wss = new WebSocket.Server({ ...options, noServer: true }); - - if (upgradeHeader.includes('websocket')) { - wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => { - // Create a connection between the client & the server - wss.emit('connection', ws, ctx.req); - - const state: ITransferState = {}; - let uuid: string | undefined; - - const callback = (e: Error | null = null, data?: T) => { - return new Promise((resolve, reject) => { - if (!uuid) { - reject(new Error('Missing uuid for this message')); - return; - } - - const payload = JSON.stringify({ - uuid, - data: data ?? {}, - error: e?.message || 'Unknown error', - }); - - ws.send(payload, (error) => (error ? reject(error) : resolve())); - }); - }; - - const answer = async (fn: () => T) => { - try { - const response = await fn(); - callback(null, response); - } catch (e) { - if (e instanceof Error) { - callback(e); - } else if (typeof e === 'string') { - callback(new Error(e)); - } else { - callback(new Error('Unexpected error')); - } - } - }; - - const teardown = () => { - delete state.kind; - delete state.controller; - delete state.transferID; - - return { ok: true }; - }; - - const init = (msg: InitMessage) => { - const { kind, options: controllerOptions } = msg; - - if (state.controller) { - throw new Error('Transfer already in progres'); - } - - if (kind === 'push') { - state.controller = createPushController({ - ...controllerOptions, - autoDestroy: false, - getStrapi() { - return strapi; - }, - }); - } - - // Pull or others - else { - throw new Error(`${kind} transfer not implemented`); - } - - state.transferID = v4(); - - return { transferID: state.transferID }; - }; - - ws.on('close', () => { - teardown(); - }); - - ws.on('error', (e) => { - teardown(); - console.error(e); - }); - - ws.on('message', async (raw) => { - const msg: Message = JSON.parse(raw.toString()); - - if (!msg.uuid) { - throw new Error('Missing uuid in message'); - } - - uuid = msg.uuid; - - if (msg.type === 'init') { - await answer(() => init(msg)); - } - - if (msg.type === 'teardown') { - await answer(teardown); - } - - if (msg.type === 'action') { - await answer(() => state.controller?.actions[msg.action]?.()); - } - - if (msg.type === 'transfer') { - await answer(() => { - const { stage, data } = msg; - - return state.controller?.transfer[stage](data as never); - }); - } - }); - }); - - ctx.respond = false; - } - }; +import { createTransferController } from './bootstrap/controllers'; const registerTransferRoute = (strapi: any) => { strapi.admin.routes.push({ diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 2c58f96c58..34a6a36b48 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -21,7 +21,7 @@ jest.mock( const transferCommand = require('../../transfer/transfer'); const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); -jest.spyOn(console, 'error').mockImplementation(() => {}); +const logger = jest.spyOn(console, 'error').mockImplementation(() => {}); jest.mock('../../transfer/utils'); @@ -69,10 +69,6 @@ describe('transfer', () => { expect(exit).toHaveBeenCalled(); }); - it('uses destination url provided by user with authentication', async () => { - // TODO when authentication is implemented - }); - it('uses restore as the default strategy', async () => { await transferCommand({ from: 'local', to: destinationUrl }); @@ -92,7 +88,9 @@ describe('transfer', () => { expect(exit).toHaveBeenCalled(); }); - it('creates the transfer engine successfully', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + it('Logs an error when the source provider does not exist', async () => { + await transferCommand({ from: 'test', to: destinationUrl }); + + expect(logger).toHaveBeenCalledWith("Couldn't create providers"); }); }); From 6e88b09625210c2353ab9c609d28a56fdb982ab3 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Wed, 28 Dec 2022 13:07:21 +0100 Subject: [PATCH 12/44] Remove unknown error message because it seems to be breaking the tranfser --- .../core/data-transfer/lib/bootstrap/controllers/transfer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts index d52a848a57..cf57c2f514 100644 --- a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts +++ b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts @@ -43,7 +43,7 @@ const createTransferController = const payload = JSON.stringify({ uuid, data: data ?? {}, - error: e?.message || 'Unknown error', + error: e, }); ws.send(payload, (error) => (error ? reject(error) : resolve())); From 60d8e9fc196875a553bef78a0bc48a62315f7061 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Wed, 28 Dec 2022 14:57:28 +0100 Subject: [PATCH 13/44] simplify --- .../core/strapi/lib/commands/transfer/transfer.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index e462c837ad..f05f7a8a86 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -45,6 +45,11 @@ module.exports = async (opts) => { let source; let destination; + if (!opts.from && !opts.to) { + logger.error('At least one source (from) or destination (to) option must be provided'); + process.exit(1); + } + // if no URL provided, use local Strapi if (!opts.from) { source = createLocalStrapiSourceProvider({ @@ -57,12 +62,6 @@ module.exports = async (opts) => { // if no URL provided, use local Strapi if (!opts.to) { - // at least one of the two must be remote, however - if (!opts.from) { - logger.error(`At least 'to' or 'from' must be provided.`); - process.exit(1); - } - destination = createLocalStrapiDestinationProvider({ getStrapi: () => strapi, }); From e938cb5be45970e899e45e193404b669eca5451e Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Wed, 28 Dec 2022 14:59:31 +0100 Subject: [PATCH 14/44] comments --- packages/core/strapi/lib/commands/transfer/transfer.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index f05f7a8a86..419a588387 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -55,8 +55,10 @@ module.exports = async (opts) => { source = createLocalStrapiSourceProvider({ getStrapi: () => strapi, }); - } else { - logger.error(`Cannot transfer from provider '${opts.from}'`); + } + // if URL provided, set up a remote source provider + else { + logger.error(`Remote Strapi destination provider not yet implemented`); process.exit(1); } @@ -65,7 +67,9 @@ module.exports = async (opts) => { destination = createLocalStrapiDestinationProvider({ getStrapi: () => strapi, }); - } else { + } + // if URL provided, set up a remote destination provider + else { destination = createRemoteStrapiDestinationProvider({ url: opts.to, auth: false, From 00e52aba23fa58b62dec22163a4c3cd5126f1a9f Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Wed, 28 Dec 2022 15:13:01 +0100 Subject: [PATCH 15/44] Update packages/core/strapi/lib/commands/transfer/transfer.js --- packages/core/strapi/lib/commands/transfer/transfer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index 419a588387..77a0577ac3 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -58,7 +58,7 @@ module.exports = async (opts) => { } // if URL provided, set up a remote source provider else { - logger.error(`Remote Strapi destination provider not yet implemented`); + logger.error(`Remote Strapi source provider not yet implemented`); process.exit(1); } From 5f702ab1ced3d6bf622ad491fe4963c9903ba41f Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Wed, 28 Dec 2022 18:45:24 +0100 Subject: [PATCH 16/44] lint --- packages/core/admin/server/register.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/core/admin/server/register.js b/packages/core/admin/server/register.js index 1789abf77e..e804832e2c 100644 --- a/packages/core/admin/server/register.js +++ b/packages/core/admin/server/register.js @@ -1,5 +1,7 @@ 'use strict'; +// TODO: we need to solve this issue with typescript modules +// eslint-disable-next-line import/no-unresolved, node/no-missing-require const { register: registerDataTransfer } = require('@strapi/data-transfer'); const registerAdminPanelRoute = require('./routes/serve-admin-panel'); From 807800b9a5d6fd3a496eb55ea04cdc324fabcfb7 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 29 Dec 2022 11:12:09 +0100 Subject: [PATCH 17/44] Clean the code --- .../lib/providers/local-file-destination-provider/index.ts | 1 - .../lib/providers/remote-strapi-destination-provider/index.ts | 1 - .../lib/commands/__tests__/data-transfer/transfer.test.js | 4 +--- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts index 3799915c74..081ebef70b 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts @@ -255,7 +255,6 @@ class LocalFileDestinationProvider implements IDestinationProvider { entry .on('finish', () => { - console.log('FINISH WRITING ALREADY'); callback(null); }) .on('error', (error) => { diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 3424cb2559..d2ac160085 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -172,7 +172,6 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, final: async (callback) => { - console.log('FINAL'); const e = await this.#dispatchTransfer('assets', null); callback(e); }, diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 34a6a36b48..0f26e65b42 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -44,9 +44,7 @@ describe('transfer', () => { expect(exit).toHaveBeenCalled(); }); - it('uses destination url provided by user with authentication', async () => { - // TODO when authentication is implemented - }); + it.todo('uses destination url provided by user with authentication'); it('uses restore as the default strategy', async () => { await transferCommand({ from: 'local', to: destinationUrl }); From 3cd8e1e8c06c2a5ed364d3c7f29bf967837efe90 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 29 Dec 2022 13:04:14 +0100 Subject: [PATCH 18/44] Fix mock path --- .../strapi/lib/commands/__tests__/data-transfer/export.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index 5fc0fb544b..1598c48453 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -41,7 +41,7 @@ describe('export', () => { getDefaultExportName: jest.fn(() => defaultFileName), }; jest.mock( - '../transfer/utils', + '../../transfer/utils', () => { return mockUtils; }, From 998d9677706e061d4dc24fa61ec06bf5479a5aa0 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 29 Dec 2022 13:21:10 +0100 Subject: [PATCH 19/44] Fix the exit verifications --- .../__tests__/data-transfer/transfer.test.js | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 0f26e65b42..508384672c 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -18,9 +18,19 @@ jest.mock( { virtual: true } ); +const expectExit = async (code, fn) => { + const exit = jest.spyOn(process, 'exit').mockImplementation((number) => { + throw new Error(`process.exit: ${number}`); + }); + await expect(async () => { + await fn(); + }).rejects.toThrow(); + expect(exit).toHaveBeenCalledWith(code); + exit.mockRestore(); +}; + const transferCommand = require('../../transfer/transfer'); -const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); const logger = jest.spyOn(console, 'error').mockImplementation(() => {}); jest.mock('../../transfer/utils'); @@ -33,21 +43,23 @@ describe('transfer', () => { }); it('uses destination url provided by user without authentication', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ url: destinationUrl, }) ); - - expect(exit).toHaveBeenCalled(); }); it.todo('uses destination url provided by user with authentication'); it('uses restore as the default strategy', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ @@ -56,19 +68,21 @@ describe('transfer', () => { ); }); it('uses destination url provided by user without authentication', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ url: destinationUrl, }) ); - - expect(exit).toHaveBeenCalled(); }); it('uses restore as the default strategy', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ @@ -78,16 +92,18 @@ describe('transfer', () => { }); it('uses local strapi instance when local specified', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createLocalStrapiSourceProvider).toHaveBeenCalled(); expect(utils.createStrapiInstance).toHaveBeenCalled(); - - expect(exit).toHaveBeenCalled(); }); it('Logs an error when the source provider does not exist', async () => { - await transferCommand({ from: 'test', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'test', to: destinationUrl }); + }); expect(logger).toHaveBeenCalledWith("Couldn't create providers"); }); From e4eae7aa7949c36cc874072d71e4ba63add6f9c0 Mon Sep 17 00:00:00 2001 From: Convly Date: Thu, 29 Dec 2022 16:02:54 +0100 Subject: [PATCH 20/44] (tmp) Add first remote transfer protocol typings --- .eslintrc.back.js | 1 + .eslintrc.back.typescript.js | 1 + .../types/remote/protocol/client/commands.ts | 23 +++++++++++++++ .../types/remote/protocol/client/index.ts | 8 +++++ .../remote/protocol/client/transfer/action.ts | 3 ++ .../remote/protocol/client/transfer/index.ts | 13 +++++++++ .../remote/protocol/client/transfer/pull.ts | 8 +++++ .../remote/protocol/client/transfer/push.ts | 20 +++++++++++++ .../remote/protocol/client/transfer/utils.ts | 5 ++++ .../types/remote/protocol/server/error.ts | 29 +++++++++++++++++++ .../types/remote/protocol/server/index.ts | 2 ++ .../types/remote/protocol/server/messaging.ts | 10 +++++++ 12 files changed, 123 insertions(+) create mode 100644 packages/core/data-transfer/types/remote/protocol/client/commands.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/client/index.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/client/transfer/action.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/client/transfer/index.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/client/transfer/pull.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/client/transfer/push.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/client/transfer/utils.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/server/error.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/server/index.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/server/messaging.ts diff --git a/.eslintrc.back.js b/.eslintrc.back.js index 1c079b8326..a4068ece55 100644 --- a/.eslintrc.back.js +++ b/.eslintrc.back.js @@ -31,6 +31,7 @@ module.exports = { 'no-continue': 'warn', 'no-process-exit': 'off', 'no-loop-func': 'off', + 'max-classes-per-file': 'off', 'no-param-reassign': [ 'error', { diff --git a/.eslintrc.back.typescript.js b/.eslintrc.back.typescript.js index 2e4260bd42..0c6543c94e 100644 --- a/.eslintrc.back.typescript.js +++ b/.eslintrc.back.typescript.js @@ -22,6 +22,7 @@ module.exports = { '@typescript-eslint/brace-style': 'off', // TODO: fix conflict with prettier/prettier in data-transfer/engine/index.ts // to be cleaned up throughout codebase (too many to fix at the moment) '@typescript-eslint/no-use-before-define': 'warn', + '@typescript-eslint/comma-dangle': 'off', }, // Disable only for tests overrides: [ diff --git a/packages/core/data-transfer/types/remote/protocol/client/commands.ts b/packages/core/data-transfer/types/remote/protocol/client/commands.ts new file mode 100644 index 0000000000..3fbc45866a --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/commands.ts @@ -0,0 +1,23 @@ +import { ILocalStrapiDestinationProviderOptions } from '../../../../lib'; + +export type CommandType = 'command'; + +export type CommandMessage = { type: CommandType } & (InitCommand | EndCommand); + +export type InitCommand = CreateCommand< + 'init', + | { + transfer: 'push'; + options: Pick; + } + | { transfer: 'pull' } +>; + +export type EndCommand = CreateCommand<'end', { uid: string }>; + +export type StatusCommand = CreateCommand<'status'>; + +type CreateCommand = never> = { + type: 'command'; + command: T; +} & ([U] extends [never] ? unknown : { params: U }); diff --git a/packages/core/data-transfer/types/remote/protocol/client/index.ts b/packages/core/data-transfer/types/remote/protocol/client/index.ts new file mode 100644 index 0000000000..fd6ef79544 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/index.ts @@ -0,0 +1,8 @@ +import type { CommandMessage } from './commands'; +import type { TransferMessage } from './transfer'; + +export * from './commands'; +export * from './transfer'; + +export type Message = CommandMessage | TransferMessage; +export type MessageType = Message['type']; diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/action.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/action.ts new file mode 100644 index 0000000000..c729791b8b --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/action.ts @@ -0,0 +1,3 @@ +import type { CreateTransferMessage } from './utils'; + +export type Action = CreateTransferMessage<'action', { action: string }>; diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/index.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/index.ts new file mode 100644 index 0000000000..387f3a596e --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/index.ts @@ -0,0 +1,13 @@ +import type { Action } from './action'; +import type { TransferPullMessage } from './pull'; +import type { TransferPushMessage } from './push'; + +export * from './action'; +export * from './pull'; +export * from './push'; + +export type TransferMessage = { type: 'transfer' } & ( + | Action + | TransferPushMessage + | TransferPullMessage +); diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.ts new file mode 100644 index 0000000000..43b07f62d2 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.ts @@ -0,0 +1,8 @@ +import { CreateTransferMessage } from './utils'; + +export type TransferPullMessage = CreateTransferMessage< + 'step', + { + action: 'start' | 'stop'; + } +>; diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/push.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/push.ts new file mode 100644 index 0000000000..0703f46b4a --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/push.ts @@ -0,0 +1,20 @@ +import type { CreateTransferMessage } from './utils'; +import type { IEntity, ILink, IConfiguration, IAsset } from '../../../../common-entities'; + +export type TransferPushMessage = CreateTransferMessage< + 'step', + | TransferStepCommands<'entities', IEntity> + | TransferStepCommands<'links', ILink> + | TransferStepCommands<'configuration', IConfiguration> + | TransferStepCommands<'assets', TransferAssetFlow> +>; + +type TransferStepCommands = { step: T } & TransferStepFlow; + +type TransferStepFlow = { action: 'start' } | { action: 'stream'; data: U } | { action: 'end' }; + +type TransferAssetFlow = { assetID: string } & ( + | { action: 'start'; data: Omit } + | { action: 'stream'; data: Uint8Array } + | { action: 'end' } +); diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.ts new file mode 100644 index 0000000000..be51e5ef5b --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.ts @@ -0,0 +1,5 @@ +export type CreateTransferMessage = { + type: 'transfer'; + kind: T; + id: string; +} & U; diff --git a/packages/core/data-transfer/types/remote/protocol/server/error.ts b/packages/core/data-transfer/types/remote/protocol/server/error.ts new file mode 100644 index 0000000000..5925f89235 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/error.ts @@ -0,0 +1,29 @@ +export enum ErrorKind { + // Generic + Unknown = 0, + // Chunk transfer + DiscardChunk = 1, + InvalidChunkFormat = 2, +} + +export class ServerError extends Error { + constructor( + public code: ErrorKind, + public message: string, + public details?: Record | null + ) { + super(message); + } +} + +export class UnknownError extends ServerError { + constructor(message: string, details?: Record | null) { + super(ErrorKind.Unknown, message, details); + } +} + +export class DiscardChunkError extends ServerError { + constructor(message: string, details?: Record | null) { + super(ErrorKind.DiscardChunk, message, details); + } +} diff --git a/packages/core/data-transfer/types/remote/protocol/server/index.ts b/packages/core/data-transfer/types/remote/protocol/server/index.ts new file mode 100644 index 0000000000..4e327b46e4 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/index.ts @@ -0,0 +1,2 @@ +export * as messaging from './messaging'; +export * as error from './error'; diff --git a/packages/core/data-transfer/types/remote/protocol/server/messaging.ts b/packages/core/data-transfer/types/remote/protocol/server/messaging.ts new file mode 100644 index 0000000000..5d12bbf065 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/messaging.ts @@ -0,0 +1,10 @@ +import type { ServerError } from './error'; + +export type Message = { + id?: string; + data?: T | null; + error?: ServerError | null; +}; + +// Successful +export type OK = Message<{ ok: true }>; From 160a1683b50dd0487dd9fffc66888e800c616b6c Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Fri, 30 Dec 2022 09:48:36 +0100 Subject: [PATCH 21/44] Add no missing require to data-transfer --- packages/core/admin/server/register.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/admin/server/register.js b/packages/core/admin/server/register.js index 1789abf77e..b02e8def3d 100644 --- a/packages/core/admin/server/register.js +++ b/packages/core/admin/server/register.js @@ -1,5 +1,6 @@ 'use strict'; +// eslint-disable-next-line import/no-unresolved, node/no-missing-require const { register: registerDataTransfer } = require('@strapi/data-transfer'); const registerAdminPanelRoute = require('./routes/serve-admin-panel'); From 9c0be5731a803da2de356420c8c49ede5c1c9d23 Mon Sep 17 00:00:00 2001 From: Convly Date: Fri, 30 Dec 2022 10:51:01 +0100 Subject: [PATCH 22/44] Add status command --- .../data-transfer/types/remote/protocol/client/commands.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/core/data-transfer/types/remote/protocol/client/commands.ts b/packages/core/data-transfer/types/remote/protocol/client/commands.ts index 3fbc45866a..3cdc3b86f0 100644 --- a/packages/core/data-transfer/types/remote/protocol/client/commands.ts +++ b/packages/core/data-transfer/types/remote/protocol/client/commands.ts @@ -1,8 +1,6 @@ import { ILocalStrapiDestinationProviderOptions } from '../../../../lib'; -export type CommandType = 'command'; - -export type CommandMessage = { type: CommandType } & (InitCommand | EndCommand); +export type CommandMessage = { type: 'command' } & (InitCommand | EndCommand | StatusCommand); export type InitCommand = CreateCommand< 'init', From 3eac888f72dc74f914e51c525f5a65a6f19609db Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 09:34:38 +0100 Subject: [PATCH 23/44] fix comments --- packages/core/strapi/lib/commands/transfer/transfer.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index 77a0577ac3..ac20042842 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -20,18 +20,18 @@ const { const logger = console; /** - * @typedef ImportCommandOptions Options given to the CLI import command + * @typedef TransferCommandOptions Options given to the CLI transfer command * - * @property {string} [to] The destination provider to use ("local" or the url of a remote Strapi) - * @property {string} [from] The source provider to use ("local" or the url of a remote Strapi) + * @property {string} [to] The url of a remote Strapi to use as remote destination + * @property {string} [from] The url of a remote Strapi to use as remote source */ /** - * Import command. + * Transfer command. * * It transfers data from a local file to a local strapi instance * - * @param {ImportCommandOptions} opts + * @param {TransferCommandOptions} opts */ module.exports = async (opts) => { // Validate inputs from Commander From adb318fc2c9971f9a472fdfd4b02538d7931a74b Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 09:53:20 +0100 Subject: [PATCH 24/44] fix typedef --- packages/core/strapi/lib/commands/transfer/transfer.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index ac20042842..48b42c527b 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -22,8 +22,8 @@ const logger = console; /** * @typedef TransferCommandOptions Options given to the CLI transfer command * - * @property {string} [to] The url of a remote Strapi to use as remote destination - * @property {string} [from] The url of a remote Strapi to use as remote source + * @property {string|undefined} [to] The url of a remote Strapi to use as remote destination + * @property {string|undefined} [from] The url of a remote Strapi to use as remote source */ /** From 611f991820e78f847a232cf171782f9fba43f4fb Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 2 Jan 2023 15:30:37 +0100 Subject: [PATCH 25/44] data-transfer: Move from lib to src & update folder architecture --- packages/core/data-transfer/.gitignore | 1 + packages/core/data-transfer/.npmignore | 11 +-- .../lib/bootstrap/controllers/index.ts | 2 - packages/core/data-transfer/lib/index.ts | 4 - .../lib/providers/__tests__/register.test.ts | 32 ------- .../core/data-transfer/lib/providers/index.ts | 9 -- .../lib/providers/shared/index.ts | 1 - .../lib/providers/test-utils/index.ts | 83 ------------------- packages/core/data-transfer/lib/register.ts | 16 ---- .../core/data-transfer/lib/utils/schema.ts | 20 ----- .../core/data-transfer/lib/utils/stream.ts | 48 ----------- packages/core/data-transfer/package.json | 10 +-- .../{lib => src}/__tests__/test-utils.ts | 7 ++ .../engine/__tests__/engine.test.ts | 0 .../{lib => src}/engine/index.ts | 2 +- .../src/engine/validation/index.ts | 1 + .../engine/validation/schemas}/index.ts | 6 +- packages/core/data-transfer/src/file/index.ts | 1 + .../destination}/__tests__/index.test.ts | 6 +- .../destination}/__tests__/utils.test.ts | 0 .../file/providers/destination}/index.ts | 4 +- .../file/providers/destination}/utils.ts | 0 .../data-transfer/src/file/providers/index.ts | 2 + .../providers/source}/__tests__/index.test.ts | 0 .../file/providers/source}/index.ts | 27 +++--- packages/core/data-transfer/src/index.ts | 4 + .../src/strapi/__tests__/register.test.ts | 36 ++++++++ .../core/data-transfer/src/strapi/index.ts | 5 ++ .../src/strapi/providers/index.ts | 6 ++ .../__tests__/assets.test.ts | 4 +- .../__tests__/index.test.ts | 6 +- .../__tests__/restore.test.ts | 8 +- .../providers/local-destination}/index.ts | 8 +- .../local-destination}/strategies/index.ts | 0 .../strategies/restore/configuration.ts | 2 +- .../strategies/restore/entities.ts | 28 +++++-- .../strategies/restore/index.ts | 4 +- .../strategies/restore/links.ts | 4 +- .../__tests__/configuration.test.ts | 6 +- .../local-source}/__tests__/entities.test.ts | 4 +- .../local-source}/__tests__/index.test.ts | 8 +- .../local-source}/__tests__/links.test.ts | 2 +- .../strapi/providers/local-source}/assets.ts | 2 +- .../providers/local-source}/configuration.ts | 2 +- .../providers/local-source}/entities.ts | 4 +- .../strapi/providers/local-source}/index.ts | 4 +- .../strapi/providers/local-source}/links.ts | 4 +- .../__tests__/index.test.ts | 0 .../__tests__/utils.test.ts | 0 .../providers/remote-destination}/index.ts | 4 +- .../providers/remote-destination}/utils.ts | 0 .../strapi => src/strapi/queries}/entity.ts | 0 .../strapi => src/strapi/queries}/index.ts | 0 .../strapi => src/strapi/queries}/link.ts | 3 +- .../core/data-transfer/src/strapi/register.ts | 12 +++ .../src/strapi/remote/controllers/index.ts | 1 + .../strapi/remote}/controllers/push.ts | 8 +- .../strapi/remote/handlers.ts} | 9 +- .../data-transfer/src/strapi/remote/index.ts | 2 + .../data-transfer/src/strapi/remote/routes.ts | 35 ++++++++ .../encryption/__tests__/encrypt.test.ts | 0 .../{lib => src/utils}/encryption/decrypt.ts | 10 ++- .../{lib => src/utils}/encryption/encrypt.ts | 10 ++- .../{lib => src/utils}/encryption/index.ts | 0 .../data-transfer/{lib => src}/utils/index.ts | 1 + .../data-transfer/{lib => src}/utils/json.ts | 9 +- .../core/data-transfer/src/utils/schema.ts | 27 ++++++ .../core/data-transfer/src/utils/stream.ts | 72 ++++++++++++++++ packages/core/data-transfer/tsconfig.json | 6 +- packages/core/data-transfer/types/remote.d.ts | 2 +- 70 files changed, 352 insertions(+), 303 deletions(-) create mode 100644 packages/core/data-transfer/.gitignore delete mode 100644 packages/core/data-transfer/lib/bootstrap/controllers/index.ts delete mode 100644 packages/core/data-transfer/lib/index.ts delete mode 100644 packages/core/data-transfer/lib/providers/__tests__/register.test.ts delete mode 100644 packages/core/data-transfer/lib/providers/index.ts delete mode 100644 packages/core/data-transfer/lib/providers/shared/index.ts delete mode 100644 packages/core/data-transfer/lib/providers/test-utils/index.ts delete mode 100644 packages/core/data-transfer/lib/register.ts delete mode 100644 packages/core/data-transfer/lib/utils/schema.ts delete mode 100644 packages/core/data-transfer/lib/utils/stream.ts rename packages/core/data-transfer/{lib => src}/__tests__/test-utils.ts (97%) rename packages/core/data-transfer/{lib => src}/engine/__tests__/engine.test.ts (100%) rename packages/core/data-transfer/{lib => src}/engine/index.ts (99%) create mode 100644 packages/core/data-transfer/src/engine/validation/index.ts rename packages/core/data-transfer/{lib/strategies => src/engine/validation/schemas}/index.ts (87%) create mode 100644 packages/core/data-transfer/src/file/index.ts rename packages/core/data-transfer/{lib/providers/local-file-destination-provider => src/file/providers/destination}/__tests__/index.test.ts (97%) rename packages/core/data-transfer/{lib/providers/local-file-destination-provider => src/file/providers/destination}/__tests__/utils.test.ts (100%) rename packages/core/data-transfer/{lib/providers/local-file-destination-provider => src/file/providers/destination}/index.ts (98%) rename packages/core/data-transfer/{lib/providers/local-file-destination-provider => src/file/providers/destination}/utils.ts (100%) create mode 100644 packages/core/data-transfer/src/file/providers/index.ts rename packages/core/data-transfer/{lib/providers/local-file-source-provider => src/file/providers/source}/__tests__/index.test.ts (100%) rename packages/core/data-transfer/{lib/providers/local-file-source-provider => src/file/providers/source}/index.ts (91%) create mode 100644 packages/core/data-transfer/src/index.ts create mode 100644 packages/core/data-transfer/src/strapi/__tests__/register.test.ts create mode 100644 packages/core/data-transfer/src/strapi/index.ts create mode 100644 packages/core/data-transfer/src/strapi/providers/index.ts rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/__tests__/assets.test.ts (94%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/__tests__/index.test.ts (97%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/__tests__/restore.test.ts (95%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/index.ts (96%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/strategies/index.ts (100%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/strategies/restore/configuration.ts (95%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/strategies/restore/entities.ts (71%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/strategies/restore/index.ts (96%) rename packages/core/data-transfer/{lib/providers/local-strapi-destination-provider => src/strapi/providers/local-destination}/strategies/restore/links.ts (88%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/__tests__/configuration.test.ts (91%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/__tests__/entities.test.ts (98%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/__tests__/index.test.ts (97%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/__tests__/links.test.ts (99%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/assets.ts (95%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/configuration.ts (94%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/entities.ts (95%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/index.ts (98%) rename packages/core/data-transfer/{lib/providers/local-strapi-source-provider => src/strapi/providers/local-source}/links.ts (87%) rename packages/core/data-transfer/{lib/providers/remote-strapi-destination-provider => src/strapi/providers/remote-destination}/__tests__/index.test.ts (100%) rename packages/core/data-transfer/{lib/providers/remote-strapi-destination-provider => src/strapi/providers/remote-destination}/__tests__/utils.test.ts (100%) rename packages/core/data-transfer/{lib/providers/remote-strapi-destination-provider => src/strapi/providers/remote-destination}/index.ts (98%) rename packages/core/data-transfer/{lib/providers/remote-strapi-destination-provider => src/strapi/providers/remote-destination}/utils.ts (100%) rename packages/core/data-transfer/{lib/providers/shared/strapi => src/strapi/queries}/entity.ts (100%) rename packages/core/data-transfer/{lib/providers/shared/strapi => src/strapi/queries}/index.ts (100%) rename packages/core/data-transfer/{lib/providers/shared/strapi => src/strapi/queries}/link.ts (98%) create mode 100644 packages/core/data-transfer/src/strapi/register.ts create mode 100644 packages/core/data-transfer/src/strapi/remote/controllers/index.ts rename packages/core/data-transfer/{lib/bootstrap => src/strapi/remote}/controllers/push.ts (94%) rename packages/core/data-transfer/{lib/bootstrap/controllers/transfer.ts => src/strapi/remote/handlers.ts} (94%) create mode 100644 packages/core/data-transfer/src/strapi/remote/index.ts create mode 100644 packages/core/data-transfer/src/strapi/remote/routes.ts rename packages/core/data-transfer/{lib => src/utils}/encryption/__tests__/encrypt.test.ts (100%) rename packages/core/data-transfer/{lib => src/utils}/encryption/decrypt.ts (86%) rename packages/core/data-transfer/{lib => src/utils}/encryption/encrypt.ts (86%) rename packages/core/data-transfer/{lib => src/utils}/encryption/index.ts (100%) rename packages/core/data-transfer/{lib => src}/utils/index.ts (70%) rename packages/core/data-transfer/{lib => src}/utils/json.ts (89%) create mode 100644 packages/core/data-transfer/src/utils/schema.ts create mode 100644 packages/core/data-transfer/src/utils/stream.ts diff --git a/packages/core/data-transfer/.gitignore b/packages/core/data-transfer/.gitignore new file mode 100644 index 0000000000..c3af857904 --- /dev/null +++ b/packages/core/data-transfer/.gitignore @@ -0,0 +1 @@ +lib/ diff --git a/packages/core/data-transfer/.npmignore b/packages/core/data-transfer/.npmignore index 5c1b474909..1bf5894edf 100644 --- a/packages/core/data-transfer/.npmignore +++ b/packages/core/data-transfer/.npmignore @@ -1,9 +1,9 @@ ############################# -# DATA TRANSFER X +# DATA TRANSFER ############################ *.js.map -lib/ +src/ types/ tsconfig.json @@ -19,14 +19,12 @@ Icon .Trashes ._* - ############################ # Linux ############################ *~ - ############################ # Windows ############################ @@ -40,7 +38,6 @@ $RECYCLE.BIN/ *.msm *.msp - ############################ # Packages ############################ @@ -69,7 +66,6 @@ $RECYCLE.BIN/ *.out *.pid - ############################ # Logs and databases ############################ @@ -79,7 +75,6 @@ $RECYCLE.BIN/ *.sql *.sqlite - ############################ # Misc. ############################ @@ -89,7 +84,6 @@ $RECYCLE.BIN/ nbproject .vscode/ - ############################ # Node.js ############################ @@ -107,7 +101,6 @@ package-lock.json !docs/package-lock.json *.heapsnapshot - ############################ # Tests ############################ diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/index.ts b/packages/core/data-transfer/lib/bootstrap/controllers/index.ts deleted file mode 100644 index 314b26e75b..0000000000 --- a/packages/core/data-transfer/lib/bootstrap/controllers/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './push'; -export { default as createTransferController } from './transfer'; diff --git a/packages/core/data-transfer/lib/index.ts b/packages/core/data-transfer/lib/index.ts deleted file mode 100644 index eba8ce7d72..0000000000 --- a/packages/core/data-transfer/lib/index.ts +++ /dev/null @@ -1,4 +0,0 @@ -export * from './engine'; -export * from './providers'; - -export { default as register } from './register'; diff --git a/packages/core/data-transfer/lib/providers/__tests__/register.test.ts b/packages/core/data-transfer/lib/providers/__tests__/register.test.ts deleted file mode 100644 index 56ed87157b..0000000000 --- a/packages/core/data-transfer/lib/providers/__tests__/register.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { createTransferController } from '../../bootstrap/controllers'; -import register from '../../register'; - -afterEach(() => { - jest.clearAllMocks(); -}); - -const strapiMock = { - admin: { - routes: { - push: jest.fn(), - }, - }, -}; - -jest.mock('../../bootstrap/controllers', () => ({ - createTransferController: jest.fn(), -})); - -describe('Register the Transfer route', () => { - test('registers the /transfer route', () => { - register(strapiMock); - expect(strapiMock.admin.routes.push).toHaveBeenCalledWith({ - method: 'GET', - path: '/transfer', - handler: createTransferController(), - config: { - auth: false, - }, - }); - }); -}); diff --git a/packages/core/data-transfer/lib/providers/index.ts b/packages/core/data-transfer/lib/providers/index.ts deleted file mode 100644 index 04397d7349..0000000000 --- a/packages/core/data-transfer/lib/providers/index.ts +++ /dev/null @@ -1,9 +0,0 @@ -// source providers -export * from './local-file-source-provider'; -export * from './local-strapi-source-provider'; - -// destination providers -export * from './local-file-destination-provider'; -export * from './local-strapi-destination-provider'; - -export * from './remote-strapi-destination-provider'; diff --git a/packages/core/data-transfer/lib/providers/shared/index.ts b/packages/core/data-transfer/lib/providers/shared/index.ts deleted file mode 100644 index d3f3965b83..0000000000 --- a/packages/core/data-transfer/lib/providers/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * as strapi from './strapi'; diff --git a/packages/core/data-transfer/lib/providers/test-utils/index.ts b/packages/core/data-transfer/lib/providers/test-utils/index.ts deleted file mode 100644 index c09362e900..0000000000 --- a/packages/core/data-transfer/lib/providers/test-utils/index.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Readable } from 'stream'; - -/** - * Collect every entity in a Readable stream - */ -export const collect = (stream: Readable): Promise => { - const chunks: T[] = []; - - return new Promise((resolve, reject) => { - stream - .on('data', (chunk) => chunks.push(chunk)) - .on('close', () => resolve(chunks)) - .on('error', reject); - }); -}; - -/** - * Create a "Strapi" like object factory based on the - * given params and cast it to the correct type - */ -export const getStrapiFactory = - < - T extends { - [key in keyof Partial]: unknown; - } - >( - properties?: T - ) => - () => { - return { ...properties } as Strapi.Strapi; - }; - -/** - * Union type used to represent the default content types available - */ -export type ContentType = 'foo' | 'bar'; - -/** - * Factory to get default content types test values - */ -export const getContentTypes = (): { - [key in ContentType]: { uid: key; attributes: { [attribute: string]: unknown } }; -} => ({ - foo: { uid: 'foo', attributes: { title: { type: 'string' } } }, - bar: { uid: 'bar', attributes: { age: { type: 'number' } } }, -}); - -/** - * Create a factory of readable streams (wrapped with a jest mock function) - */ -export const createMockedReadableFactory = (source: { - [ct in T]: Array<{ id: number; [key: string]: unknown }>; -}) => - jest.fn((uid: T) => { - return Readable.from(source[uid] || []); - }); - -/** - * Create a factory of mocked query builders - */ -export const createMockedQueryBuilder = (data: { - [key in T]: unknown[]; -}) => - jest.fn((uid: T) => { - const state: { [key: string]: unknown } = { populate: undefined }; - - return { - populate(populate: unknown) { - state.populate = populate; - return this; - }, - stream() { - return Readable.from(data[uid]); - }, - }; - }); - -/** - * Update the global store with the given strapi value - */ -export const setGlobalStrapi = (strapi: Strapi.Strapi): void => { - (global as unknown as Global).strapi = strapi; -}; diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts deleted file mode 100644 index c971fbce77..0000000000 --- a/packages/core/data-transfer/lib/register.ts +++ /dev/null @@ -1,16 +0,0 @@ -import { createTransferController } from './bootstrap/controllers'; - -const registerTransferRoute = (strapi: any) => { - strapi.admin.routes.push({ - method: 'GET', - path: '/transfer', - handler: createTransferController(), - config: { auth: false }, - }); -}; - -const register = (strapi: any) => { - registerTransferRoute(strapi); -}; - -export default register; diff --git a/packages/core/data-transfer/lib/utils/schema.ts b/packages/core/data-transfer/lib/utils/schema.ts deleted file mode 100644 index d59c12e22a..0000000000 --- a/packages/core/data-transfer/lib/utils/schema.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { Schema } from '@strapi/strapi'; -import { mapValues, pick } from 'lodash/fp'; - -const schemaSelectedKeys = [ - 'collectionName', - 'info', - 'options', - 'pluginOptions', - 'attributes', - 'kind', - 'modelType', - 'modelName', - 'uid', - 'plugin', - 'globalId', -]; - -export const mapSchemasValues = (schemas: Record) => { - return mapValues(pick(schemaSelectedKeys), schemas); -}; diff --git a/packages/core/data-transfer/lib/utils/stream.ts b/packages/core/data-transfer/lib/utils/stream.ts deleted file mode 100644 index 21c1c48da8..0000000000 --- a/packages/core/data-transfer/lib/utils/stream.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Transform, Readable } from 'stream'; - -type TransformOptions = ConstructorParameters[0]; - -export const filter = ( - predicate: (value: T) => boolean | Promise, - options: TransformOptions = { objectMode: true } -): Transform => { - return new Transform({ - ...options, - - async transform(chunk, _encoding, callback) { - const keep = await predicate(chunk); - - callback(null, keep ? chunk : undefined); - }, - }); -}; - -export const map = ( - predicate: (value: T) => T | Promise, - options: TransformOptions = { objectMode: true } -): Transform => { - return new Transform({ - ...options, - - async transform(chunk, _encoding, callback) { - const mappedValue = await predicate(chunk); - - callback(null, mappedValue); - }, - }); -}; - -/** - * Collect every entity in a Readable stream - */ -export const collect = (stream: Readable): Promise => { - const chunks: T[] = []; - - return new Promise((resolve) => { - stream.on('data', (chunk) => chunks.push(chunk)); - stream.on('end', () => { - stream.destroy(); - resolve(chunks); - }); - }); -}; diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index f6c905aec4..88a1808c94 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -24,17 +24,17 @@ "url": "https://strapi.io" } ], - "main": "./dist/index.js", - "types": "./dist/index.d.ts", + "main": "./lib/index.js", + "types": "./lib/index.d.ts", "scripts": { "build": "tsc -p tsconfig.json", - "clean": "rimraf ./dist", + "clean": "rimraf ./lib", "build:clean": "yarn clean && yarn build", "watch": "yarn build -w --preserveWatchOutput", "test:unit": "jest --verbose" }, "directories": { - "lib": "./dist" + "lib": "./lib" }, "dependencies": { "@strapi/logger": "4.5.4", @@ -61,7 +61,7 @@ "@types/tar": "6.1.3", "@types/tar-stream": "2.2.2", "@types/uuid": "9.0.0", - "koa": "2.14.1", + "@types/koa": "2.13.1", "rimraf": "3.0.2", "typescript": "4.6.2" }, diff --git a/packages/core/data-transfer/lib/__tests__/test-utils.ts b/packages/core/data-transfer/src/__tests__/test-utils.ts similarity index 97% rename from packages/core/data-transfer/lib/__tests__/test-utils.ts rename to packages/core/data-transfer/src/__tests__/test-utils.ts index 476f2992fa..8a86e4b3e7 100644 --- a/packages/core/data-transfer/lib/__tests__/test-utils.ts +++ b/packages/core/data-transfer/src/__tests__/test-utils.ts @@ -102,6 +102,13 @@ export const destinationStages = [ 'getSchemasStream', ]; +/** + * Update the global store with the given strapi value + */ +export const setGlobalStrapi = (strapi: Strapi.Strapi): void => { + (global as unknown as Global).strapi = strapi; +}; + /** * Add jest expect helpers */ diff --git a/packages/core/data-transfer/lib/engine/__tests__/engine.test.ts b/packages/core/data-transfer/src/engine/__tests__/engine.test.ts similarity index 100% rename from packages/core/data-transfer/lib/engine/__tests__/engine.test.ts rename to packages/core/data-transfer/src/engine/__tests__/engine.test.ts diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/src/engine/index.ts similarity index 99% rename from packages/core/data-transfer/lib/engine/index.ts rename to packages/core/data-transfer/src/engine/index.ts index f7d15e1f05..9e080295dd 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/src/engine/index.ts @@ -19,7 +19,7 @@ import type { } from '../../types'; import type { Diff } from '../utils/json'; -import compareSchemas from '../strategies'; +import { compareSchemas } from './validation/schemas'; import { filter, map } from '../utils/stream'; export const TRANSFER_STAGES: ReadonlyArray = Object.freeze([ diff --git a/packages/core/data-transfer/src/engine/validation/index.ts b/packages/core/data-transfer/src/engine/validation/index.ts new file mode 100644 index 0000000000..197f1788ef --- /dev/null +++ b/packages/core/data-transfer/src/engine/validation/index.ts @@ -0,0 +1 @@ +export * as schemas from './schemas'; diff --git a/packages/core/data-transfer/lib/strategies/index.ts b/packages/core/data-transfer/src/engine/validation/schemas/index.ts similarity index 87% rename from packages/core/data-transfer/lib/strategies/index.ts rename to packages/core/data-transfer/src/engine/validation/schemas/index.ts index c0b69bb2f2..4d9f096235 100644 --- a/packages/core/data-transfer/lib/strategies/index.ts +++ b/packages/core/data-transfer/src/engine/validation/schemas/index.ts @@ -1,5 +1,5 @@ -import type { Diff } from '../utils/json'; -import * as utils from '../utils'; +import type { Diff } from '../../../utils/json'; +import * as utils from '../../../utils'; const strategies = { // No diffs @@ -32,4 +32,4 @@ const compareSchemas = (a: T, b: P, strategy: keyof typeof strategies) => return strategies[strategy](diffs); }; -export default compareSchemas; +export { compareSchemas }; diff --git a/packages/core/data-transfer/src/file/index.ts b/packages/core/data-transfer/src/file/index.ts new file mode 100644 index 0000000000..15f74296f5 --- /dev/null +++ b/packages/core/data-transfer/src/file/index.ts @@ -0,0 +1 @@ +export * as providers from './providers'; diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts similarity index 97% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts index 85580dece6..245d22070a 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts @@ -6,7 +6,7 @@ jest.mock('fs'); import fs from 'fs-extra'; import { Writable } from 'stream-chain'; import { createLocalFileDestinationProvider, ILocalFileDestinationProviderOptions } from '..'; -import * as encryption from '../../../encryption/encrypt'; +import * as encryption from '../../../../utils/encryption'; import { createFilePathFactory, createTarEntryStream } from '../utils'; fs.createWriteStream = jest.fn().mockReturnValue( @@ -18,14 +18,14 @@ fs.createWriteStream = jest.fn().mockReturnValue( const filePath = './test-file'; -jest.mock('../../../encryption/encrypt', () => { +jest.mock('../../../../utils/encryption', () => { return { __esModule: true, createEncryptionCipher() {}, }; }); -jest.mock('../../local-file-destination-provider/utils'); +jest.mock('../utils'); describe('Local File Destination Provider', () => { (createFilePathFactory as jest.Mock).mockImplementation(jest.fn()); diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts b/packages/core/data-transfer/src/file/providers/destination/__tests__/utils.test.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts rename to packages/core/data-transfer/src/file/providers/destination/__tests__/utils.test.ts diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts b/packages/core/data-transfer/src/file/providers/destination/index.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts rename to packages/core/data-transfer/src/file/providers/destination/index.ts index 081ebef70b..e064b5435b 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts +++ b/packages/core/data-transfer/src/file/providers/destination/index.ts @@ -6,7 +6,7 @@ import { stringer } from 'stream-json/jsonl/Stringer'; import { chain } from 'stream-chain'; import { Readable, Writable } from 'stream'; -import { createEncryptionCipher } from '../../encryption/encrypt'; +import { createEncryptionCipher } from '../../../utils/encryption'; import type { IAsset, IDestinationProvider, @@ -14,7 +14,7 @@ import type { IMetadata, ProviderType, Stream, -} from '../../../types'; +} from '../../../../types'; import { createFilePathFactory, createTarEntryStream } from './utils'; export interface ILocalFileDestinationProviderOptions { diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts b/packages/core/data-transfer/src/file/providers/destination/utils.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts rename to packages/core/data-transfer/src/file/providers/destination/utils.ts diff --git a/packages/core/data-transfer/src/file/providers/index.ts b/packages/core/data-transfer/src/file/providers/index.ts new file mode 100644 index 0000000000..d3c54e88d0 --- /dev/null +++ b/packages/core/data-transfer/src/file/providers/index.ts @@ -0,0 +1,2 @@ +export * from './source'; +export * from './destination'; diff --git a/packages/core/data-transfer/lib/providers/local-file-source-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/file/providers/source/__tests__/index.test.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-file-source-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/file/providers/source/__tests__/index.test.ts diff --git a/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts b/packages/core/data-transfer/src/file/providers/source/index.ts similarity index 91% rename from packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts rename to packages/core/data-transfer/src/file/providers/source/index.ts index ff078043dd..b020769e32 100644 --- a/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts +++ b/packages/core/data-transfer/src/file/providers/source/index.ts @@ -8,10 +8,9 @@ import { keyBy } from 'lodash/fp'; import { chain } from 'stream-chain'; import { pipeline, PassThrough } from 'stream'; import { parser } from 'stream-json/jsonl/Parser'; -import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../types'; +import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../../types'; -import { createDecryptionCipher } from '../../encryption'; -import * as utils from '../../utils'; +import * as utils from '../../../utils'; type StreamItemArray = Parameters[0]; @@ -152,7 +151,7 @@ class LocalFileSourceProvider implements ISourceProvider { } if (encryption.enabled && encryption.key) { - streams.push(createDecryptionCipher(encryption.key)); + streams.push(utils.encryption.createDecryptionCipher(encryption.key)); } if (compression.enabled) { @@ -227,20 +226,22 @@ class LocalFileSourceProvider implements ISourceProvider { return entryPath === filePath && entry.type === 'File'; }, - /** - * Whenever an entry passes the filter method, process it - */ async onentry(entry) { // Collect all the content of the entry file const content = await entry.collect(); - // Parse from buffer to string to JSON - const parsedContent = JSON.parse(content.toString()); - // Resolve the Promise with the parsed content - resolve(parsedContent); + try { + // Parse from buffer to string to JSON + const parsedContent = JSON.parse(content.toString()); - // Cleanup (close the stream associated to the entry) - entry.destroy(); + // Resolve the Promise with the parsed content + resolve(parsedContent); + } catch (e) { + reject(e); + } finally { + // Cleanup (close the stream associated to the entry) + entry.destroy(); + } }, }), ], diff --git a/packages/core/data-transfer/src/index.ts b/packages/core/data-transfer/src/index.ts new file mode 100644 index 0000000000..f6c4c97772 --- /dev/null +++ b/packages/core/data-transfer/src/index.ts @@ -0,0 +1,4 @@ +export * as engine from './engine'; +export * as strapi from './strapi'; +export * as file from './file'; +export * as utils from './utils'; diff --git a/packages/core/data-transfer/src/strapi/__tests__/register.test.ts b/packages/core/data-transfer/src/strapi/__tests__/register.test.ts new file mode 100644 index 0000000000..dc739fb52d --- /dev/null +++ b/packages/core/data-transfer/src/strapi/__tests__/register.test.ts @@ -0,0 +1,36 @@ +import { getStrapiFactory } from '../../__tests__/test-utils'; + +import { createTransferHandler } from '../remote/handlers'; +import register from '../register'; + +afterEach(() => { + jest.clearAllMocks(); +}); + +const strapiMockFactory = getStrapiFactory({ + admin: { + routes: { + push: jest.fn(), + }, + }, +}); + +jest.mock('../remote/handlers', () => ({ + createTransferHandler: jest.fn(), +})); + +describe('Register the Transfer route', () => { + test('registers the /transfer route', () => { + const strapi = strapiMockFactory(); + + register(strapi); + expect(strapi.admin.routes.push).toHaveBeenCalledWith({ + method: 'GET', + path: '/transfer', + handler: createTransferHandler(), + config: { + auth: false, + }, + }); + }); +}); diff --git a/packages/core/data-transfer/src/strapi/index.ts b/packages/core/data-transfer/src/strapi/index.ts new file mode 100644 index 0000000000..5866e4cf11 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/index.ts @@ -0,0 +1,5 @@ +export * as providers from './providers'; +export * as queries from './queries'; +export * as remote from './remote'; + +export { default as register } from './register'; diff --git a/packages/core/data-transfer/src/strapi/providers/index.ts b/packages/core/data-transfer/src/strapi/providers/index.ts new file mode 100644 index 0000000000..4f3e079af7 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/providers/index.ts @@ -0,0 +1,6 @@ +// Local +export * from './local-destination'; +export * from './local-source'; + +// Remote +export * from './remote-destination'; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/assets.test.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts similarity index 94% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/assets.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts index 6851617b5c..33ea37edc4 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/assets.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts @@ -1,8 +1,8 @@ import fse from 'fs-extra'; import { Writable, Readable } from 'stream'; -import type { IAsset } from '../../../../types'; +import type { IAsset } from '../../../../../types'; -import { getStrapiFactory } from '../../../__tests__/test-utils'; +import { getStrapiFactory } from '../../../../__tests__/test-utils'; import { createLocalStrapiDestinationProvider } from '../index'; const write = jest.fn((_chunk, _encoding, callback) => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts similarity index 97% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts index 3e4c8ff88f..1be96c02bf 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts @@ -1,6 +1,10 @@ import { createLocalStrapiDestinationProvider } from '../index'; import * as restoreApi from '../strategies/restore'; -import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils'; +import { + getStrapiFactory, + getContentTypes, + setGlobalStrapi, +} from '../../../../__tests__/test-utils'; afterEach(() => { jest.clearAllMocks(); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts similarity index 95% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts index 71b969c461..0776fc6dcc 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts @@ -1,6 +1,10 @@ import { deleteRecords, restoreConfigs } from '../strategies/restore'; -import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils'; -import { IConfiguration } from '../../../../types'; +import { + getStrapiFactory, + getContentTypes, + setGlobalStrapi, +} from '../../../../__tests__/test-utils'; +import { IConfiguration } from '../../../../../types'; const entities = [ { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts similarity index 96% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/index.ts index b89e8001de..0225b83571 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts @@ -1,10 +1,10 @@ import { Writable } from 'stream'; import path from 'path'; import * as fse from 'fs-extra'; -import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../types'; +import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../../types'; import { restore } from './strategies'; -import * as utils from '../../utils'; +import * as utils from '../../../utils'; export const VALID_CONFLICT_STRATEGIES = ['restore', 'merge']; export const DEFAULT_CONFLICT_STRATEGY = 'restore'; @@ -25,6 +25,9 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { strapi?: Strapi.Strapi; + /** + * The entities mapper is used to map old entities to their new IDs + */ #entitiesMapper: { [type: string]: { [id: number]: number } }; constructor(options: ILocalStrapiDestinationProviderOptions) { @@ -123,6 +126,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { throw new Error(`Invalid strategy supplied: "${strategy}"`); } + // TODO: Move this logic to the restore strategy async getAssetsStream(): Promise { if (!this.strapi) { throw new Error('Not able to stream Assets. Strapi instance not found'); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/index.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/index.ts diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/configuration.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts similarity index 95% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/configuration.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts index eae954e619..e1cb04b345 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/configuration.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts @@ -1,6 +1,6 @@ import { Writable } from 'stream'; import chalk from 'chalk'; -import { IConfiguration } from '../../../../../types'; +import { IConfiguration } from '../../../../../../types'; const restoreCoreStore = async (strapi: Strapi.Strapi, data: T) => { return strapi.db.query('strapi::core-store').create({ diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts similarity index 71% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts index c120098599..d3eb397a00 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts @@ -3,9 +3,9 @@ import type { SchemaUID } from '@strapi/strapi/lib/types/utils'; import { get } from 'lodash/fp'; import { Writable } from 'stream'; -import type { IEntity } from '../../../../../types'; -import { json } from '../../../../utils'; -import * as shared from '../../../shared'; +import type { IEntity } from '../../../../../../types'; +import { json } from '../../../../../utils'; +import * as queries from '../../../../queries'; interface IEntitiesRestoreStreamOptions { strapi: Strapi.Strapi; @@ -14,7 +14,7 @@ interface IEntitiesRestoreStreamOptions { const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { const { strapi, updateMappingTable } = options; - const query = shared.strapi.entity.createEntityQuery(strapi); + const query = queries.entity.createEntityQuery(strapi); return new Writable({ objectMode: true, @@ -24,13 +24,19 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { const { create, getDeepPopulateComponentLikeQuery } = query(type); const contentType = strapi.getModel(type); - const resolveType = (paths: string[]) => { + /** + * Resolve the component UID of an entity's attribute based + * on a given path (components & dynamic zones only) + */ + const resolveType = (paths: string[]): string | undefined => { let cType = contentType; let value: unknown = data; for (const path of paths) { value = get(path, value); + // Needed when the value of cType should be computed + // based on the next value (eg: dynamic zones) if (typeof cType === 'function') { cType = cType(value); } @@ -48,24 +54,32 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { } } - return cType.uid; + return cType?.uid; }; try { - // Create the entity const created = await create({ data, populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }), select: 'id', }); + // Compute differences between original & new entities const diffs = json.diff(data, created); updateMappingTable(type, id, created.id); + // For each difference found on an ID attribute, + // update the mapping the table accordingly diffs.forEach((diff) => { if (diff.kind === 'modified' && diff.path.at(-1) === 'id') { const target = resolveType(diff.path); + + // If no type is found for the given path, then ignore the diff + if (!target) { + return; + } + const [oldID, newID] = diff.values as [number, number]; updateMappingTable(target, oldID, newID); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts similarity index 96% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts index f325c67ecd..90f9505fdb 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts @@ -1,5 +1,5 @@ import type { ContentTypeSchema } from '@strapi/strapi'; -import * as shared from '../../../shared'; +import * as queries from '../../../../queries'; export interface IRestoreOptions { assets?: boolean; @@ -36,7 +36,7 @@ const deleteEntitiesRecord = async ( options: IRestoreOptions = {} ): Promise => { const { entities } = options; - const query = shared.strapi.entity.createEntityQuery(strapi); + const query = queries.entity.createEntityQuery(strapi); const contentTypes = Object.values(strapi.contentTypes); const contentTypesToClear = contentTypes.filter((contentType) => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/links.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts similarity index 88% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/links.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts index d4258625a4..89e9b0b695 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/links.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts @@ -1,6 +1,6 @@ import { Writable } from 'stream'; -import { ILink } from '../../../../../types'; -import { createLinkQuery } from '../../../shared/strapi/link'; +import { ILink } from '../../../../../../types'; +import { createLinkQuery } from '../../../../queries/link'; export const createLinksWriteStream = ( mapID: (uid: string, id: number) => number | undefined, diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/configuration.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts similarity index 91% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/configuration.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts index 088d92471b..225a886b28 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/configuration.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts @@ -1,6 +1,10 @@ import { Readable } from 'stream'; -import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils'; +import { + collect, + createMockedQueryBuilder, + getStrapiFactory, +} from '../../../../__tests__/test-utils'; import { createConfigurationStream } from '../configuration'; describe('Configuration', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts index ef4c29c15b..e29ab5e4f2 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts @@ -1,12 +1,12 @@ import { Readable, PassThrough } from 'stream'; -import type { IEntity } from '../../../../types'; +import type { IEntity } from '../../../../../types'; import { collect, getStrapiFactory, getContentTypes, createMockedQueryBuilder, -} from '../../../__tests__/test-utils'; +} from '../../../../__tests__/test-utils'; import { createEntitiesStream, createEntitiesTransformStream } from '../entities'; describe('Local Strapi Source Provider - Entities Streaming', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts similarity index 97% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts index 342c5a9ba2..77ecb8e9db 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts @@ -1,7 +1,11 @@ import { Readable } from 'stream'; -import type { IEntity } from '../../../../types'; +import type { IEntity } from '../../../../../types'; -import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils'; +import { + collect, + createMockedQueryBuilder, + getStrapiFactory, +} from '../../../../__tests__/test-utils'; import { createLocalStrapiSourceProvider } from '..'; describe('Local Strapi Source Provider', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts similarity index 99% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts index be6420d615..5c4aec8f70 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts @@ -1,7 +1,7 @@ import { Readable } from 'stream'; import { createLinksStream } from '../links'; -import { collect, getStrapiFactory } from '../../../__tests__/test-utils'; +import { collect, getStrapiFactory } from '../../../../__tests__/test-utils'; // TODO: entityService needs to be replaced with a mocked wrapper of db.connection and provide real metadata describe.skip('Local Strapi Source Provider - Entities Streaming', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/assets.ts b/packages/core/data-transfer/src/strapi/providers/local-source/assets.ts similarity index 95% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/assets.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/assets.ts index 3edc89aaba..8cdbf67699 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/assets.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/assets.ts @@ -2,7 +2,7 @@ import { join } from 'path'; import { readdir, stat, createReadStream } from 'fs-extra'; import { Duplex } from 'stream'; -import type { IAsset } from '../../../types'; +import type { IAsset } from '../../../../types'; const IGNORED_FILES = ['.gitkeep']; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts b/packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts similarity index 94% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts index c956887398..9340d98ded 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts @@ -2,7 +2,7 @@ import { chain } from 'stream-chain'; import { Readable } from 'stream'; import { set } from 'lodash/fp'; -import type { IConfiguration } from '../../../types'; +import type { IConfiguration } from '../../../../types'; /** * Create a readable stream that export the Strapi app configuration diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts b/packages/core/data-transfer/src/strapi/providers/local-source/entities.ts similarity index 95% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/entities.ts index fe765fbcd7..7aba49f26f 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/entities.ts @@ -1,8 +1,8 @@ import type { ContentTypeSchema } from '@strapi/strapi'; import { Readable, PassThrough } from 'stream'; -import * as shared from '../shared/strapi'; -import { IEntity } from '../../../types'; +import * as shared from '../../queries'; +import { IEntity } from '../../../../types'; /** * Generate and consume content-types streams in order to stream each entity individually diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts b/packages/core/data-transfer/src/strapi/providers/local-source/index.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/index.ts index 181eed4c25..f8cfa75232 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/index.ts @@ -1,12 +1,12 @@ import { chain } from 'stream-chain'; import { Readable } from 'stream'; -import type { IMetadata, ISourceProvider, ProviderType } from '../../../types'; +import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types'; import { createEntitiesStream, createEntitiesTransformStream } from './entities'; import { createLinksStream } from './links'; import { createConfigurationStream } from './configuration'; import { createAssetsStream } from './assets'; -import * as utils from '../../utils'; +import * as utils from '../../../utils'; export interface ILocalStrapiSourceProviderOptions { getStrapi(): Strapi.Strapi | Promise; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links.ts b/packages/core/data-transfer/src/strapi/providers/local-source/links.ts similarity index 87% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/links.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/links.ts index 5387c45002..91e4d3fd96 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/links.ts @@ -1,7 +1,7 @@ import { Readable } from 'stream'; -import type { ILink } from '../../../types'; -import { createLinkQuery } from '../shared/strapi/link'; +import type { ILink } from '../../../../types'; +import { createLinkQuery } from '../../queries/link'; /** * Create a Readable which will stream all the links from a Strapi instance diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts rename to packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts rename to packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts index d2ac160085..4b3003792e 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts @@ -11,8 +11,8 @@ import type { IConfiguration, TransferStage, IAsset, -} from '../../../types'; -import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; +} from '../../../../types'; +import type { ILocalStrapiDestinationProviderOptions } from '../local-destination'; import { dispatch } from './utils'; interface ITokenAuth { diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts rename to packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts diff --git a/packages/core/data-transfer/lib/providers/shared/strapi/entity.ts b/packages/core/data-transfer/src/strapi/queries/entity.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/shared/strapi/entity.ts rename to packages/core/data-transfer/src/strapi/queries/entity.ts diff --git a/packages/core/data-transfer/lib/providers/shared/strapi/index.ts b/packages/core/data-transfer/src/strapi/queries/index.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/shared/strapi/index.ts rename to packages/core/data-transfer/src/strapi/queries/index.ts diff --git a/packages/core/data-transfer/lib/providers/shared/strapi/link.ts b/packages/core/data-transfer/src/strapi/queries/link.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/shared/strapi/link.ts rename to packages/core/data-transfer/src/strapi/queries/link.ts index 55855a4667..ca6f7be345 100644 --- a/packages/core/data-transfer/lib/providers/shared/strapi/link.ts +++ b/packages/core/data-transfer/src/strapi/queries/link.ts @@ -1,5 +1,6 @@ +import { RelationAttribute } from '@strapi/strapi'; import { clone, isNil } from 'lodash/fp'; -import { ILink } from '../../../../types'; +import { ILink } from '../../../types'; // TODO: Remove any types when we'll have types for DB metadata diff --git a/packages/core/data-transfer/src/strapi/register.ts b/packages/core/data-transfer/src/strapi/register.ts new file mode 100644 index 0000000000..0217733e4f --- /dev/null +++ b/packages/core/data-transfer/src/strapi/register.ts @@ -0,0 +1,12 @@ +import { routes } from './remote'; + +/** + * This is intended to be called on Strapi register phase. + * + * It registers a transfer route in the Strapi admin router. + */ +const register = (strapi: Strapi.Strapi) => { + routes.registerAdminTransferRoute(strapi); +}; + +export default register; diff --git a/packages/core/data-transfer/src/strapi/remote/controllers/index.ts b/packages/core/data-transfer/src/strapi/remote/controllers/index.ts new file mode 100644 index 0000000000..553009a4ea --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/controllers/index.ts @@ -0,0 +1 @@ +export * from './push'; diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/push.ts b/packages/core/data-transfer/src/strapi/remote/controllers/push.ts similarity index 94% rename from packages/core/data-transfer/lib/bootstrap/controllers/push.ts rename to packages/core/data-transfer/src/strapi/remote/controllers/push.ts index 06a6e1981b..caec12632e 100644 --- a/packages/core/data-transfer/lib/bootstrap/controllers/push.ts +++ b/packages/core/data-transfer/src/strapi/remote/controllers/push.ts @@ -1,6 +1,6 @@ import { PassThrough, Writable } from 'stream-chain'; -import { IAsset, IMetadata, PushTransferMessage, PushTransferStage } from '../../../types'; +import { IAsset, IMetadata, PushTransferMessage, PushTransferStage } from '../../../../types'; import { createLocalStrapiDestinationProvider, ILocalStrapiDestinationProviderOptions, @@ -71,7 +71,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): streams.entities = provider.getEntitiesStream(); } - await writeAsync(streams.entities!, entity); + await writeAsync(streams.entities, entity); }, async links(link) { @@ -79,7 +79,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): streams.links = await provider.getLinksStream(); } - await writeAsync(streams.links!, link); + await writeAsync(streams.links, link); }, async configuration(config) { @@ -87,7 +87,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): streams.configuration = await provider.getConfigurationStream(); } - await writeAsync(streams.configuration!, config); + await writeAsync(streams.configuration, config); }, async assets(payload) { diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts b/packages/core/data-transfer/src/strapi/remote/handlers.ts similarity index 94% rename from packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts rename to packages/core/data-transfer/src/strapi/remote/handlers.ts index cf57c2f514..dff8124001 100644 --- a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts +++ b/packages/core/data-transfer/src/strapi/remote/handlers.ts @@ -1,13 +1,14 @@ +// eslint-disable-next-line node/no-extraneous-import import type { Context } from 'koa'; import type { ServerOptions } from 'ws'; import { v4 } from 'uuid'; import { WebSocket } from 'ws'; -import type { IPushController } from './push'; +import type { IPushController } from './controllers/push'; import { InitMessage, Message, TransferKind } from '../../../types'; -import createPushController from './push'; +import createPushController from './controllers/push'; interface ITransferState { kind?: TransferKind; @@ -15,7 +16,7 @@ interface ITransferState { controller?: IPushController; } -const createTransferController = +export const createTransferHandler = (options: ServerOptions = {}) => async (ctx: Context) => { const upgradeHeader = (ctx.request.headers.upgrade || '') @@ -143,5 +144,3 @@ const createTransferController = ctx.respond = false; } }; - -export default createTransferController; diff --git a/packages/core/data-transfer/src/strapi/remote/index.ts b/packages/core/data-transfer/src/strapi/remote/index.ts new file mode 100644 index 0000000000..af2810cd4c --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/index.ts @@ -0,0 +1,2 @@ +export * as controllers from './controllers'; +export * as routes from './routes'; diff --git a/packages/core/data-transfer/src/strapi/remote/routes.ts b/packages/core/data-transfer/src/strapi/remote/routes.ts new file mode 100644 index 0000000000..34fc5585e7 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/routes.ts @@ -0,0 +1,35 @@ +// eslint-disable-next-line node/no-extraneous-import +import type { Context } from 'koa'; + +import { createTransferHandler } from './handlers'; + +// Extend Strapi interface type to access the admin routes' API +// TODO: Remove this when the Strapi instances will be better typed +declare module '@strapi/strapi' { + interface Strapi { + admin: { + routes: { + method: string; + path: string; + handler: (ctx: Context) => Promise; + config: unknown; + }[]; + }; + } +} + +/** + * Register a transfer route in the Strapi admin router. + * + * It exposes a WS server that can be used to run and manage transfer processes. + * + * @param strapi - A Strapi instance + */ +export const registerAdminTransferRoute = (strapi: Strapi.Strapi) => { + strapi.admin.routes.push({ + method: 'GET', + path: '/transfer', + handler: createTransferHandler(), + config: { auth: false }, + }); +}; diff --git a/packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts b/packages/core/data-transfer/src/utils/encryption/__tests__/encrypt.test.ts similarity index 100% rename from packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts rename to packages/core/data-transfer/src/utils/encryption/__tests__/encrypt.test.ts diff --git a/packages/core/data-transfer/lib/encryption/decrypt.ts b/packages/core/data-transfer/src/utils/encryption/decrypt.ts similarity index 86% rename from packages/core/data-transfer/lib/encryption/decrypt.ts rename to packages/core/data-transfer/src/utils/encryption/decrypt.ts index e7a8e4c7a2..3687e9af43 100644 --- a/packages/core/data-transfer/lib/encryption/decrypt.ts +++ b/packages/core/data-transfer/src/utils/encryption/decrypt.ts @@ -1,5 +1,5 @@ import { Cipher, scryptSync, CipherKey, BinaryLike, createDecipheriv } from 'crypto'; -import { EncryptionStrategy, Strategies, Algorithm } from '../../types'; +import { EncryptionStrategy, Strategies, Algorithm } from '../../../types'; // different key values depending on algorithm chosen const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { @@ -33,6 +33,14 @@ const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { return strategies[algorithm]; }; +/** + * It creates a cipher instance used for decryption + * + * @param key - The decryption key + * @param algorithm - The algorithm to use to create the Cipher + * + * @returns A {@link Cipher} instance created with the given key & algorithm + */ export const createDecryptionCipher = ( key: string, algorithm: Algorithm = 'aes-128-ecb' diff --git a/packages/core/data-transfer/lib/encryption/encrypt.ts b/packages/core/data-transfer/src/utils/encryption/encrypt.ts similarity index 86% rename from packages/core/data-transfer/lib/encryption/encrypt.ts rename to packages/core/data-transfer/src/utils/encryption/encrypt.ts index d32e2cd6e9..5fea673d76 100644 --- a/packages/core/data-transfer/lib/encryption/encrypt.ts +++ b/packages/core/data-transfer/src/utils/encryption/encrypt.ts @@ -1,5 +1,5 @@ import { createCipheriv, Cipher, scryptSync, CipherKey, BinaryLike } from 'crypto'; -import { EncryptionStrategy, Strategies, Algorithm } from '../../types'; +import { EncryptionStrategy, Strategies, Algorithm } from '../../../types'; // different key values depending on algorithm chosen const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { @@ -33,6 +33,14 @@ const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { return strategies[algorithm]; }; +/** + * It creates a cipher instance used for encryption + * + * @param key - The encryption key + * @param algorithm - The algorithm to use to create the Cipher + * + * @returns A {@link Cipher} instance created with the given key & algorithm + */ export const createEncryptionCipher = ( key: string, algorithm: Algorithm = 'aes-128-ecb' diff --git a/packages/core/data-transfer/lib/encryption/index.ts b/packages/core/data-transfer/src/utils/encryption/index.ts similarity index 100% rename from packages/core/data-transfer/lib/encryption/index.ts rename to packages/core/data-transfer/src/utils/encryption/index.ts diff --git a/packages/core/data-transfer/lib/utils/index.ts b/packages/core/data-transfer/src/utils/index.ts similarity index 70% rename from packages/core/data-transfer/lib/utils/index.ts rename to packages/core/data-transfer/src/utils/index.ts index 069e3aae93..0203b50860 100644 --- a/packages/core/data-transfer/lib/utils/index.ts +++ b/packages/core/data-transfer/src/utils/index.ts @@ -1,3 +1,4 @@ +export * as encryption from './encryption'; export * as stream from './stream'; export * as json from './json'; export * as schema from './schema'; diff --git a/packages/core/data-transfer/lib/utils/json.ts b/packages/core/data-transfer/src/utils/json.ts similarity index 89% rename from packages/core/data-transfer/lib/utils/json.ts rename to packages/core/data-transfer/src/utils/json.ts index ee14f29c6b..d305ee1cab 100644 --- a/packages/core/data-transfer/lib/utils/json.ts +++ b/packages/core/data-transfer/src/utils/json.ts @@ -2,6 +2,13 @@ import { isArray, isObject, zip, isEqual, uniq } from 'lodash/fp'; const createContext = (): Context => ({ path: [] }); +/** + * Compute differences between two JSON objects and returns them + * + * @param a - First object + * @param b - Second object + * @param ctx - Context used to keep track of the current path during recursion + */ export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Diff[] => { const diffs: Diff[] = []; const { path } = ctx; @@ -70,7 +77,7 @@ export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Di } if (!isEqual(a, b)) { - modified(); + return modified(); } return diffs; diff --git a/packages/core/data-transfer/src/utils/schema.ts b/packages/core/data-transfer/src/utils/schema.ts new file mode 100644 index 0000000000..1ab4c1dfde --- /dev/null +++ b/packages/core/data-transfer/src/utils/schema.ts @@ -0,0 +1,27 @@ +import type { Schema } from '@strapi/strapi'; +import { mapValues, pick } from 'lodash/fp'; + +/** + * List of schema properties that should be kept when sanitizing schemas + */ +const VALID_SCHEMA_PROPERTIES = [ + 'collectionName', + 'info', + 'options', + 'pluginOptions', + 'attributes', + 'kind', + 'modelType', + 'modelName', + 'uid', + 'plugin', + 'globalId', +]; + +/** + * Sanitize a schemas dictionnary by omiting unwanted properties + * The list of allowed properties can be found here: {@link VALID_SCHEMA_PROPERTIES} + */ +export const mapSchemasValues = (schemas: Record) => { + return mapValues(pick(VALID_SCHEMA_PROPERTIES), schemas); +}; diff --git a/packages/core/data-transfer/src/utils/stream.ts b/packages/core/data-transfer/src/utils/stream.ts new file mode 100644 index 0000000000..1715f31122 --- /dev/null +++ b/packages/core/data-transfer/src/utils/stream.ts @@ -0,0 +1,72 @@ +import { Transform, Readable } from 'stream'; + +type TransformOptions = ConstructorParameters[0]; + +/** + * Create a filter stream that discard chunks which doesn't satisfies the given predicate + * + * @param predicate - A filter predicate, takes a stream data chunk as parameter and returns a boolean value + * @param options - Transform stream options + */ +export const filter = ( + predicate: (value: T) => boolean | Promise, + options: TransformOptions = { objectMode: true } +): Transform => { + return new Transform({ + ...options, + + async transform(chunk, _encoding, callback) { + const keep = await predicate(chunk); + + callback(null, keep ? chunk : undefined); + }, + }); +}; + +/** + * Create a map stream that transform chunks using the given predicate + * + * @param predicate - A map predicate, takes a stream data chunk as parameter and returns a mapped value + * @param options - Transform stream options + */ +export const map = ( + predicate: (value: T) => U | Promise, + options: TransformOptions = { objectMode: true } +): Transform => { + return new Transform({ + ...options, + + async transform(chunk, _encoding, callback) { + const mappedValue = await predicate(chunk); + + callback(null, mappedValue); + }, + }); +}; + +/** + * Collect every chunks from a Readable stream. + * + * @param stream - The redable stream to collect data from + * @param options.destroy - If set to true, it automatically calls `destroy()` on the given stream upon receiving the 'end' event + */ +export const collect = ( + stream: Readable, + options: { destroy: boolean } = { destroy: true } +): Promise => { + const chunks: T[] = []; + + return new Promise((resolve, reject) => { + stream + .on('close', () => resolve(chunks)) + .on('error', reject) + .on('data', (chunk) => chunks.push(chunk)) + .on('end', () => { + if (options.destroy) { + stream.destroy(); + } + + resolve(chunks); + }); + }); +}; diff --git a/packages/core/data-transfer/tsconfig.json b/packages/core/data-transfer/tsconfig.json index ff65f30217..cf0ec9c76a 100644 --- a/packages/core/data-transfer/tsconfig.json +++ b/packages/core/data-transfer/tsconfig.json @@ -4,11 +4,11 @@ "strict": true, "lib": ["ESNEXT"], "skipLibCheck": true, - "outDir": "dist", + "outDir": "lib", "declaration": true, "sourceMap": true, "esModuleInterop": true }, - "include": ["types", "lib/**/*.ts"], - "exclude": ["node_modules", "lib/**/__tests__"] + "include": ["types", "src/**/*.ts"], + "exclude": ["node_modules", "src/**/__tests__"] } diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts index cb64d420c1..c46db32bd1 100644 --- a/packages/core/data-transfer/types/remote.d.ts +++ b/packages/core/data-transfer/types/remote.d.ts @@ -1,4 +1,4 @@ -import type { ILocalStrapiDestinationProviderOptions } from '../lib'; +import type { ILocalStrapiDestinationProviderOptions } from '../src/strapi/providers'; import type { IAsset, IConfiguration, IEntity, ILink } from './common-entities'; /** From 34ca7adbe24d136cc0f814a8053dd7a227a05061 Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 2 Jan 2023 15:33:15 +0100 Subject: [PATCH 26/44] Update import path & fix tests --- packages/core/admin/server/register.js | 4 +- .../__tests__/data-transfer/export.test.js | 60 ++++++++++--------- .../__tests__/data-transfer/transfer.test.js | 45 ++++++++------ .../strapi/lib/commands/transfer/export.js | 12 ++-- .../strapi/lib/commands/transfer/import.js | 19 +++--- .../strapi/lib/commands/transfer/transfer.js | 9 +-- 6 files changed, 83 insertions(+), 66 deletions(-) diff --git a/packages/core/admin/server/register.js b/packages/core/admin/server/register.js index 1789abf77e..932f1b7f69 100644 --- a/packages/core/admin/server/register.js +++ b/packages/core/admin/server/register.js @@ -1,12 +1,12 @@ 'use strict'; -const { register: registerDataTransfer } = require('@strapi/data-transfer'); +const { register: registerDataTransfer } = require('@strapi/data-transfer/lib/strapi'); const registerAdminPanelRoute = require('./routes/serve-admin-panel'); const adminAuthStrategy = require('./strategies/admin'); const apiTokenAuthStrategy = require('./strategies/api-token'); -module.exports = ({ strapi }) => { +module.exports = async ({ strapi }) => { const passportMiddleware = strapi.admin.services.passport.init(); strapi.server.api('admin').use(passportMiddleware); diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index 1598c48453..cc04449279 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -1,33 +1,39 @@ 'use strict'; -describe('export', () => { +describe('Export', () => { const defaultFileName = 'defaultFilename'; - // mock @strapi/data-transfer const mockDataTransfer = { - createLocalFileDestinationProvider: jest.fn().mockReturnValue({ name: 'testDest' }), - createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testSource' }), - createTransferEngine() { - return { - transfer: jest.fn().mockReturnValue(Promise.resolve({})), - progress: { - on: jest.fn(), - stream: { + file: { + providers: { + createLocalFileDestinationProvider: jest.fn().mockReturnValue({ name: 'testDest' }), + }, + }, + strapi: { + providers: { + createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testSource' }), + }, + }, + engine: { + createTransferEngine() { + return { + transfer: jest.fn().mockReturnValue(Promise.resolve({})), + progress: { on: jest.fn(), + stream: { + on: jest.fn(), + }, }, - }, - sourceProvider: { name: 'testSource' }, - destinationProvider: { name: 'testDestination' }, - }; + sourceProvider: { name: 'testSource' }, + destinationProvider: { name: 'testDestination' }, + }; + }, }, }; - jest.mock( - '@strapi/data-transfer', - () => { - return mockDataTransfer; - }, - { virtual: true } - ); + + jest.mock('@strapi/data-transfer/lib/engine', () => mockDataTransfer.engine, { virtual: true }); + jest.mock('@strapi/data-transfer/lib/strapi', () => mockDataTransfer.strapi, { virtual: true }); + jest.mock('@strapi/data-transfer/lib/file', () => mockDataTransfer.file, { virtual: true }); // mock utils const mockUtils = { @@ -76,7 +82,7 @@ describe('export', () => { await exportCommand({ file: filename }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ file: { path: filename }, }) @@ -90,7 +96,7 @@ describe('export', () => { }); expect(mockUtils.getDefaultExportName).toHaveBeenCalledTimes(1); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ file: { path: defaultFileName }, }) @@ -103,7 +109,7 @@ describe('export', () => { await exportCommand({ encrypt }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ encryption: { enabled: encrypt }, }) @@ -117,7 +123,7 @@ describe('export', () => { await exportCommand({ encrypt, key }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ encryption: { enabled: encrypt, key }, }) @@ -129,7 +135,7 @@ describe('export', () => { await exportCommand({ compress: false }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ compression: { enabled: false }, }) @@ -137,7 +143,7 @@ describe('export', () => { await expectExit(1, async () => { await exportCommand({ compress: true }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ compression: { enabled: true }, }) diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 508384672c..ff6d3c6d45 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -3,20 +3,21 @@ const utils = require('../../transfer/utils'); const mockDataTransfer = { - createRemoteStrapiDestinationProvider: jest.fn(), - createLocalStrapiSourceProvider: jest.fn(), - createTransferEngine: jest.fn().mockReturnValue({ - transfer: jest.fn().mockReturnValue(Promise.resolve({})), - }), + strapi: { + providers: { + createRemoteStrapiDestinationProvider: jest.fn(), + createLocalStrapiSourceProvider: jest.fn(), + }, + }, + engine: { + createTransferEngine: jest.fn().mockReturnValue({ + transfer: jest.fn().mockReturnValue(Promise.resolve({})), + }), + }, }; -jest.mock( - '@strapi/data-transfer', - () => { - return mockDataTransfer; - }, - { virtual: true } -); +jest.mock('@strapi/data-transfer/lib/engine', () => mockDataTransfer.engine, { virtual: true }); +jest.mock('@strapi/data-transfer/lib/strapi', () => mockDataTransfer.strapi, { virtual: true }); const expectExit = async (code, fn) => { const exit = jest.spyOn(process, 'exit').mockImplementation((number) => { @@ -37,7 +38,7 @@ jest.mock('../../transfer/utils'); const destinationUrl = 'ws://strapi.com'; -describe('transfer', () => { +describe('Transfer', () => { beforeEach(() => { jest.resetAllMocks(); }); @@ -47,7 +48,9 @@ describe('transfer', () => { await transferCommand({ from: 'local', to: destinationUrl }); }); - expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( expect.objectContaining({ url: destinationUrl, }) @@ -61,7 +64,9 @@ describe('transfer', () => { await transferCommand({ from: 'local', to: destinationUrl }); }); - expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( expect.objectContaining({ strategy: 'restore', }) @@ -72,7 +77,9 @@ describe('transfer', () => { await transferCommand({ from: 'local', to: destinationUrl }); }); - expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( expect.objectContaining({ url: destinationUrl, }) @@ -84,7 +91,9 @@ describe('transfer', () => { await transferCommand({ from: 'local', to: destinationUrl }); }); - expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( expect.objectContaining({ strategy: 'restore', }) @@ -96,7 +105,7 @@ describe('transfer', () => { await transferCommand({ from: 'local', to: destinationUrl }); }); - expect(mockDataTransfer.createLocalStrapiSourceProvider).toHaveBeenCalled(); + expect(mockDataTransfer.strapi.providers.createLocalStrapiSourceProvider).toHaveBeenCalled(); expect(utils.createStrapiInstance).toHaveBeenCalled(); }); diff --git a/packages/core/strapi/lib/commands/transfer/export.js b/packages/core/strapi/lib/commands/transfer/export.js index 7c3a017063..be5d33eb08 100644 --- a/packages/core/strapi/lib/commands/transfer/export.js +++ b/packages/core/strapi/lib/commands/transfer/export.js @@ -1,12 +1,12 @@ 'use strict'; const { - createLocalFileDestinationProvider, - createLocalStrapiSourceProvider, - createTransferEngine, - // TODO: we need to solve this issue with typescript modules - // eslint-disable-next-line import/no-unresolved, node/no-missing-require -} = require('@strapi/data-transfer'); + providers: { createLocalFileDestinationProvider }, +} = require('@strapi/data-transfer/lib/file'); +const { + providers: { createLocalStrapiSourceProvider }, +} = require('@strapi/data-transfer/lib/strapi'); +const { createTransferEngine } = require('@strapi/data-transfer/lib/engine'); const { isObject, isString, isFinite, toNumber } = require('lodash/fp'); const fs = require('fs-extra'); const chalk = require('chalk'); diff --git a/packages/core/strapi/lib/commands/transfer/import.js b/packages/core/strapi/lib/commands/transfer/import.js index 4bb3e4a63e..72fd378d79 100644 --- a/packages/core/strapi/lib/commands/transfer/import.js +++ b/packages/core/strapi/lib/commands/transfer/import.js @@ -1,15 +1,17 @@ 'use strict'; const { - createLocalFileSourceProvider, - createLocalStrapiDestinationProvider, + providers: { createLocalFileSourceProvider }, +} = require('@strapi/data-transfer/lib/file'); +const { + providers: { createLocalStrapiDestinationProvider, DEFAULT_CONFLICT_STRATEGY }, +} = require('@strapi/data-transfer/lib/strapi'); +const { createTransferEngine, DEFAULT_VERSION_STRATEGY, DEFAULT_SCHEMA_STRATEGY, - DEFAULT_CONFLICT_STRATEGY, - // TODO: we need to solve this issue with typescript modules - // eslint-disable-next-line import/no-unresolved, node/no-missing-require -} = require('@strapi/data-transfer'); +} = require('@strapi/data-transfer/lib/engine'); + const { isObject } = require('lodash/fp'); const path = require('path'); @@ -45,6 +47,7 @@ module.exports = async (opts) => { async getStrapi() { return strapiInstance; }, + autoDestroy: false, strategy: opts.conflictStrategy || DEFAULT_CONFLICT_STRATEGY, restore: { entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES }, @@ -109,7 +112,9 @@ module.exports = async (opts) => { } // Note: Telemetry can't be sent in a finish event, because it runs async after this block but we can't await it, so if process.exit is used it won't send - await strapi.telemetry.send('didDEITSProcessFinish', getTelemetryPayload()); + await strapiInstance.telemetry.send('didDEITSProcessFinish', getTelemetryPayload()); + await strapiInstance.destroy(); + process.exit(0); }; diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index e3f2db936f..b875defa86 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -1,12 +1,9 @@ 'use strict'; +const { createTransferEngine } = require('@strapi/data-transfer/lib/engine'); const { - createRemoteStrapiDestinationProvider, - createLocalStrapiSourceProvider, - createTransferEngine, - // TODO: we need to solve this issue with typescript modules - // eslint-disable-next-line import/no-unresolved, node/no-missing-require -} = require('@strapi/data-transfer'); + providers: { createRemoteStrapiDestinationProvider, createLocalStrapiSourceProvider }, +} = require('@strapi/data-transfer/lib/strapi'); const { isObject } = require('lodash/fp'); const chalk = require('chalk'); From e7e6516e06b418a0d0eff8ba6d9202649462a0ea Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 2 Jan 2023 15:40:53 +0100 Subject: [PATCH 27/44] Update yarn lock --- yarn.lock | 45 ++++++++++++++++----------------------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/yarn.lock b/yarn.lock index d97a01ab49..d95b622520 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6367,6 +6367,20 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/koa@2.13.1": + version "2.13.1" + resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.1.tgz#e29877a6b5ad3744ab1024f6ec75b8cbf6ec45db" + integrity sha512-Qbno7FWom9nNqu0yHZ6A0+RWt4mrYBhw3wpBAQ3+IuzGcLlfeYkzZrnMq5wsxulN2np8M4KKeUpTodsOsSad5Q== + dependencies: + "@types/accepts" "*" + "@types/content-disposition" "*" + "@types/cookies" "*" + "@types/http-assert" "*" + "@types/http-errors" "*" + "@types/keygrip" "*" + "@types/koa-compose" "*" + "@types/node" "*" + "@types/koa__cors@^3.0.1": version "3.3.0" resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.0.tgz#2986b320d3d7ddf05c4e2e472b25a321cb16bd3b" @@ -15520,35 +15534,6 @@ koa@2.13.4, koa@^2.13.4: type-is "^1.6.16" vary "^1.1.2" -koa@2.14.1: - version "2.14.1" - resolved "https://registry.yarnpkg.com/koa/-/koa-2.14.1.tgz#defb9589297d8eb1859936e777f3feecfc26925c" - integrity sha512-USJFyZgi2l0wDgqkfD27gL4YGno7TfUkcmOe6UOLFOVuN+J7FwnNu4Dydl4CUQzraM1lBAiGed0M9OVJoT0Kqw== - dependencies: - accepts "^1.3.5" - cache-content-type "^1.0.0" - content-disposition "~0.5.2" - content-type "^1.0.4" - cookies "~0.8.0" - debug "^4.3.2" - delegates "^1.0.0" - depd "^2.0.0" - destroy "^1.0.4" - encodeurl "^1.0.2" - escape-html "^1.0.3" - fresh "~0.5.2" - http-assert "^1.3.0" - http-errors "^1.6.3" - is-generator-function "^1.0.7" - koa-compose "^4.1.0" - koa-convert "^2.0.0" - on-finished "^2.3.0" - only "~0.0.2" - parseurl "^1.3.2" - statuses "^1.5.0" - type-is "^1.6.16" - vary "^1.1.2" - kuler@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/kuler/-/kuler-2.0.0.tgz#e2c570a3800388fb44407e851531c1d670b061b3" @@ -18317,6 +18302,8 @@ path-case@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/path-case/-/path-case-2.1.1.tgz#94b8037c372d3fe2906e465bb45e25d226e8eea5" integrity sha1-lLgDfDctP+KQbkZbtF4l0ibo7qU= + dependencies: + no-case "^2.2.0" path-dirname@^1.0.0: version "1.0.2" From 3b9dbbe4b97412f2680bdaa23b52f2f8ee52eb62 Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 2 Jan 2023 16:07:16 +0100 Subject: [PATCH 28/44] Remove useless async --- packages/core/admin/server/register.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/admin/server/register.js b/packages/core/admin/server/register.js index 932f1b7f69..d99d6326f8 100644 --- a/packages/core/admin/server/register.js +++ b/packages/core/admin/server/register.js @@ -6,7 +6,7 @@ const registerAdminPanelRoute = require('./routes/serve-admin-panel'); const adminAuthStrategy = require('./strategies/admin'); const apiTokenAuthStrategy = require('./strategies/api-token'); -module.exports = async ({ strapi }) => { +module.exports = ({ strapi }) => { const passportMiddleware = strapi.admin.services.passport.init(); strapi.server.api('admin').use(passportMiddleware); From c700027a6d10a398cab8f0cd49ad97bb99bc43c8 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 17:11:51 +0100 Subject: [PATCH 29/44] reenable lint --- packages/core/strapi/lib/commands/transfer/transfer.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index 48b42c527b..21ecea30fd 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -5,8 +5,6 @@ const { createLocalStrapiSourceProvider, createTransferEngine, createLocalStrapiDestinationProvider, - // TODO: we need to solve this issue with typescript modules - // eslint-disable-next-line import/no-unresolved, node/no-missing-require } = require('@strapi/data-transfer'); const { isObject } = require('lodash/fp'); const chalk = require('chalk'); From a0e1b8d31eea4b5524ba62b6f9fbe6ad0e45543e Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 17:35:43 +0100 Subject: [PATCH 30/44] fix tests --- .../__tests__/data-transfer/transfer.test.js | 22 +++++++------------ 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 508384672c..c026803ac2 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -31,7 +31,9 @@ const expectExit = async (code, fn) => { const transferCommand = require('../../transfer/transfer'); -const logger = jest.spyOn(console, 'error').mockImplementation(() => {}); +jest.spyOn(console, 'error').mockImplementation(() => {}); +jest.spyOn(console, 'warn').mockImplementation(() => {}); +jest.spyOn(console, 'log').mockImplementation(() => {}); jest.mock('../../transfer/utils'); @@ -44,7 +46,7 @@ describe('transfer', () => { it('uses destination url provided by user without authentication', async () => { await expectExit(1, async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await transferCommand({ from: undefined, to: destinationUrl }); }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( @@ -58,7 +60,7 @@ describe('transfer', () => { it('uses restore as the default strategy', async () => { await expectExit(1, async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await transferCommand({ from: undefined, to: destinationUrl }); }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( @@ -69,7 +71,7 @@ describe('transfer', () => { }); it('uses destination url provided by user without authentication', async () => { await expectExit(1, async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await transferCommand({ from: undefined, to: destinationUrl }); }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( @@ -81,7 +83,7 @@ describe('transfer', () => { it('uses restore as the default strategy', async () => { await expectExit(1, async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await transferCommand({ from: undefined, to: destinationUrl }); }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( @@ -93,18 +95,10 @@ describe('transfer', () => { it('uses local strapi instance when local specified', async () => { await expectExit(1, async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await transferCommand({ from: undefined, to: destinationUrl }); }); expect(mockDataTransfer.createLocalStrapiSourceProvider).toHaveBeenCalled(); expect(utils.createStrapiInstance).toHaveBeenCalled(); }); - - it('Logs an error when the source provider does not exist', async () => { - await expectExit(1, async () => { - await transferCommand({ from: 'test', to: destinationUrl }); - }); - - expect(logger).toHaveBeenCalledWith("Couldn't create providers"); - }); }); From 00f2b99705c20728125097c7cc9eb8363ed36a38 Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 2 Jan 2023 18:10:51 +0100 Subject: [PATCH 31/44] Add lib to the eslintignore file --- packages/core/data-transfer/.eslintignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/core/data-transfer/.eslintignore diff --git a/packages/core/data-transfer/.eslintignore b/packages/core/data-transfer/.eslintignore new file mode 100644 index 0000000000..a65b41774a --- /dev/null +++ b/packages/core/data-transfer/.eslintignore @@ -0,0 +1 @@ +lib From 8a3fe03bf4d0b648c37d9e622f71bafee5959b7d Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 18:12:13 +0100 Subject: [PATCH 32/44] add missing dep --- packages/core/strapi/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/strapi/package.json b/packages/core/strapi/package.json index 2b5b4a8743..d7b2b05a3b 100644 --- a/packages/core/strapi/package.json +++ b/packages/core/strapi/package.json @@ -81,6 +81,7 @@ "@koa/cors": "3.4.3", "@koa/router": "10.1.1", "@strapi/admin": "4.5.5", + "@strapi/data-transfer": "4.5.5", "@strapi/database": "4.5.5", "@strapi/generate-new": "4.5.5", "@strapi/generators": "4.5.5", From b98c873a8c745db657eab4a8d5e71d3b8955bf7a Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 2 Jan 2023 18:15:59 +0100 Subject: [PATCH 33/44] Add data-transfer dependency to strapi/strapi --- packages/core/strapi/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/strapi/package.json b/packages/core/strapi/package.json index 2b5b4a8743..e4cd6b2980 100644 --- a/packages/core/strapi/package.json +++ b/packages/core/strapi/package.json @@ -82,6 +82,7 @@ "@koa/router": "10.1.1", "@strapi/admin": "4.5.5", "@strapi/database": "4.5.5", + "@strapi/data-transfer": "4.5.5", "@strapi/generate-new": "4.5.5", "@strapi/generators": "4.5.5", "@strapi/logger": "4.5.5", From b934f3b4c92bcd1ad7b2831c99479fc296eceba1 Mon Sep 17 00:00:00 2001 From: Convly Date: Mon, 2 Jan 2023 18:21:48 +0100 Subject: [PATCH 34/44] Update the api workflow --- .github/workflows/tests.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f683c85b0a..768fbea22f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -133,6 +133,8 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Build TypeScript packages + run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=postgres --dbhost=localhost --dbport=5432 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -171,6 +173,8 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Build TypeScript packages + run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=mysql --dbhost=localhost --dbport=3306 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -208,6 +212,8 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Build TypeScript packages + run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=mysql --dbhost=localhost --dbport=3306 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -230,6 +236,8 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Build TypeScript packages + run: yarn build:t - uses: ./.github/actions/run-api-tests env: SQLITE_PKG: ${{ matrix.sqlite_pkg }} @@ -276,6 +284,8 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Build TypeScript packages + run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=postgres --dbhost=localhost --dbport=5432 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -318,6 +328,8 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Build TypeScript packages + run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=mysql --dbhost=localhost --dbport=3306 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -344,6 +356,8 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Build TypeScript packages + run: yarn build:t - uses: ./.github/actions/run-api-tests env: SQLITE_PKG: ${{ matrix.sqlite_pkg }} From fe63c1d5594c5be90b1f12e1bf61e5e20ada3001 Mon Sep 17 00:00:00 2001 From: Convly Date: Tue, 3 Jan 2023 17:36:25 +0100 Subject: [PATCH 35/44] Update data-transfer with remote protocol types --- .../providers/remote-destination/index.ts | 113 ++++++++----- .../providers/remote-destination/utils.ts | 149 ++++++++++++++---- .../src/strapi/remote/controllers/push.ts | 28 ++-- .../src/strapi/remote/handlers.ts | 146 +++++++++++++---- packages/core/data-transfer/types/index.d.ts | 2 +- packages/core/data-transfer/types/remote.d.ts | 90 ----------- .../data-transfer/types/remote/index.d.ts | 1 + .../client/{commands.ts => commands.d.ts} | 13 +- .../protocol/client/{index.ts => index.d.ts} | 2 +- .../transfer/{action.ts => action.d.ts} | 0 .../client/transfer/{index.ts => index.d.ts} | 2 +- .../client/transfer/{pull.ts => pull.d.ts} | 0 .../client/transfer/{push.ts => push.d.ts} | 15 +- .../client/transfer/{utils.ts => utils.d.ts} | 2 +- .../types/remote/protocol/index.d.ts | 2 + .../protocol/server/{error.ts => error.d.ts} | 0 .../types/remote/protocol/server/index.d.ts | 2 + .../types/remote/protocol/server/index.ts | 2 - .../remote/protocol/server/messaging.d.ts | 14 ++ .../types/remote/protocol/server/messaging.ts | 10 -- 20 files changed, 368 insertions(+), 225 deletions(-) delete mode 100644 packages/core/data-transfer/types/remote.d.ts create mode 100644 packages/core/data-transfer/types/remote/index.d.ts rename packages/core/data-transfer/types/remote/protocol/client/{commands.ts => commands.d.ts} (54%) rename packages/core/data-transfer/types/remote/protocol/client/{index.ts => index.d.ts} (72%) rename packages/core/data-transfer/types/remote/protocol/client/transfer/{action.ts => action.d.ts} (100%) rename packages/core/data-transfer/types/remote/protocol/client/transfer/{index.ts => index.d.ts} (78%) rename packages/core/data-transfer/types/remote/protocol/client/transfer/{pull.ts => pull.d.ts} (100%) rename packages/core/data-transfer/types/remote/protocol/client/transfer/{push.ts => push.d.ts} (64%) rename packages/core/data-transfer/types/remote/protocol/client/transfer/{utils.ts => utils.d.ts} (82%) create mode 100644 packages/core/data-transfer/types/remote/protocol/index.d.ts rename packages/core/data-transfer/types/remote/protocol/server/{error.ts => error.d.ts} (100%) create mode 100644 packages/core/data-transfer/types/remote/protocol/server/index.d.ts delete mode 100644 packages/core/data-transfer/types/remote/protocol/server/index.ts create mode 100644 packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts delete mode 100644 packages/core/data-transfer/types/remote/protocol/server/messaging.ts diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts index 4b3003792e..ad776a3540 100644 --- a/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts @@ -2,6 +2,8 @@ import { WebSocket } from 'ws'; import { v4 } from 'uuid'; import { Writable } from 'stream'; +import { createDispatcher } from './utils'; + import type { IDestinationProvider, IEntity, @@ -9,11 +11,10 @@ import type { IMetadata, ProviderType, IConfiguration, - TransferStage, IAsset, } from '../../../../types'; +import type { client, server } from '../../../../types/remote/protocol'; import type { ILocalStrapiDestinationProviderOptions } from '../local-destination'; -import { dispatch } from './utils'; interface ITokenAuth { type: 'token'; @@ -32,14 +33,6 @@ export interface IRemoteStrapiDestinationProviderOptions auth?: ITokenAuth | ICredentialsAuth; } -type Actions = 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; - -export const createRemoteStrapiDestinationProvider = ( - options: IRemoteStrapiDestinationProviderOptions -) => { - return new RemoteStrapiDestinationProvider(options); -}; - class RemoteStrapiDestinationProvider implements IDestinationProvider { name = 'destination::remote-strapi'; @@ -49,23 +42,55 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { ws: WebSocket | null; + dispatcher: ReturnType | null; + constructor(options: IRemoteStrapiDestinationProviderOptions) { this.options = options; this.ws = null; + this.dispatcher = null; } - async #dispatchAction(action: Actions) { - return dispatch(this.ws, { type: 'action', action }); + async initTransfer(): Promise { + const { strategy, restore } = this.options; + + // Wait for the connection to be made to the server, then init the transfer + return new Promise((resolve, reject) => { + this.ws + ?.once('open', async () => { + const query = this.dispatcher?.dispatchCommand({ + command: 'init', + params: { options: { strategy, restore }, transfer: 'push' }, + }); + + const res = (await query) as server.Payload; + + if (!res?.transferID) { + return reject(new Error('Init failed, invalid response from the server')); + } + + resolve(res.transferID); + }) + .once('error', reject); + }); } - async #dispatchTransfer(stage: TransferStage, data: T) { + async #streamStep( + step: T, + data: client.GetTransferPushStreamData + ) { + const query = this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data }); + try { - await dispatch(this.ws, { type: 'transfer', stage, data }); + await query; } catch (e) { if (e instanceof Error) { return e; } + if (typeof e === 'string') { + return new Error(e); + } + return new Error('Unexpected error'); } @@ -73,7 +98,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { } async bootstrap(): Promise { - const { url, auth, strategy, restore } = this.options; + const { url, auth } = this.options; let ws: WebSocket; @@ -95,21 +120,17 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { } this.ws = ws; + this.dispatcher = createDispatcher(this.ws); - // Wait for the connection to be made to the server, then init the transfer - await new Promise((resolve, reject) => { - ws.once('open', async () => { - await dispatch(this.ws, { type: 'init', kind: 'push', options: { strategy, restore } }); - resolve(); - }).once('error', reject); - }); + const transferID = await this.initTransfer(); - // Run the bootstrap - await this.#dispatchAction('bootstrap'); + this.dispatcher.setTransferProperties({ id: transferID, kind: 'push' }); + + await this.dispatcher.dispatchTransferAction('bootstrap'); } async close() { - await this.#dispatchAction('close'); + await this.dispatcher?.dispatchTransferAction('close'); await new Promise((resolve) => { const { ws } = this; @@ -124,22 +145,26 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { } getMetadata() { - return this.#dispatchAction('getMetadata'); + return this.dispatcher?.dispatchTransferAction('getMetadata') ?? null; } async beforeTransfer() { - await this.#dispatchAction('beforeTransfer'); + await this.dispatcher?.dispatchTransferAction('beforeTransfer'); } - getSchemas(): Promise { - return this.#dispatchAction('getSchemas'); + getSchemas(): Promise { + if (!this.dispatcher) { + return Promise.resolve(null); + } + + return this.dispatcher.dispatchTransferAction('getSchemas'); } getEntitiesStream(): Writable { return new Writable({ objectMode: true, write: async (entity: IEntity, _encoding, callback) => { - const e = await this.#dispatchTransfer('entities', entity); + const e = await this.#streamStep('entities', entity); callback(e); }, @@ -150,7 +175,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, write: async (link: ILink, _encoding, callback) => { - const e = await this.#dispatchTransfer('links', link); + const e = await this.#streamStep('links', link); callback(e); }, @@ -161,7 +186,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, write: async (configuration: IConfiguration, _encoding, callback) => { - const e = await this.#dispatchTransfer('configuration', configuration); + const e = await this.#streamStep('configuration', configuration); callback(e); }, @@ -172,29 +197,31 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, final: async (callback) => { - const e = await this.#dispatchTransfer('assets', null); + // TODO: replace this stream call by an end call + const e = await this.#streamStep('assets', null); + callback(e); }, write: async (asset: IAsset, _encoding, callback) => { const { filename, filepath, stats, stream } = asset; const assetID = v4(); - await this.#dispatchTransfer('assets', { - step: 'start', + await this.#streamStep('assets', { + action: 'start', assetID, data: { filename, filepath, stats }, }); for await (const chunk of stream) { - await this.#dispatchTransfer('assets', { - step: 'stream', + await this.#streamStep('assets', { + action: 'stream', assetID, - data: { chunk }, + data: chunk, }); } - await this.#dispatchTransfer('assets', { - step: 'end', + await this.#streamStep('assets', { + action: 'end', assetID, }); @@ -203,3 +230,9 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { }); } } + +export const createRemoteStrapiDestinationProvider = ( + options: IRemoteStrapiDestinationProviderOptions +) => { + return new RemoteStrapiDestinationProvider(options); +}; diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts index b79fc64818..0a1931cb4e 100644 --- a/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts @@ -1,34 +1,123 @@ +import { set } from 'lodash/fp'; import { v4 } from 'uuid'; -import { WebSocket } from 'ws'; +import { RawData, WebSocket } from 'ws'; -export async function dispatch( - ws: WebSocket | null, - message: T -): Promise { - if (!ws) { - throw new Error('No websocket connection found'); - } +import type { client, server } from '../../../../types/remote/protocol'; - return new Promise((resolve, reject) => { - const uuid = v4(); - const payload = JSON.stringify({ ...message, uuid }); - - ws.send(payload, (error) => { - if (error) { - reject(error); - } - }); - - ws.once('message', (raw) => { - const response: { uuid: string; data: U; error: string | null } = JSON.parse(raw.toString()); - - if (response.error) { - return reject(new Error(response.error)); - } - - if (response.uuid === uuid) { - return resolve(response.data); - } - }); - }); +interface IDispatcherState { + transfer?: { kind: client.TransferKind; id: string }; } + +interface IDispatchOptions { + attachTransfer?: boolean; +} + +type Dispatch = Omit; + +const createDispatcher = (ws: WebSocket) => { + const state: IDispatcherState = {}; + + type DispatchMessage = Dispatch; + + const dispatch = async ( + message: DispatchMessage, + options: IDispatchOptions = {} + ): Promise => { + if (!ws) { + throw new Error('No websocket connection found'); + } + + return new Promise((resolve, reject) => { + const uuid = v4(); + const payload = { ...message, uuid }; + + if (options.attachTransfer) { + Object.assign(payload, { transferID: state.transfer?.id }); + } + + const stringifiedPayload = JSON.stringify(payload); + + ws.send(stringifiedPayload, (error) => { + if (error) { + reject(error); + } + }); + + const onResponse = (raw: RawData) => { + const response: server.Message = JSON.parse(raw.toString()); + if (response.uuid === uuid) { + if (response.error) { + return reject(new Error(response.error.message)); + } + + resolve(response.data ?? null); + } else { + ws.once('message', onResponse); + } + }; + + // TODO: What happens if the server sends another message (not a response to this message) + ws.once('message', onResponse); + }); + }; + + const dispatchCommand = ( + payload: { + command: U; + } & ([client.GetCommandParams] extends [never] + ? unknown + : { params: client.GetCommandParams }) + ) => { + return dispatch({ type: 'command', ...payload } as client.CommandMessage); + }; + + const dispatchTransferAction = async (action: client.Action['action']) => { + const payload: Dispatch = { type: 'transfer', kind: 'action', action }; + + return dispatch(payload, { attachTransfer: true }) ?? Promise.resolve(null); + }; + + const dispatchTransferStep = async < + T, + A extends client.TransferPushMessage['action'] = client.TransferPushMessage['action'], + S extends client.TransferPushStep = client.TransferPushStep + >( + payload: { + step: S; + action: A; + } & (A extends 'stream' ? { data: client.GetTransferPushStreamData } : unknown) + ) => { + const message: Dispatch = { + type: 'transfer', + kind: 'step', + ...payload, + }; + + return dispatch(message, { attachTransfer: true }) ?? Promise.resolve(null); + }; + + const setTransferProperties = ( + properties: Exclude + ): void => { + state.transfer = { ...properties }; + }; + + return { + get transferID() { + return state.transfer?.id; + }, + + get transferKind() { + return state.transfer?.kind; + }, + + setTransferProperties, + + dispatch, + dispatchCommand, + dispatchTransferAction, + dispatchTransferStep, + }; +}; + +export { createDispatcher }; diff --git a/packages/core/data-transfer/src/strapi/remote/controllers/push.ts b/packages/core/data-transfer/src/strapi/remote/controllers/push.ts index caec12632e..3a2487a4d4 100644 --- a/packages/core/data-transfer/src/strapi/remote/controllers/push.ts +++ b/packages/core/data-transfer/src/strapi/remote/controllers/push.ts @@ -1,13 +1,17 @@ import { PassThrough, Writable } from 'stream-chain'; -import { IAsset, IMetadata, PushTransferMessage, PushTransferStage } from '../../../../types'; +import type { IAsset, IMetadata } from '../../../../types'; +import type { + TransferPushMessage, + TransferPushStep, +} from '../../../../types/remote/protocol/client'; import { createLocalStrapiDestinationProvider, ILocalStrapiDestinationProviderOptions, } from '../../providers'; export interface IPushController { - streams: { [stage in PushTransferStage]?: Writable }; + streams: { [stage in TransferPushStep]?: Writable }; actions: { getMetadata(): Promise; getSchemas(): Strapi.Schemas; @@ -16,8 +20,8 @@ export interface IPushController { beforeTransfer(): Promise; }; transfer: { - [key in PushTransferStage]: ( - value: T extends { stage: key; data: infer U } ? U : never + [key in TransferPushStep]: ( + value: T extends { step: key; data: infer U } ? U : never ) => Promise; }; } @@ -25,7 +29,7 @@ export interface IPushController { const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { const provider = createLocalStrapiDestinationProvider(options); - const streams: { [stage in PushTransferStage]?: Writable } = {}; + const streams: { [stage in TransferPushStep]?: Writable } = {}; const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; const writeAsync = (stream: Writable, data: T) => { @@ -91,29 +95,33 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): }, async assets(payload) { + // TODO: close the stream upong receiving an 'end' event instead if (payload === null) { streams.assets?.end(); return; } - const { step, assetID } = payload; + const { action, assetID } = payload; if (!streams.assets) { streams.assets = await provider.getAssetsStream(); } - if (step === 'start') { + if (action === 'start') { assets[assetID] = { ...payload.data, stream: new PassThrough() }; writeAsync(streams.assets, assets[assetID]); } - if (step === 'stream') { - const chunk = Buffer.from(payload.data.chunk.data); + if (action === 'stream') { + // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array } + // We need to transform it back into a Buffer instance + const rawBuffer = payload.data as unknown as { type: 'Buffer'; data: Uint8Array }; + const chunk = Buffer.from(rawBuffer.data); await writeAsync(assets[assetID].stream, chunk); } - if (step === 'end') { + if (action === 'end') { await new Promise((resolve, reject) => { const { stream } = assets[assetID]; diff --git a/packages/core/data-transfer/src/strapi/remote/handlers.ts b/packages/core/data-transfer/src/strapi/remote/handlers.ts index dff8124001..a461abd33f 100644 --- a/packages/core/data-transfer/src/strapi/remote/handlers.ts +++ b/packages/core/data-transfer/src/strapi/remote/handlers.ts @@ -2,17 +2,16 @@ import type { Context } from 'koa'; import type { ServerOptions } from 'ws'; -import { v4 } from 'uuid'; +import { randomUUID } from 'crypto'; import { WebSocket } from 'ws'; import type { IPushController } from './controllers/push'; -import { InitMessage, Message, TransferKind } from '../../../types'; import createPushController from './controllers/push'; +import type { client, server } from '../../../types/remote/protocol'; interface ITransferState { - kind?: TransferKind; - transferID?: string; + transfer?: { id: string; kind: client.TransferKind }; controller?: IPushController; } @@ -34,6 +33,9 @@ export const createTransferHandler = const state: ITransferState = {}; let uuid: string | undefined; + /** + * Format error & message to follow the remote transfer protocol + */ const callback = (e: Error | null = null, data?: T) => { return new Promise((resolve, reject) => { if (!uuid) { @@ -43,14 +45,22 @@ export const createTransferHandler = const payload = JSON.stringify({ uuid, - data: data ?? {}, - error: e, + data: data ?? null, + error: e + ? { + code: 'ERR', + message: e?.message, + } + : null, }); ws.send(payload, (error) => (error ? reject(error) : resolve())); }); }; + /** + * Wrap a function call to catch errors and answer the request with the correct format + */ const answer = async (fn: () => T) => { try { const response = await fn(); @@ -66,39 +76,110 @@ export const createTransferHandler = } }; - const teardown = () => { - delete state.kind; + const teardown = (): server.Payload => { delete state.controller; - delete state.transferID; + delete state.transfer; return { ok: true }; }; - const init = (msg: InitMessage) => { - const { kind, options: controllerOptions } = msg; - + const init = (msg: client.InitCommand): server.Payload => { if (state.controller) { throw new Error('Transfer already in progres'); } - if (kind === 'push') { + const { transfer } = msg.params; + + // Push transfer + if (transfer === 'push') { + const { options: controllerOptions } = msg.params; + state.controller = createPushController({ ...controllerOptions, autoDestroy: false, - getStrapi() { - return strapi; - }, + getStrapi: () => strapi, }); } - // Pull or others + // Pull or any other string else { - throw new Error(`${kind} transfer not implemented`); + throw new Error(`Transfer not implemented: "${transfer}"`); } - state.transferID = v4(); + state.transfer = { id: randomUUID(), kind: transfer }; - return { transferID: state.transferID }; + return { transferID: state.transfer.id }; + }; + + /** + * On command message (init, end, status, ...) + */ + const onCommand = async (msg: client.CommandMessage) => { + const { command } = msg; + + if (command === 'init') { + await answer(() => init(msg)); + } + + if (command === 'end') { + await answer(teardown); + } + + if (command === 'status') { + await callback(new Error('Command not implemented: "status"')); + } + }; + + const onTransferCommand = async (msg: client.TransferMessage) => { + const { transferID, kind } = msg; + const { controller } = state; + + // TODO: (re)move this check + // It shouldn't be possible to strart a pull transfer for now, so reaching + // this code should be impossible too, but this has been added by security + if (state.transfer?.kind === 'pull') { + return callback(new Error('Pull transfer not implemented')); + } + + if (!controller) { + return callback(new Error("The transfer hasn't been initialized")); + } + + if (!transferID) { + return callback(new Error('Missing transfer ID')); + } + + // Action + if (kind === 'action') { + const { action } = msg; + + if (!(action in controller.actions)) { + return callback(new Error(`Invalid action provided: "${action}"`)); + } + + await answer(() => controller.actions[action as keyof typeof controller.actions]()); + } + + // Transfer + else if (kind === 'step') { + // We can only have push transfer message for the moment + const message = msg as client.TransferPushMessage; + + // TODO: lock transfer process + if (message.action === 'start') { + // console.log('Starting transfer for ', message.step); + } + + // Stream step + else if (message.action === 'stream') { + await answer(() => controller.transfer[message.step]?.(message.data as never)); + } + + // TODO: unlock transfer process + else if (message.action === 'end') { + // console.log('Ending transfer for ', message.step); + } + } }; ws.on('close', () => { @@ -111,7 +192,7 @@ export const createTransferHandler = }); ws.on('message', async (raw) => { - const msg: Message = JSON.parse(raw.toString()); + const msg: client.Message = JSON.parse(raw.toString()); if (!msg.uuid) { throw new Error('Missing uuid in message'); @@ -119,24 +200,19 @@ export const createTransferHandler = uuid = msg.uuid; - if (msg.type === 'init') { - await answer(() => init(msg)); + // Regular command message (init, end, status) + if (msg.type === 'command') { + await onCommand(msg); } - if (msg.type === 'teardown') { - await answer(teardown); + // Transfer message (the transfer must be initialized first) + else if (msg.type === 'transfer') { + await onTransferCommand(msg); } - if (msg.type === 'action') { - await answer(() => state.controller?.actions[msg.action]?.()); - } - - if (msg.type === 'transfer') { - await answer(() => { - const { stage, data } = msg; - - return state.controller?.transfer[stage](data as never); - }); + // Invalid messages + else { + await callback(new Error('Bad request')); } }); }); diff --git a/packages/core/data-transfer/types/index.d.ts b/packages/core/data-transfer/types/index.d.ts index 51f3087de4..8a8de1bcf5 100644 --- a/packages/core/data-transfer/types/index.d.ts +++ b/packages/core/data-transfer/types/index.d.ts @@ -3,4 +3,4 @@ export * from './providers'; export * from './transfer-engine'; export * from './utils'; export * from './encryption'; -export * from './remote'; +export * as remote from './remote'; diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts deleted file mode 100644 index c46db32bd1..0000000000 --- a/packages/core/data-transfer/types/remote.d.ts +++ /dev/null @@ -1,90 +0,0 @@ -import type { ILocalStrapiDestinationProviderOptions } from '../src/strapi/providers'; -import type { IAsset, IConfiguration, IEntity, ILink } from './common-entities'; - -/** - * Utils - */ - -type EmptyObject = Record; - -/** - * Messages - */ - -export type Message = { uuid: string | null | undefined } & ( - | InitMessage - | ActionMessage - | PushTransferMessage - | TeardownMessage -); - -export type MessageType = Message['type']; -export type TransferKind = InitMessage['kind']; -export type PushTransferStage = PushTransferMessage['stage']; - -/** - * Init - */ - -// init should return a transfer ID used in the teardown -export type InitMessage = { type: 'init' } & ( - | { kind: 'pull'; options: EmptyObject } - | { kind: 'push'; options: Pick } -); - -/** - * Action - */ - -export type ActionMessage = { type: 'action' } & ( - | { action: 'getMetadata'; options: EmptyObject } - | { action: 'getSchemas'; options: EmptyObject } - | { action: 'bootstrap'; options: EmptyObject } - | { action: 'close'; options: EmptyObject } - | { action: 'beforeTransfer'; options: EmptyObject } -); - -/** - * Transfer - */ - -export type PushTransferMessage = { - type: 'transfer'; -} & ( - | PushEntitiesTransferMessage - | PushLinksTransferMessage - | PushConfigurationTransferMessage - | PushAssetTransferMessage -); - -export type PushEntitiesTransferMessage = { - stage: 'entities'; - data: IEntity | null; -}; - -export type PushLinksTransferMessage = { stage: 'links'; data: ILink | null }; - -export type PushConfigurationTransferMessage = { - stage: 'configuration'; - data: IConfiguration | null; -}; - -export type PushAssetTransferMessage = { - stage: 'assets'; - data: - | ({ assetID: string } & ( - | { step: 'start'; data: Omit } - | { step: 'stream'; data: { chunk: { type: 'Buffer'; data: number[] } } } - | { step: 'end'; data: EmptyObject } - )) - | null; -}; - -/** - * Teardown - */ - -export type TeardownMessage = { - type: 'teardown'; - transferID: string; -}; diff --git a/packages/core/data-transfer/types/remote/index.d.ts b/packages/core/data-transfer/types/remote/index.d.ts new file mode 100644 index 0000000000..24b40f78af --- /dev/null +++ b/packages/core/data-transfer/types/remote/index.d.ts @@ -0,0 +1 @@ +export * as protocol from './protocol'; diff --git a/packages/core/data-transfer/types/remote/protocol/client/commands.ts b/packages/core/data-transfer/types/remote/protocol/client/commands.d.ts similarity index 54% rename from packages/core/data-transfer/types/remote/protocol/client/commands.ts rename to packages/core/data-transfer/types/remote/protocol/client/commands.d.ts index 3cdc3b86f0..af0153dc4d 100644 --- a/packages/core/data-transfer/types/remote/protocol/client/commands.ts +++ b/packages/core/data-transfer/types/remote/protocol/client/commands.d.ts @@ -1,7 +1,15 @@ -import { ILocalStrapiDestinationProviderOptions } from '../../../../lib'; +import type { ILocalStrapiDestinationProviderOptions } from '../../../../src/strapi/providers'; export type CommandMessage = { type: 'command' } & (InitCommand | EndCommand | StatusCommand); +export type Command = CommandMessage['command']; + +export type GetCommandParams = { + [key in Command]: { command: key } & CommandMessage; +}[T] extends { params: infer U } + ? U + : never; + export type InitCommand = CreateCommand< 'init', | { @@ -10,8 +18,9 @@ export type InitCommand = CreateCommand< } | { transfer: 'pull' } >; +export type TransferKind = InitCommand['params']['transfer']; -export type EndCommand = CreateCommand<'end', { uid: string }>; +export type EndCommand = CreateCommand<'end', { transferID: string }>; export type StatusCommand = CreateCommand<'status'>; diff --git a/packages/core/data-transfer/types/remote/protocol/client/index.ts b/packages/core/data-transfer/types/remote/protocol/client/index.d.ts similarity index 72% rename from packages/core/data-transfer/types/remote/protocol/client/index.ts rename to packages/core/data-transfer/types/remote/protocol/client/index.d.ts index fd6ef79544..5701cd242e 100644 --- a/packages/core/data-transfer/types/remote/protocol/client/index.ts +++ b/packages/core/data-transfer/types/remote/protocol/client/index.d.ts @@ -4,5 +4,5 @@ import type { TransferMessage } from './transfer'; export * from './commands'; export * from './transfer'; -export type Message = CommandMessage | TransferMessage; +export type Message = { uuid: string } & (CommandMessage | TransferMessage); export type MessageType = Message['type']; diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/action.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/action.d.ts similarity index 100% rename from packages/core/data-transfer/types/remote/protocol/client/transfer/action.ts rename to packages/core/data-transfer/types/remote/protocol/client/transfer/action.d.ts diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/index.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/index.d.ts similarity index 78% rename from packages/core/data-transfer/types/remote/protocol/client/transfer/index.ts rename to packages/core/data-transfer/types/remote/protocol/client/transfer/index.d.ts index 387f3a596e..037b3bc61f 100644 --- a/packages/core/data-transfer/types/remote/protocol/client/transfer/index.ts +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/index.d.ts @@ -6,7 +6,7 @@ export * from './action'; export * from './pull'; export * from './push'; -export type TransferMessage = { type: 'transfer' } & ( +export type TransferMessage = { type: 'transfer'; transferID: string } & ( | Action | TransferPushMessage | TransferPullMessage diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.d.ts similarity index 100% rename from packages/core/data-transfer/types/remote/protocol/client/transfer/pull.ts rename to packages/core/data-transfer/types/remote/protocol/client/transfer/pull.d.ts diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/push.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/push.d.ts similarity index 64% rename from packages/core/data-transfer/types/remote/protocol/client/transfer/push.ts rename to packages/core/data-transfer/types/remote/protocol/client/transfer/push.d.ts index 0703f46b4a..cc5fc9b760 100644 --- a/packages/core/data-transfer/types/remote/protocol/client/transfer/push.ts +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/push.d.ts @@ -6,15 +6,26 @@ export type TransferPushMessage = CreateTransferMessage< | TransferStepCommands<'entities', IEntity> | TransferStepCommands<'links', ILink> | TransferStepCommands<'configuration', IConfiguration> - | TransferStepCommands<'assets', TransferAssetFlow> + | TransferStepCommands<'assets', TransferAssetFlow | null> >; +export type GetTransferPushStreamData = { + [key in TransferPushStep]: { + action: 'stream'; + step: key; + } & TransferPushMessage; +}[T] extends { data: infer U } + ? U + : never; + +export type TransferPushStep = TransferPushMessage['step']; + type TransferStepCommands = { step: T } & TransferStepFlow; type TransferStepFlow = { action: 'start' } | { action: 'stream'; data: U } | { action: 'end' }; type TransferAssetFlow = { assetID: string } & ( | { action: 'start'; data: Omit } - | { action: 'stream'; data: Uint8Array } + | { action: 'stream'; data: Buffer } | { action: 'end' } ); diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.d.ts similarity index 82% rename from packages/core/data-transfer/types/remote/protocol/client/transfer/utils.ts rename to packages/core/data-transfer/types/remote/protocol/client/transfer/utils.d.ts index be51e5ef5b..b9d838c95e 100644 --- a/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.ts +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.d.ts @@ -1,5 +1,5 @@ export type CreateTransferMessage = { type: 'transfer'; kind: T; - id: string; + transferID: string; } & U; diff --git a/packages/core/data-transfer/types/remote/protocol/index.d.ts b/packages/core/data-transfer/types/remote/protocol/index.d.ts new file mode 100644 index 0000000000..4cbe5b42a3 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/index.d.ts @@ -0,0 +1,2 @@ +export * as client from './client'; +export * as server from './server'; diff --git a/packages/core/data-transfer/types/remote/protocol/server/error.ts b/packages/core/data-transfer/types/remote/protocol/server/error.d.ts similarity index 100% rename from packages/core/data-transfer/types/remote/protocol/server/error.ts rename to packages/core/data-transfer/types/remote/protocol/server/error.d.ts diff --git a/packages/core/data-transfer/types/remote/protocol/server/index.d.ts b/packages/core/data-transfer/types/remote/protocol/server/index.d.ts new file mode 100644 index 0000000000..cbe3473c90 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/index.d.ts @@ -0,0 +1,2 @@ +export * from './messaging'; +export * as error from './error'; diff --git a/packages/core/data-transfer/types/remote/protocol/server/index.ts b/packages/core/data-transfer/types/remote/protocol/server/index.ts deleted file mode 100644 index 4e327b46e4..0000000000 --- a/packages/core/data-transfer/types/remote/protocol/server/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * as messaging from './messaging'; -export * as error from './error'; diff --git a/packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts b/packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts new file mode 100644 index 0000000000..1c0d5cd71d --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts @@ -0,0 +1,14 @@ +import type { ServerError } from './error'; + +export type Message = { + uuid?: string; + data?: T | null; + error?: ServerError | null; +}; + +// Successful +export type OKMessage = Message<{ ok: true }>; +export type InitMessage = Message<{ transferID: string }>; +export type EndMessage = OKMessage; + +export type Payload = T['data']; diff --git a/packages/core/data-transfer/types/remote/protocol/server/messaging.ts b/packages/core/data-transfer/types/remote/protocol/server/messaging.ts deleted file mode 100644 index 5d12bbf065..0000000000 --- a/packages/core/data-transfer/types/remote/protocol/server/messaging.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { ServerError } from './error'; - -export type Message = { - id?: string; - data?: T | null; - error?: ServerError | null; -}; - -// Successful -export type OK = Message<{ ok: true }>; From 85490de3b9eb82f26eb0480582648adb32d23d4d Mon Sep 17 00:00:00 2001 From: Convly Date: Tue, 3 Jan 2023 17:39:35 +0100 Subject: [PATCH 36/44] Remove wrong build:ts config from api tests --- .github/workflows/tests.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 768fbea22f..f683c85b0a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -133,8 +133,6 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages - run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=postgres --dbhost=localhost --dbport=5432 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -173,8 +171,6 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages - run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=mysql --dbhost=localhost --dbport=3306 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -212,8 +208,6 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages - run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=mysql --dbhost=localhost --dbport=3306 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -236,8 +230,6 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages - run: yarn build:t - uses: ./.github/actions/run-api-tests env: SQLITE_PKG: ${{ matrix.sqlite_pkg }} @@ -284,8 +276,6 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages - run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=postgres --dbhost=localhost --dbport=5432 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -328,8 +318,6 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages - run: yarn build:t - uses: ./.github/actions/run-api-tests with: dbOptions: '--dbclient=mysql --dbhost=localhost --dbport=3306 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' @@ -356,8 +344,6 @@ jobs: path: '**/node_modules' key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages - run: yarn build:t - uses: ./.github/actions/run-api-tests env: SQLITE_PKG: ${{ matrix.sqlite_pkg }} From 582a5f244a9064c263e9a967ce5e4b21fd52bfe4 Mon Sep 17 00:00:00 2001 From: Convly Date: Tue, 3 Jan 2023 17:45:40 +0100 Subject: [PATCH 37/44] Move the transfer url to a constant --- packages/core/data-transfer/src/strapi/remote/constants.ts | 1 + packages/core/data-transfer/src/strapi/remote/index.ts | 1 + packages/core/data-transfer/src/strapi/remote/routes.ts | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 packages/core/data-transfer/src/strapi/remote/constants.ts diff --git a/packages/core/data-transfer/src/strapi/remote/constants.ts b/packages/core/data-transfer/src/strapi/remote/constants.ts new file mode 100644 index 0000000000..6f913eeba6 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/constants.ts @@ -0,0 +1 @@ +export const TRANSFER_URL = '/transfer'; diff --git a/packages/core/data-transfer/src/strapi/remote/index.ts b/packages/core/data-transfer/src/strapi/remote/index.ts index af2810cd4c..719e07906a 100644 --- a/packages/core/data-transfer/src/strapi/remote/index.ts +++ b/packages/core/data-transfer/src/strapi/remote/index.ts @@ -1,2 +1,3 @@ export * as controllers from './controllers'; export * as routes from './routes'; +export * as constants from './constants'; diff --git a/packages/core/data-transfer/src/strapi/remote/routes.ts b/packages/core/data-transfer/src/strapi/remote/routes.ts index 34fc5585e7..746bed09bb 100644 --- a/packages/core/data-transfer/src/strapi/remote/routes.ts +++ b/packages/core/data-transfer/src/strapi/remote/routes.ts @@ -1,6 +1,7 @@ // eslint-disable-next-line node/no-extraneous-import import type { Context } from 'koa'; +import { TRANSFER_URL } from './constants'; import { createTransferHandler } from './handlers'; // Extend Strapi interface type to access the admin routes' API @@ -28,7 +29,7 @@ declare module '@strapi/strapi' { export const registerAdminTransferRoute = (strapi: Strapi.Strapi) => { strapi.admin.routes.push({ method: 'GET', - path: '/transfer', + path: TRANSFER_URL, handler: createTransferHandler(), config: { auth: false }, }); From c0ee93f71c6410b10a62419cb03e20c6afe9ac58 Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 4 Jan 2023 11:08:08 +0100 Subject: [PATCH 38/44] Update eslint config to ignore the data-transfer lib dir --- .eslintignore | 1 + packages/core/data-transfer/.eslintignore | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 packages/core/data-transfer/.eslintignore diff --git a/.eslintignore b/.eslintignore index b9cf1235d6..9662e15d5d 100644 --- a/.eslintignore +++ b/.eslintignore @@ -12,6 +12,7 @@ packages/core/helper-plugin/build/** packages/core/helper-plugin/lib/src/old/components/** packages/core/helper-plugin/lib/src/testUtils/** packages/core/helper-plugin/lib/src/utils/** +packages/core/data-transfer/lib .eslintrc.js .eslintrc.front.js .eslintrc.back.js diff --git a/packages/core/data-transfer/.eslintignore b/packages/core/data-transfer/.eslintignore deleted file mode 100644 index a65b41774a..0000000000 --- a/packages/core/data-transfer/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -lib From 08b1ae274318bee40181d5e59ffaa7ed3d413dfa Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 4 Jan 2023 11:08:08 +0100 Subject: [PATCH 39/44] Update eslint config to ignore the data-transfer lib dir --- .eslintignore | 1 + packages/core/data-transfer/.eslintignore | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 packages/core/data-transfer/.eslintignore diff --git a/.eslintignore b/.eslintignore index b9cf1235d6..9662e15d5d 100644 --- a/.eslintignore +++ b/.eslintignore @@ -12,6 +12,7 @@ packages/core/helper-plugin/build/** packages/core/helper-plugin/lib/src/old/components/** packages/core/helper-plugin/lib/src/testUtils/** packages/core/helper-plugin/lib/src/utils/** +packages/core/data-transfer/lib .eslintrc.js .eslintrc.front.js .eslintrc.back.js diff --git a/packages/core/data-transfer/.eslintignore b/packages/core/data-transfer/.eslintignore deleted file mode 100644 index a65b41774a..0000000000 --- a/packages/core/data-transfer/.eslintignore +++ /dev/null @@ -1 +0,0 @@ -lib From 3347f21f51b1283609423abf07ec0a9199090925 Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 4 Jan 2023 11:39:26 +0100 Subject: [PATCH 40/44] Update PR comments --- .../src/strapi/providers/remote-destination/index.ts | 4 +--- packages/core/data-transfer/types/index.d.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts index ad776a3540..7c88c84b96 100644 --- a/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts @@ -78,10 +78,8 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { step: T, data: client.GetTransferPushStreamData ) { - const query = this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data }); - try { - await query; + await this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data }); } catch (e) { if (e instanceof Error) { return e; diff --git a/packages/core/data-transfer/types/index.d.ts b/packages/core/data-transfer/types/index.d.ts index 8a8de1bcf5..51f3087de4 100644 --- a/packages/core/data-transfer/types/index.d.ts +++ b/packages/core/data-transfer/types/index.d.ts @@ -3,4 +3,4 @@ export * from './providers'; export * from './transfer-engine'; export * from './utils'; export * from './encryption'; -export * as remote from './remote'; +export * from './remote'; From cb42a4fda583abc7c9ce07c44ae053b672fe7023 Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 4 Jan 2023 11:48:08 +0100 Subject: [PATCH 41/44] Fix tests --- .../remote-destination/__tests__/index.test.ts | 12 +++++++++--- .../remote-destination/__tests__/utils.test.ts | 10 ++++++---- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts index 8a7a744af4..24dbce0012 100644 --- a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts @@ -1,14 +1,15 @@ +import { WebSocket } from 'ws'; import type { IRemoteStrapiDestinationProviderOptions } from '..'; import { createRemoteStrapiDestinationProvider } from '..'; const defaultOptions: IRemoteStrapiDestinationProviderOptions = { strategy: 'restore', - url: 'ws://test.com/admin/transfer', + url: '', }; jest.mock('../utils', () => ({ - dispatch: jest.fn(), + createDispatcher: jest.fn(), })); jest.mock('ws', () => ({ @@ -40,9 +41,14 @@ describe('Remote Strapi Destination', () => { test('Should have a defined websocket connection if bootstrap has been called', async () => { const provider = createRemoteStrapiDestinationProvider(defaultOptions); - await provider.bootstrap(); + try { + await provider.bootstrap(); + } catch { + // ignore ws connection error + } expect(provider.ws).not.toBeNull(); + expect(provider.ws?.readyState).toBe(WebSocket.CLOSED); }); }); }); diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts index 1cd4148b7a..8c907c756b 100644 --- a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts @@ -1,5 +1,6 @@ import { WebSocket } from 'ws'; -import { dispatch } from '../utils'; +import { CommandMessage } from '../../../../../types/remote/protocol/client'; +import { createDispatcher } from '../utils'; jest.mock('ws', () => ({ WebSocket: jest.fn().mockImplementation(() => { @@ -18,11 +19,12 @@ afterEach(() => { describe('Remote Strapi Destination Utils', () => { test('Dispatch method sends payload', () => { const ws = new WebSocket('ws://test/admin/transfer'); - const message = { - test: 'hello', + const message: CommandMessage = { + type: 'command', + command: 'status', }; - dispatch(ws, message); + createDispatcher(ws).dispatch(message); expect.extend({ toContain(receivedString, expected) { From 4ed00c9eb87f1432f7a0feeaa653baa4ec9118a3 Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 4 Jan 2023 14:03:22 +0100 Subject: [PATCH 42/44] Simplify utils usage --- .../core/data-transfer/src/file/providers/source/index.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/core/data-transfer/src/file/providers/source/index.ts b/packages/core/data-transfer/src/file/providers/source/index.ts index b020769e32..5b7c3e4f93 100644 --- a/packages/core/data-transfer/src/file/providers/source/index.ts +++ b/packages/core/data-transfer/src/file/providers/source/index.ts @@ -10,7 +10,8 @@ import { pipeline, PassThrough } from 'stream'; import { parser } from 'stream-json/jsonl/Parser'; import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../../types'; -import * as utils from '../../../utils'; +import { createDecryptionCipher } from '../../../utils/encryption'; +import { collect } from '../../../utils/stream'; type StreamItemArray = Parameters[0]; @@ -82,7 +83,7 @@ class LocalFileSourceProvider implements ISourceProvider { } async getSchemas() { - const schemas = await utils.stream.collect(this.streamSchemas()); + const schemas = await collect(this.streamSchemas()); return keyBy('uid', schemas); } @@ -151,7 +152,7 @@ class LocalFileSourceProvider implements ISourceProvider { } if (encryption.enabled && encryption.key) { - streams.push(utils.encryption.createDecryptionCipher(encryption.key)); + streams.push(createDecryptionCipher(encryption.key)); } if (compression.enabled) { From 4191788aaab4b4362fe912aaa2af2953b4f91308 Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 4 Jan 2023 14:13:37 +0100 Subject: [PATCH 43/44] Remove eslint ignore annotation & add back koa to dev dependencies --- packages/core/data-transfer/package.json | 3 ++- packages/core/data-transfer/src/strapi/remote/handlers.ts | 1 - yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index d2a75d6303..0cef69b748 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -63,7 +63,8 @@ "@types/tar": "6.1.3", "@types/tar-stream": "2.2.2", "@types/uuid": "9.0.0", - "@types/koa": "2.13.1", + "koa": "2.13.4", + "@types/koa": "2.13.4", "rimraf": "3.0.2", "typescript": "4.6.2" }, diff --git a/packages/core/data-transfer/src/strapi/remote/handlers.ts b/packages/core/data-transfer/src/strapi/remote/handlers.ts index a461abd33f..f6c9315b21 100644 --- a/packages/core/data-transfer/src/strapi/remote/handlers.ts +++ b/packages/core/data-transfer/src/strapi/remote/handlers.ts @@ -1,4 +1,3 @@ -// eslint-disable-next-line node/no-extraneous-import import type { Context } from 'koa'; import type { ServerOptions } from 'ws'; diff --git a/yarn.lock b/yarn.lock index 8c3cd89b20..a424b6d1b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6300,10 +6300,10 @@ "@types/koa-compose" "*" "@types/node" "*" -"@types/koa@2.13.1": - version "2.13.1" - resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.1.tgz#e29877a6b5ad3744ab1024f6ec75b8cbf6ec45db" - integrity sha512-Qbno7FWom9nNqu0yHZ6A0+RWt4mrYBhw3wpBAQ3+IuzGcLlfeYkzZrnMq5wsxulN2np8M4KKeUpTodsOsSad5Q== +"@types/koa@2.13.4": + version "2.13.4" + resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b" + integrity sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw== dependencies: "@types/accepts" "*" "@types/content-disposition" "*" From 0282635a0ef9a153e273bce35fdb8923cd300c9a Mon Sep 17 00:00:00 2001 From: Convly Date: Wed, 4 Jan 2023 15:03:29 +0100 Subject: [PATCH 44/44] Revert auto format changes --- CODE_OF_CONDUCT.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 4748c9d57c..46b43452cf 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -72,12 +72,12 @@ Community leaders will follow these Community Impact Guidelines in determining t This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at [https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. -Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][mozilla coc]. +Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder][Mozilla CoC]. -For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][faq]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. +For answers to common questions about this code of conduct, see the FAQ at [https://www.contributor-covenant.org/faq][FAQ]. Translations are available at [https://www.contributor-covenant.org/translations][translations]. [homepage]: https://www.contributor-covenant.org [v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html -[mozilla coc]: https://github.com/mozilla/diversity -[faq]: https://www.contributor-covenant.org/faq +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq [translations]: https://www.contributor-covenant.org/translations