From 4190687975f0f045af7f6b4f014d4b23fe44e45d Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Tue, 27 Dec 2022 09:55:26 +0100 Subject: [PATCH] Add tests and refactor --- .../core/data-transfer/lib/engine/index.ts | 35 +++---- .../__tests__/index.test.ts | 48 +++++++++ .../__tests__/utils.test.ts | 43 ++++++++ .../index.ts | 53 +++------- .../utils.ts | 34 +++++++ packages/core/data-transfer/lib/register.ts | 54 ++++------ .../{ => data-transfer}/export.test.js | 6 +- .../__tests__/data-transfer/transfer.test.js | 98 +++++++++++++++++++ .../strapi/lib/commands/transfer/transfer.js | 10 +- 9 files changed, 274 insertions(+), 107 deletions(-) create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts rename packages/core/strapi/lib/commands/__tests__/{ => data-transfer}/export.test.js (94%) create mode 100644 packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index 779f2d3fd7..489ee01c9b 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -338,28 +338,23 @@ class TransferEngine< async transfer(): Promise> { try { await this.bootstrap(); - await this.init(); - - const isValidTransfer = await this.integrityCheck(); - - if (!isValidTransfer) { - // TODO: provide the log from the integrity check - throw new Error( - `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` - ); - } - - await this.beforeTransfer(); - + // await this.init(); + // const isValidTransfer = await this.integrityCheck(); + // if (!isValidTransfer) { + // // TODO: provide the log from the integrity check + // throw new Error( + // `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` + // ); + // } + // await this.beforeTransfer(); // Run the transfer stages - await this.transferSchemas(); - await this.transferEntities(); - await this.transferAssets(); - await this.transferLinks(); - await this.transferConfiguration(); - + // await this.transferSchemas(); + // await this.transferEntities(); + // await this.transferAssets(); + // await this.transferLinks(); + // await this.transferConfiguration(); // Gracefully close the providers - await this.close(); + // await this.close(); } catch (e: unknown) { // Rollback the destination provider if an exception is thrown during the transfer // Note: This will be configurable in the future diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts new file mode 100644 index 0000000000..8a7a744af4 --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts @@ -0,0 +1,48 @@ +import type { IRemoteStrapiDestinationProviderOptions } from '..'; + +import { createRemoteStrapiDestinationProvider } from '..'; + +const defaultOptions: IRemoteStrapiDestinationProviderOptions = { + strategy: 'restore', + url: 'ws://test.com/admin/transfer', +}; + +jest.mock('../utils', () => ({ + dispatch: jest.fn(), +})); + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn((type, callback) => { + callback(); + return { + once: jest.fn((t, c) => c), + }; + }), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination', () => { + describe('Bootstrap', () => { + test('Should not have a defined websocket connection if bootstrap has not been called', () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + + expect(provider.ws).toBeNull(); + }); + + test('Should have a defined websocket connection if bootstrap has been called', async () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + await provider.bootstrap(); + + expect(provider.ws).not.toBeNull(); + }); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts new file mode 100644 index 0000000000..1cd4148b7a --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts @@ -0,0 +1,43 @@ +import { WebSocket } from 'ws'; +import { dispatch } from '../utils'; + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn(), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination Utils', () => { + test('Dispatch method sends payload', () => { + const ws = new WebSocket('ws://test/admin/transfer'); + const message = { + test: 'hello', + }; + + dispatch(ws, message); + + expect.extend({ + toContain(receivedString, expected) { + const jsonReceived = JSON.parse(receivedString); + const pass = Object.keys(expected).every((key) => jsonReceived[key] === expected[key]); + + return { + message: () => + `Expected ${jsonReceived} ${!pass && 'not'} to contain properties ${expected}`, + pass, + }; + }, + }); + + // @ts-ignore + expect(ws.send).toHaveBeenCalledWith(expect.toContain(message), expect.anything()); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 1d9661e55f..03fcefa284 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -13,6 +13,7 @@ import type { IAsset, } from '../../../types'; import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; +import { dispatch } from './utils'; interface ITokenAuth { type: 'token'; @@ -25,7 +26,7 @@ interface ICredentialsAuth { password: string; } -interface IRemoteStrapiDestinationProvider +export interface IRemoteStrapiDestinationProviderOptions extends Pick { url: string; auth?: ITokenAuth | ICredentialsAuth; @@ -34,7 +35,7 @@ interface IRemoteStrapiDestinationProvider type Actions = 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; export const createRemoteStrapiDestinationProvider = ( - options: IRemoteStrapiDestinationProvider + options: IRemoteStrapiDestinationProviderOptions ) => { return new RemoteStrapiDestinationProvider(options); }; @@ -44,61 +45,28 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { type: ProviderType = 'destination'; - options: IRemoteStrapiDestinationProvider; + options: IRemoteStrapiDestinationProviderOptions; ws: WebSocket | null; - constructor(options: IRemoteStrapiDestinationProvider) { + constructor(options: IRemoteStrapiDestinationProviderOptions) { this.options = options; this.ws = null; } - async #dispatch(message: T): Promise { - const { ws } = this; - - if (!ws) { - throw new Error('No ws connection found'); - } - - return new Promise((resolve, reject) => { - const uuid = v4(); - const payload = JSON.stringify({ ...message, uuid }); - - ws.send(payload, (error) => { - if (error) { - reject(error); - } - }); - - ws.once('message', (raw) => { - const response: { uuid: string; data: U; error: string | null } = JSON.parse( - raw.toString() - ); - - if (response.error) { - return reject(new Error(response.error)); - } - - if (response.uuid === uuid) { - return resolve(response.data); - } - }); - }); - } - async #dispatchAction(action: Actions) { - return this.#dispatch({ type: 'action', action }); + return dispatch(this.ws, { type: 'action', action }); } async #dispatchTransfer(stage: TransferStage, data: T) { try { - await this.#dispatch({ type: 'transfer', stage, data }); + await dispatch(this.ws, { type: 'transfer', stage, data }); } catch (e) { if (e instanceof Error) { return e; } - return new Error('Unexected error'); + return new Error('Unexpected error'); } return null; @@ -131,7 +99,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { // Wait for the connection to be made to the server, then init the transfer await new Promise((resolve, reject) => { ws.once('open', async () => { - await this.#dispatch({ type: 'init', kind: 'push', data: { strategy, restore } }); + await dispatch(this.ws, { type: 'init', kind: 'push', data: { strategy, restore } }); resolve(); }).once('error', reject); }); @@ -204,6 +172,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, final: async (callback) => { + console.log('FINAL'); const e = await this.#dispatchTransfer('assets', null); callback(e); }, @@ -217,6 +186,8 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { data: { filename, filepath, stats }, }); + console.log('is writing'); + for await (const chunk of stream) { await this.#dispatchTransfer('assets', { step: 'stream', diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts new file mode 100644 index 0000000000..b79fc64818 --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts @@ -0,0 +1,34 @@ +import { v4 } from 'uuid'; +import { WebSocket } from 'ws'; + +export async function dispatch( + ws: WebSocket | null, + message: T +): Promise { + if (!ws) { + throw new Error('No websocket connection found'); + } + + return new Promise((resolve, reject) => { + const uuid = v4(); + const payload = JSON.stringify({ ...message, uuid }); + + ws.send(payload, (error) => { + if (error) { + reject(error); + } + }); + + ws.once('message', (raw) => { + const response: { uuid: string; data: U; error: string | null } = JSON.parse(raw.toString()); + + if (response.error) { + return reject(new Error(response.error)); + } + + if (response.uuid === uuid) { + return resolve(response.data); + } + }); + }); +} diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4f358ab65f..4a6e75015a 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -5,12 +5,9 @@ import { Writable, PassThrough } from 'stream'; import { v4 } from 'uuid'; import { IAsset, - IConfiguration, Message, - ILink, IMetadata, PushTransferMessage, - PushEntitiesTransferMessage, TransferKind, InitMessage, PushTransferStage, @@ -38,8 +35,8 @@ interface IPushController { beforeTransfer(): Promise; }; transfer: { - [key in PushTransferStage]: ( - value: T extends { stage: P; data: infer U } ? U : never + [key in PushTransferStage]: ( + value: T extends { stage: key; data: infer U } ? U : never ) => Promise; }; } @@ -66,7 +63,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): streams, actions: { - async getSchemas() { + async getSchemas(): Promise { return provider.getSchemas(); }, @@ -112,35 +109,35 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): await writeAsync(streams.configuration!, config); }, - async assets(asset: any) { - if (asset === null) { + async assets(payload) { + console.log('llega'); + if (payload === null) { streams.assets?.end(); return; } - const { step, assetID } = asset; + const { step, assetID } = payload; if (!streams.assets) { streams.assets = await provider.getAssetsStream(); } - // on init, we create a passthrough stream for the asset chunks - // send to the assets destination stream the metadata for the current asset - // + the stream that we just created for the asset if (step === 'start') { - assets[assetID] = { ...asset.data, stream: new PassThrough() }; - writeAsync(streams.assets!, assets[assetID]); + assets[assetID] = { ...payload.data, stream: new PassThrough() }; + writeAsync(streams.assets, assets[assetID]); } - // propagate the chunk if (step === 'stream') { - await writeAsync(assets[assetID].stream, Buffer.from(asset.data.chunk)); + const chunk = Buffer.from(payload.data.chunk.data); + + await writeAsync(assets[assetID].stream, chunk); } - // on end, we indicate that all the chunks have been sent if (step === 'end') { await new Promise((resolve, reject) => { - assets[assetID].stream + const { stream } = assets[assetID]; + + stream .on('close', () => { delete assets[assetID]; resolve(); @@ -271,11 +268,9 @@ const createTransferController = if (msg.type === 'transfer') { await answer(() => { - const fn = state.controller?.transfer[msg.stage]; + const { stage, data } = msg; - type Msg = typeof msg; - - fn?.(msg.data); + return state.controller?.transfer[stage](data as never); }); } }); @@ -299,18 +294,3 @@ const register = (strapi: any) => { }; export default register; - -/** - * entities:start - * entities:transfer - * entities:end - * - * - * assets:start - * - * assets:transfer:start - * assets:transfer:stream - * assets:transfer:end - * - * assets:end - */ diff --git a/packages/core/strapi/lib/commands/__tests__/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js similarity index 94% rename from packages/core/strapi/lib/commands/__tests__/export.test.js rename to packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index bf30ebeeda..5d81b7aa58 100644 --- a/packages/core/strapi/lib/commands/__tests__/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -1,6 +1,6 @@ 'use strict'; -const utils = require('../transfer/utils'); +const utils = require('../../transfer/utils'); const mockDataTransfer = { createLocalFileDestinationProvider: jest.fn(), @@ -18,12 +18,12 @@ jest.mock( { virtual: true } ); -const exportCommand = require('../transfer/export'); +const exportCommand = require('../../transfer/export'); const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); jest.spyOn(console, 'error').mockImplementation(() => {}); -jest.mock('../transfer/utils'); +jest.mock('../../transfer/utils'); const defaultFileName = 'defaultFilename'; diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js new file mode 100644 index 0000000000..2c58f96c58 --- /dev/null +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -0,0 +1,98 @@ +'use strict'; + +const utils = require('../../transfer/utils'); + +const mockDataTransfer = { + createRemoteStrapiDestinationProvider: jest.fn(), + createLocalStrapiSourceProvider: jest.fn(), + createTransferEngine: jest.fn().mockReturnValue({ + transfer: jest.fn().mockReturnValue(Promise.resolve({})), + }), +}; + +jest.mock( + '@strapi/data-transfer', + () => { + return mockDataTransfer; + }, + { virtual: true } +); + +const transferCommand = require('../../transfer/transfer'); + +const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); +jest.spyOn(console, 'error').mockImplementation(() => {}); + +jest.mock('../../transfer/utils'); + +const destinationUrl = 'ws://strapi.com'; + +describe('transfer', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('uses destination url provided by user without authentication', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + + expect(exit).toHaveBeenCalled(); + }); + + it('uses destination url provided by user with authentication', async () => { + // TODO when authentication is implemented + }); + + it('uses restore as the default strategy', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + it('uses destination url provided by user without authentication', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + + expect(exit).toHaveBeenCalled(); + }); + + it('uses destination url provided by user with authentication', async () => { + // TODO when authentication is implemented + }); + + it('uses restore as the default strategy', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + + it('uses local strapi instance when local specified', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createLocalStrapiSourceProvider).toHaveBeenCalled(); + expect(utils.createStrapiInstance).toHaveBeenCalled(); + + expect(exit).toHaveBeenCalled(); + }); + + it('creates the transfer engine successfully', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); +}); diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index e91e175763..e3f2db936f 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -17,12 +17,10 @@ const { } = require('./utils'); /** - * @typedef ImportCommandOptions Options given to the CLI import command + * @typedef TransferCommandOptions Options given to the CLI import command * - * @property {string} [file] The file path to import - * @property {boolean} [encrypt] Used to encrypt the final archive - * @property {string} [key] Encryption key, only useful when encryption is enabled - * @property {boolean} [compress] Used to compress the final archive + * @property {string} [from] The source strapi project + * @property {string} [to] The destination strapi project */ const logger = console; @@ -32,7 +30,7 @@ const logger = console; * * It transfers data from a local file to a local strapi instance * - * @param {ImportCommandOptions} opts + * @param {TransferCommandOptions} opts */ module.exports = async (opts) => { // Validate inputs from Commander