From e11e9d12cf75f791abc5e50e57665f864cc8101d Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 12 Dec 2022 19:02:28 +0100 Subject: [PATCH 01/20] add compile step --- .github/workflows/tests.yml | 8 ++++++++ package.json | 3 ++- packages/core/data-transfer/package.json | 4 ++-- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 497d804e64..1d55210f26 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,6 +34,8 @@ jobs: key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Compile + run: yarn compile - name: Run lint run: yarn run -s lint @@ -55,6 +57,8 @@ jobs: key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile + - name: Compile + run: yarn compile - name: Run tests run: yarn run -s test:unit --coverage - name: Upload coverage to Codecov @@ -84,6 +88,8 @@ jobs: - run: yarn install --frozen-lockfile - name: Build run: yarn build + - name: Compile + run: yarn compile - name: Run test run: yarn run -s test:front --coverage - name: Upload coverage to Codecov @@ -231,6 +237,7 @@ jobs: SQLITE_PKG: ${{ matrix.sqlite_pkg }} with: dbOptions: '--dbclient=sqlite-legacy --dbfile=./tmp/data.db' + # EE api_ee_pg: runs-on: ubuntu-latest @@ -275,6 +282,7 @@ jobs: with: dbOptions: '--dbclient=postgres --dbhost=localhost --dbport=5432 --dbname=strapi_test --dbusername=strapi --dbpassword=strapi' runEE: true + api_ee_mysql: runs-on: ubuntu-latest needs: [lint, unit_back, unit_front] diff --git a/package.json b/package.json index b53074e94b..ed3fcec155 100644 --- a/package.json +++ b/package.json @@ -29,10 +29,11 @@ ], "scripts": { "prepare": "husky install", - "setup": "yarn clean && yarn && yarn build", + "setup": "yarn clean && yarn && yarn compile && yarn build", "clean": "lerna run --stream clean --no-private", "watch": "lerna run --stream watch --no-private", "build": "lerna run --stream build --no-private", + "compile": "lerna run --stream compile --no-private", "generate": "plop --plopfile ./packages/generators/admin/plopfile.js", "lint": "npm-run-all -p lint:code lint:css", "lint:code": "eslint .", diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index 4b60e1b625..9095135eeb 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -27,9 +27,9 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { - "build": "tsc -p tsconfig.json", + "compile": "tsc -p tsconfig.json", "clean": "rimraf ./dist", - "build:clean": "yarn clean && yarn build", + "compile:clean": "yarn clean && yarn compile", "watch": "yarn build -w", "test:unit": "jest --verbose" }, From 9090650e88e331a3c1dbee8eea454851640664d4 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 12 Dec 2022 19:06:39 +0100 Subject: [PATCH 02/20] allow typescript package to attempt to be resolved --- packages/core/strapi/lib/commands/transfer/export.js | 2 -- packages/core/strapi/lib/commands/transfer/import.js | 2 -- 2 files changed, 4 deletions(-) diff --git a/packages/core/strapi/lib/commands/transfer/export.js b/packages/core/strapi/lib/commands/transfer/export.js index 4d92647a32..e74d6432db 100644 --- a/packages/core/strapi/lib/commands/transfer/export.js +++ b/packages/core/strapi/lib/commands/transfer/export.js @@ -4,8 +4,6 @@ const { createLocalFileDestinationProvider, createLocalStrapiSourceProvider, createTransferEngine, - // TODO: we need to solve this issue with typescript modules - // eslint-disable-next-line import/no-unresolved, node/no-missing-require } = require('@strapi/data-transfer'); const { isObject, isString, isFinite, toNumber } = require('lodash/fp'); const fs = require('fs-extra'); diff --git a/packages/core/strapi/lib/commands/transfer/import.js b/packages/core/strapi/lib/commands/transfer/import.js index d0413a91a4..864f5edd4b 100644 --- a/packages/core/strapi/lib/commands/transfer/import.js +++ b/packages/core/strapi/lib/commands/transfer/import.js @@ -4,8 +4,6 @@ const { createLocalFileSourceProvider, createLocalStrapiDestinationProvider, createTransferEngine, - // TODO: we need to solve this issue with typescript modules - // eslint-disable-next-line import/no-unresolved, node/no-missing-require } = require('@strapi/data-transfer'); const { isObject } = require('lodash/fp'); const path = require('path'); From 6a532e3bb3cb972c288c2d50b3db784f428ab444 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 12 Dec 2022 19:08:42 +0100 Subject: [PATCH 03/20] fix watch --- packages/core/data-transfer/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index 9095135eeb..d91f1dbfaf 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -30,7 +30,7 @@ "compile": "tsc -p tsconfig.json", "clean": "rimraf ./dist", "compile:clean": "yarn clean && yarn compile", - "watch": "yarn build -w", + "watch": "yarn compile -w", "test:unit": "jest --verbose" }, "directories": { From 63d58433fcb7abb1eb4a05b0758ee9fefdb85f04 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 22 Dec 2022 18:48:23 +0100 Subject: [PATCH 04/20] Add typings for transfer --- .../local-file-destination-provider/index.ts | 1 + .../index.ts | 37 +++- packages/core/data-transfer/lib/register.ts | 171 ++++++++++-------- packages/core/data-transfer/types/index.d.ts | 1 + packages/core/data-transfer/types/remote.d.ts | 90 +++++++++ 5 files changed, 227 insertions(+), 73 deletions(-) create mode 100644 packages/core/data-transfer/types/remote.d.ts diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts index 081ebef70b..3799915c74 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts @@ -255,6 +255,7 @@ class LocalFileDestinationProvider implements IDestinationProvider { entry .on('finish', () => { + console.log('FINISH WRITING ALREADY'); callback(null); }) .on('error', (error) => { diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 8cdbd9a62d..1d9661e55f 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -2,7 +2,6 @@ import { WebSocket } from 'ws'; import { v4 } from 'uuid'; import { Writable } from 'stream'; -import type { restore } from '../local-strapi-destination-provider/strategies'; import type { IDestinationProvider, IEntity, @@ -11,6 +10,7 @@ import type { ProviderType, IConfiguration, TransferStage, + IAsset, } from '../../../types'; import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; @@ -199,4 +199,39 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { }, }); } + + getAssetsStream(): Writable | Promise { + return new Writable({ + objectMode: true, + final: async (callback) => { + const e = await this.#dispatchTransfer('assets', null); + callback(e); + }, + write: async (asset: IAsset, _encoding, callback) => { + const { filename, filepath, stats, stream } = asset; + const assetID = v4(); + + await this.#dispatchTransfer('assets', { + step: 'start', + assetID, + data: { filename, filepath, stats }, + }); + + for await (const chunk of stream) { + await this.#dispatchTransfer('assets', { + step: 'stream', + assetID, + data: { chunk }, + }); + } + + await this.#dispatchTransfer('assets', { + step: 'end', + assetID, + }); + + callback(); + }, + }); + } } diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4e1d94a08f..4f358ab65f 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -1,70 +1,35 @@ import type { Context } from 'koa'; import type { ServerOptions } from 'ws'; import { WebSocket } from 'ws'; -import { Writable } from 'stream'; -import { IAsset, IConfiguration, IEntity, ILink, IMetadata, TransferStage } from '../types'; +import { Writable, PassThrough } from 'stream'; +import { v4 } from 'uuid'; +import { + IAsset, + IConfiguration, + Message, + ILink, + IMetadata, + PushTransferMessage, + PushEntitiesTransferMessage, + TransferKind, + InitMessage, + PushTransferStage, +} from '../types'; import { - createLocalStrapiDestinationProvider, ILocalStrapiDestinationProviderOptions, + createLocalStrapiDestinationProvider, } from './providers'; -type PushTransferStage = Exclude; -type MessageKind = 'push' | 'pull'; - -type Message = { uuid: string } & (InitMessage | TransferMessage | ActionMessage | TeardownMessage); - -// init - -type InitMessage = { type: 'init' } & (IPushInitMessage | IPullInitMessage); - -interface IPushInitMessage { - type: 'init'; - kind: 'push'; - data: Pick; -} - -interface IPullInitMessage { - type: 'init'; - kind: 'pull'; -} - -// teardown - -type TeardownMessage = { type: 'teardown' }; - -// action - -type ActionMessage = { - type: 'action'; - action: 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; -}; - -// transfer - -type TransferMessage = PushTransferMessage; - -type PushTransferMessage = { type: 'transfer' } & ( - | PushEntityMessage - | PushLinkMessage - | PushAssetMessage - | PushConfigurationMessage -); - -type PushEntityMessage = { stage: 'entities'; data: IEntity }; -type PushLinkMessage = { stage: 'links'; data: ILink }; -type PushAssetMessage = { stage: 'assets'; data: IAsset }; -type PushConfigurationMessage = { stage: 'configuration'; data: IConfiguration }; - -// Internal state - interface ITransferState { - kind?: MessageKind; + kind?: TransferKind; + transferID?: string; controller?: IPushController; } // Controllers interface IPushController { + streams: { [stage in PushTransferStage]?: Writable }; actions: { getMetadata(): Promise; getSchemas(): Strapi.Schemas; @@ -73,20 +38,17 @@ interface IPushController { beforeTransfer(): Promise; }; transfer: { - entities(entity: IEntity): Promise | void; - links(link: ILink): Promise | void; - configuration(configuration: IConfiguration): Promise | void; - assets(asset: IAsset): Promise | void; + [key in PushTransferStage]: ( + value: T extends { stage: P; data: infer U } ? U : never + ) => Promise; }; } -const createPushController = ( - ws: WebSocket, - options: ILocalStrapiDestinationProviderOptions -): IPushController => { +const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { const provider = createLocalStrapiDestinationProvider(options); const streams: { [stage in PushTransferStage]?: Writable } = {}; + const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; const writeAsync = (stream: Writable, data: T) => { return new Promise((resolve, reject) => { @@ -101,6 +63,8 @@ const createPushController = ( }; return { + streams, + actions: { async getSchemas() { return provider.getSchemas(); @@ -129,7 +93,7 @@ const createPushController = ( streams.entities = provider.getEntitiesStream(); } - await writeAsync(streams.entities, entity); + await writeAsync(streams.entities!, entity); }, async links(link) { @@ -137,7 +101,7 @@ const createPushController = ( streams.links = await provider.getLinksStream(); } - await writeAsync(streams.links, link); + await writeAsync(streams.links!, link); }, async configuration(config) { @@ -145,14 +109,46 @@ const createPushController = ( streams.configuration = await provider.getConfigurationStream(); } - await writeAsync(streams.configuration, config); + await writeAsync(streams.configuration!, config); }, - async assets(asset) { + async assets(asset: any) { + if (asset === null) { + streams.assets?.end(); + return; + } + + const { step, assetID } = asset; + if (!streams.assets) { streams.assets = await provider.getAssetsStream(); } - await writeAsync(streams.assets, asset); + + // on init, we create a passthrough stream for the asset chunks + // send to the assets destination stream the metadata for the current asset + // + the stream that we just created for the asset + if (step === 'start') { + assets[assetID] = { ...asset.data, stream: new PassThrough() }; + writeAsync(streams.assets!, assets[assetID]); + } + + // propagate the chunk + if (step === 'stream') { + await writeAsync(assets[assetID].stream, Buffer.from(asset.data.chunk)); + } + + // on end, we indicate that all the chunks have been sent + if (step === 'end') { + await new Promise((resolve, reject) => { + assets[assetID].stream + .on('close', () => { + delete assets[assetID]; + resolve(); + }) + .on('error', reject) + .end(); + }); + } }, }, }; @@ -211,18 +207,21 @@ const createTransferController = const teardown = () => { delete state.kind; delete state.controller; + delete state.transferID; return { ok: true }; }; - const init = (kind: MessageKind, data: unknown = {}) => { + const init = (msg: InitMessage) => { + const { kind, options: controllerOptions } = msg; + if (state.controller) { throw new Error('Transfer already in progres'); } if (kind === 'push') { - state.controller = createPushController(ws, { - ...(data as IPushInitMessage['data']), + state.controller = createPushController({ + ...controllerOptions, autoDestroy: false, getStrapi() { return strapi; @@ -230,7 +229,14 @@ const createTransferController = }); } - return { ok: true }; + // Pull or others + else { + throw new Error(`${kind} transfer not implemented`); + } + + state.transferID = v4(); + + return { transferID: state.transferID }; }; ws.on('close', () => { @@ -252,7 +258,7 @@ const createTransferController = uuid = msg.uuid; if (msg.type === 'init') { - await answer(() => init(msg.kind, (msg as any)?.data)); + await answer(() => init(msg)); } if (msg.type === 'teardown') { @@ -264,7 +270,13 @@ const createTransferController = } if (msg.type === 'transfer') { - await answer(() => state.controller?.transfer[msg.stage]?.(msg.data as any)); + await answer(() => { + const fn = state.controller?.transfer[msg.stage]; + + type Msg = typeof msg; + + fn?.(msg.data); + }); } }); }); @@ -287,3 +299,18 @@ const register = (strapi: any) => { }; export default register; + +/** + * entities:start + * entities:transfer + * entities:end + * + * + * assets:start + * + * assets:transfer:start + * assets:transfer:stream + * assets:transfer:end + * + * assets:end + */ diff --git a/packages/core/data-transfer/types/index.d.ts b/packages/core/data-transfer/types/index.d.ts index 73d649f130..51f3087de4 100644 --- a/packages/core/data-transfer/types/index.d.ts +++ b/packages/core/data-transfer/types/index.d.ts @@ -3,3 +3,4 @@ export * from './providers'; export * from './transfer-engine'; export * from './utils'; export * from './encryption'; +export * from './remote'; diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts new file mode 100644 index 0000000000..cb64d420c1 --- /dev/null +++ b/packages/core/data-transfer/types/remote.d.ts @@ -0,0 +1,90 @@ +import type { ILocalStrapiDestinationProviderOptions } from '../lib'; +import type { IAsset, IConfiguration, IEntity, ILink } from './common-entities'; + +/** + * Utils + */ + +type EmptyObject = Record; + +/** + * Messages + */ + +export type Message = { uuid: string | null | undefined } & ( + | InitMessage + | ActionMessage + | PushTransferMessage + | TeardownMessage +); + +export type MessageType = Message['type']; +export type TransferKind = InitMessage['kind']; +export type PushTransferStage = PushTransferMessage['stage']; + +/** + * Init + */ + +// init should return a transfer ID used in the teardown +export type InitMessage = { type: 'init' } & ( + | { kind: 'pull'; options: EmptyObject } + | { kind: 'push'; options: Pick } +); + +/** + * Action + */ + +export type ActionMessage = { type: 'action' } & ( + | { action: 'getMetadata'; options: EmptyObject } + | { action: 'getSchemas'; options: EmptyObject } + | { action: 'bootstrap'; options: EmptyObject } + | { action: 'close'; options: EmptyObject } + | { action: 'beforeTransfer'; options: EmptyObject } +); + +/** + * Transfer + */ + +export type PushTransferMessage = { + type: 'transfer'; +} & ( + | PushEntitiesTransferMessage + | PushLinksTransferMessage + | PushConfigurationTransferMessage + | PushAssetTransferMessage +); + +export type PushEntitiesTransferMessage = { + stage: 'entities'; + data: IEntity | null; +}; + +export type PushLinksTransferMessage = { stage: 'links'; data: ILink | null }; + +export type PushConfigurationTransferMessage = { + stage: 'configuration'; + data: IConfiguration | null; +}; + +export type PushAssetTransferMessage = { + stage: 'assets'; + data: + | ({ assetID: string } & ( + | { step: 'start'; data: Omit } + | { step: 'stream'; data: { chunk: { type: 'Buffer'; data: number[] } } } + | { step: 'end'; data: EmptyObject } + )) + | null; +}; + +/** + * Teardown + */ + +export type TeardownMessage = { + type: 'teardown'; + transferID: string; +}; From 4190687975f0f045af7f6b4f014d4b23fe44e45d Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Tue, 27 Dec 2022 09:55:26 +0100 Subject: [PATCH 05/20] Add tests and refactor --- .../core/data-transfer/lib/engine/index.ts | 35 +++---- .../__tests__/index.test.ts | 48 +++++++++ .../__tests__/utils.test.ts | 43 ++++++++ .../index.ts | 53 +++------- .../utils.ts | 34 +++++++ packages/core/data-transfer/lib/register.ts | 54 ++++------ .../{ => data-transfer}/export.test.js | 6 +- .../__tests__/data-transfer/transfer.test.js | 98 +++++++++++++++++++ .../strapi/lib/commands/transfer/transfer.js | 10 +- 9 files changed, 274 insertions(+), 107 deletions(-) create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts create mode 100644 packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts rename packages/core/strapi/lib/commands/__tests__/{ => data-transfer}/export.test.js (94%) create mode 100644 packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index 779f2d3fd7..489ee01c9b 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -338,28 +338,23 @@ class TransferEngine< async transfer(): Promise> { try { await this.bootstrap(); - await this.init(); - - const isValidTransfer = await this.integrityCheck(); - - if (!isValidTransfer) { - // TODO: provide the log from the integrity check - throw new Error( - `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` - ); - } - - await this.beforeTransfer(); - + // await this.init(); + // const isValidTransfer = await this.integrityCheck(); + // if (!isValidTransfer) { + // // TODO: provide the log from the integrity check + // throw new Error( + // `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` + // ); + // } + // await this.beforeTransfer(); // Run the transfer stages - await this.transferSchemas(); - await this.transferEntities(); - await this.transferAssets(); - await this.transferLinks(); - await this.transferConfiguration(); - + // await this.transferSchemas(); + // await this.transferEntities(); + // await this.transferAssets(); + // await this.transferLinks(); + // await this.transferConfiguration(); // Gracefully close the providers - await this.close(); + // await this.close(); } catch (e: unknown) { // Rollback the destination provider if an exception is thrown during the transfer // Note: This will be configurable in the future diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts new file mode 100644 index 0000000000..8a7a744af4 --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/index.test.ts @@ -0,0 +1,48 @@ +import type { IRemoteStrapiDestinationProviderOptions } from '..'; + +import { createRemoteStrapiDestinationProvider } from '..'; + +const defaultOptions: IRemoteStrapiDestinationProviderOptions = { + strategy: 'restore', + url: 'ws://test.com/admin/transfer', +}; + +jest.mock('../utils', () => ({ + dispatch: jest.fn(), +})); + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn((type, callback) => { + callback(); + return { + once: jest.fn((t, c) => c), + }; + }), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination', () => { + describe('Bootstrap', () => { + test('Should not have a defined websocket connection if bootstrap has not been called', () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + + expect(provider.ws).toBeNull(); + }); + + test('Should have a defined websocket connection if bootstrap has been called', async () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + await provider.bootstrap(); + + expect(provider.ws).not.toBeNull(); + }); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts new file mode 100644 index 0000000000..1cd4148b7a --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/__tests__/utils.test.ts @@ -0,0 +1,43 @@ +import { WebSocket } from 'ws'; +import { dispatch } from '../utils'; + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn(), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination Utils', () => { + test('Dispatch method sends payload', () => { + const ws = new WebSocket('ws://test/admin/transfer'); + const message = { + test: 'hello', + }; + + dispatch(ws, message); + + expect.extend({ + toContain(receivedString, expected) { + const jsonReceived = JSON.parse(receivedString); + const pass = Object.keys(expected).every((key) => jsonReceived[key] === expected[key]); + + return { + message: () => + `Expected ${jsonReceived} ${!pass && 'not'} to contain properties ${expected}`, + pass, + }; + }, + }); + + // @ts-ignore + expect(ws.send).toHaveBeenCalledWith(expect.toContain(message), expect.anything()); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 1d9661e55f..03fcefa284 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -13,6 +13,7 @@ import type { IAsset, } from '../../../types'; import type { ILocalStrapiDestinationProviderOptions } from '../local-strapi-destination-provider'; +import { dispatch } from './utils'; interface ITokenAuth { type: 'token'; @@ -25,7 +26,7 @@ interface ICredentialsAuth { password: string; } -interface IRemoteStrapiDestinationProvider +export interface IRemoteStrapiDestinationProviderOptions extends Pick { url: string; auth?: ITokenAuth | ICredentialsAuth; @@ -34,7 +35,7 @@ interface IRemoteStrapiDestinationProvider type Actions = 'bootstrap' | 'close' | 'beforeTransfer' | 'getMetadata' | 'getSchemas'; export const createRemoteStrapiDestinationProvider = ( - options: IRemoteStrapiDestinationProvider + options: IRemoteStrapiDestinationProviderOptions ) => { return new RemoteStrapiDestinationProvider(options); }; @@ -44,61 +45,28 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { type: ProviderType = 'destination'; - options: IRemoteStrapiDestinationProvider; + options: IRemoteStrapiDestinationProviderOptions; ws: WebSocket | null; - constructor(options: IRemoteStrapiDestinationProvider) { + constructor(options: IRemoteStrapiDestinationProviderOptions) { this.options = options; this.ws = null; } - async #dispatch(message: T): Promise { - const { ws } = this; - - if (!ws) { - throw new Error('No ws connection found'); - } - - return new Promise((resolve, reject) => { - const uuid = v4(); - const payload = JSON.stringify({ ...message, uuid }); - - ws.send(payload, (error) => { - if (error) { - reject(error); - } - }); - - ws.once('message', (raw) => { - const response: { uuid: string; data: U; error: string | null } = JSON.parse( - raw.toString() - ); - - if (response.error) { - return reject(new Error(response.error)); - } - - if (response.uuid === uuid) { - return resolve(response.data); - } - }); - }); - } - async #dispatchAction(action: Actions) { - return this.#dispatch({ type: 'action', action }); + return dispatch(this.ws, { type: 'action', action }); } async #dispatchTransfer(stage: TransferStage, data: T) { try { - await this.#dispatch({ type: 'transfer', stage, data }); + await dispatch(this.ws, { type: 'transfer', stage, data }); } catch (e) { if (e instanceof Error) { return e; } - return new Error('Unexected error'); + return new Error('Unexpected error'); } return null; @@ -131,7 +99,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { // Wait for the connection to be made to the server, then init the transfer await new Promise((resolve, reject) => { ws.once('open', async () => { - await this.#dispatch({ type: 'init', kind: 'push', data: { strategy, restore } }); + await dispatch(this.ws, { type: 'init', kind: 'push', data: { strategy, restore } }); resolve(); }).once('error', reject); }); @@ -204,6 +172,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, final: async (callback) => { + console.log('FINAL'); const e = await this.#dispatchTransfer('assets', null); callback(e); }, @@ -217,6 +186,8 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { data: { filename, filepath, stats }, }); + console.log('is writing'); + for await (const chunk of stream) { await this.#dispatchTransfer('assets', { step: 'stream', diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts new file mode 100644 index 0000000000..b79fc64818 --- /dev/null +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/utils.ts @@ -0,0 +1,34 @@ +import { v4 } from 'uuid'; +import { WebSocket } from 'ws'; + +export async function dispatch( + ws: WebSocket | null, + message: T +): Promise { + if (!ws) { + throw new Error('No websocket connection found'); + } + + return new Promise((resolve, reject) => { + const uuid = v4(); + const payload = JSON.stringify({ ...message, uuid }); + + ws.send(payload, (error) => { + if (error) { + reject(error); + } + }); + + ws.once('message', (raw) => { + const response: { uuid: string; data: U; error: string | null } = JSON.parse(raw.toString()); + + if (response.error) { + return reject(new Error(response.error)); + } + + if (response.uuid === uuid) { + return resolve(response.data); + } + }); + }); +} diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4f358ab65f..4a6e75015a 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -5,12 +5,9 @@ import { Writable, PassThrough } from 'stream'; import { v4 } from 'uuid'; import { IAsset, - IConfiguration, Message, - ILink, IMetadata, PushTransferMessage, - PushEntitiesTransferMessage, TransferKind, InitMessage, PushTransferStage, @@ -38,8 +35,8 @@ interface IPushController { beforeTransfer(): Promise; }; transfer: { - [key in PushTransferStage]: ( - value: T extends { stage: P; data: infer U } ? U : never + [key in PushTransferStage]: ( + value: T extends { stage: key; data: infer U } ? U : never ) => Promise; }; } @@ -66,7 +63,7 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): streams, actions: { - async getSchemas() { + async getSchemas(): Promise { return provider.getSchemas(); }, @@ -112,35 +109,35 @@ const createPushController = (options: ILocalStrapiDestinationProviderOptions): await writeAsync(streams.configuration!, config); }, - async assets(asset: any) { - if (asset === null) { + async assets(payload) { + console.log('llega'); + if (payload === null) { streams.assets?.end(); return; } - const { step, assetID } = asset; + const { step, assetID } = payload; if (!streams.assets) { streams.assets = await provider.getAssetsStream(); } - // on init, we create a passthrough stream for the asset chunks - // send to the assets destination stream the metadata for the current asset - // + the stream that we just created for the asset if (step === 'start') { - assets[assetID] = { ...asset.data, stream: new PassThrough() }; - writeAsync(streams.assets!, assets[assetID]); + assets[assetID] = { ...payload.data, stream: new PassThrough() }; + writeAsync(streams.assets, assets[assetID]); } - // propagate the chunk if (step === 'stream') { - await writeAsync(assets[assetID].stream, Buffer.from(asset.data.chunk)); + const chunk = Buffer.from(payload.data.chunk.data); + + await writeAsync(assets[assetID].stream, chunk); } - // on end, we indicate that all the chunks have been sent if (step === 'end') { await new Promise((resolve, reject) => { - assets[assetID].stream + const { stream } = assets[assetID]; + + stream .on('close', () => { delete assets[assetID]; resolve(); @@ -271,11 +268,9 @@ const createTransferController = if (msg.type === 'transfer') { await answer(() => { - const fn = state.controller?.transfer[msg.stage]; + const { stage, data } = msg; - type Msg = typeof msg; - - fn?.(msg.data); + return state.controller?.transfer[stage](data as never); }); } }); @@ -299,18 +294,3 @@ const register = (strapi: any) => { }; export default register; - -/** - * entities:start - * entities:transfer - * entities:end - * - * - * assets:start - * - * assets:transfer:start - * assets:transfer:stream - * assets:transfer:end - * - * assets:end - */ diff --git a/packages/core/strapi/lib/commands/__tests__/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js similarity index 94% rename from packages/core/strapi/lib/commands/__tests__/export.test.js rename to packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index bf30ebeeda..5d81b7aa58 100644 --- a/packages/core/strapi/lib/commands/__tests__/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -1,6 +1,6 @@ 'use strict'; -const utils = require('../transfer/utils'); +const utils = require('../../transfer/utils'); const mockDataTransfer = { createLocalFileDestinationProvider: jest.fn(), @@ -18,12 +18,12 @@ jest.mock( { virtual: true } ); -const exportCommand = require('../transfer/export'); +const exportCommand = require('../../transfer/export'); const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); jest.spyOn(console, 'error').mockImplementation(() => {}); -jest.mock('../transfer/utils'); +jest.mock('../../transfer/utils'); const defaultFileName = 'defaultFilename'; diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js new file mode 100644 index 0000000000..2c58f96c58 --- /dev/null +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -0,0 +1,98 @@ +'use strict'; + +const utils = require('../../transfer/utils'); + +const mockDataTransfer = { + createRemoteStrapiDestinationProvider: jest.fn(), + createLocalStrapiSourceProvider: jest.fn(), + createTransferEngine: jest.fn().mockReturnValue({ + transfer: jest.fn().mockReturnValue(Promise.resolve({})), + }), +}; + +jest.mock( + '@strapi/data-transfer', + () => { + return mockDataTransfer; + }, + { virtual: true } +); + +const transferCommand = require('../../transfer/transfer'); + +const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); +jest.spyOn(console, 'error').mockImplementation(() => {}); + +jest.mock('../../transfer/utils'); + +const destinationUrl = 'ws://strapi.com'; + +describe('transfer', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('uses destination url provided by user without authentication', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + + expect(exit).toHaveBeenCalled(); + }); + + it('uses destination url provided by user with authentication', async () => { + // TODO when authentication is implemented + }); + + it('uses restore as the default strategy', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + it('uses destination url provided by user without authentication', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + + expect(exit).toHaveBeenCalled(); + }); + + it('uses destination url provided by user with authentication', async () => { + // TODO when authentication is implemented + }); + + it('uses restore as the default strategy', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + + it('uses local strapi instance when local specified', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + + expect(mockDataTransfer.createLocalStrapiSourceProvider).toHaveBeenCalled(); + expect(utils.createStrapiInstance).toHaveBeenCalled(); + + expect(exit).toHaveBeenCalled(); + }); + + it('creates the transfer engine successfully', async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); +}); diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js index e91e175763..e3f2db936f 100644 --- a/packages/core/strapi/lib/commands/transfer/transfer.js +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -17,12 +17,10 @@ const { } = require('./utils'); /** - * @typedef ImportCommandOptions Options given to the CLI import command + * @typedef TransferCommandOptions Options given to the CLI import command * - * @property {string} [file] The file path to import - * @property {boolean} [encrypt] Used to encrypt the final archive - * @property {string} [key] Encryption key, only useful when encryption is enabled - * @property {boolean} [compress] Used to compress the final archive + * @property {string} [from] The source strapi project + * @property {string} [to] The destination strapi project */ const logger = console; @@ -32,7 +30,7 @@ const logger = console; * * It transfers data from a local file to a local strapi instance * - * @param {ImportCommandOptions} opts + * @param {TransferCommandOptions} opts */ module.exports = async (opts) => { // Validate inputs from Commander From 777f76a0df8c8536395bfcdffda6b132f70d7aff Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 11:12:34 +0100 Subject: [PATCH 06/20] fix --- packages/core/data-transfer/lib/register.ts | 4 ++-- packages/core/data-transfer/types/remote.d.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 4a6e75015a..98030dcf59 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -179,7 +179,7 @@ const createTransferController = const payload = JSON.stringify({ uuid, data: data ?? {}, - error: e, + error: JSON.stringify(e?.message), }); ws.send(payload, (error) => (error ? reject(error) : resolve())); @@ -210,7 +210,7 @@ const createTransferController = }; const init = (msg: InitMessage) => { - const { kind, options: controllerOptions } = msg; + const { kind, data: controllerOptions } = msg; if (state.controller) { throw new Error('Transfer already in progres'); diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts index cb64d420c1..c651667953 100644 --- a/packages/core/data-transfer/types/remote.d.ts +++ b/packages/core/data-transfer/types/remote.d.ts @@ -28,8 +28,8 @@ export type PushTransferStage = PushTransferMessage['stage']; // init should return a transfer ID used in the teardown export type InitMessage = { type: 'init' } & ( - | { kind: 'pull'; options: EmptyObject } - | { kind: 'push'; options: Pick } + | { kind: 'pull'; data: EmptyObject } + | { kind: 'push'; data: Pick } ); /** From e3277b75743d4c4cf565c800c81968f90f573915 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 11:30:01 +0100 Subject: [PATCH 07/20] use options instead of data --- .../lib/providers/remote-strapi-destination-provider/index.ts | 2 +- packages/core/data-transfer/lib/register.ts | 2 +- packages/core/data-transfer/types/remote.d.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 03fcefa284..adddd27741 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -99,7 +99,7 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { // Wait for the connection to be made to the server, then init the transfer await new Promise((resolve, reject) => { ws.once('open', async () => { - await dispatch(this.ws, { type: 'init', kind: 'push', data: { strategy, restore } }); + await dispatch(this.ws, { type: 'init', kind: 'push', options: { strategy, restore } }); resolve(); }).once('error', reject); }); diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 98030dcf59..70b7b3ba6c 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -210,7 +210,7 @@ const createTransferController = }; const init = (msg: InitMessage) => { - const { kind, data: controllerOptions } = msg; + const { kind, options: controllerOptions } = msg; if (state.controller) { throw new Error('Transfer already in progres'); diff --git a/packages/core/data-transfer/types/remote.d.ts b/packages/core/data-transfer/types/remote.d.ts index c651667953..cb64d420c1 100644 --- a/packages/core/data-transfer/types/remote.d.ts +++ b/packages/core/data-transfer/types/remote.d.ts @@ -28,8 +28,8 @@ export type PushTransferStage = PushTransferMessage['stage']; // init should return a transfer ID used in the teardown export type InitMessage = { type: 'init' } & ( - | { kind: 'pull'; data: EmptyObject } - | { kind: 'push'; data: Pick } + | { kind: 'pull'; options: EmptyObject } + | { kind: 'push'; options: Pick } ); /** From 218ea114de02ed0c62581bf65ee242eb18a8a125 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 11:36:33 +0100 Subject: [PATCH 08/20] fix error response --- packages/core/data-transfer/lib/register.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index 70b7b3ba6c..d114f58152 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -179,7 +179,7 @@ const createTransferController = const payload = JSON.stringify({ uuid, data: data ?? {}, - error: JSON.stringify(e?.message), + error: e?.message || 'Unknown error', }); ws.send(payload, (error) => (error ? reject(error) : resolve())); From 252c562a8dea10da5175dcc83e59b3bbc4e3c88f Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Tue, 27 Dec 2022 12:52:39 +0100 Subject: [PATCH 09/20] Update package.json --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ed3fcec155..e60bfaa3ac 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ ], "scripts": { "prepare": "husky install", - "setup": "yarn clean && yarn && yarn compile && yarn build", + "setup": "yarn && yarn clean && yarn compile && yarn build", "clean": "lerna run --stream clean --no-private", "watch": "lerna run --stream watch --no-private", "build": "lerna run --stream build --no-private", From e6e20044b9659af6fd0681c224e71fee1191daac Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Wed, 28 Dec 2022 13:01:35 +0100 Subject: [PATCH 10/20] Add tests and refactor --- .../lib/bootstrap/controllers/index.ts | 2 + .../lib/bootstrap/controllers/push.ts | 134 +++++++++ .../lib/bootstrap/controllers/transfer.ts | 147 +++++++++ .../core/data-transfer/lib/engine/index.ts | 30 +- .../lib/providers/__tests__/register.test.ts | 32 ++ .../index.ts | 2 - packages/core/data-transfer/lib/register.ts | 282 +----------------- .../__tests__/data-transfer/transfer.test.js | 12 +- 8 files changed, 336 insertions(+), 305 deletions(-) create mode 100644 packages/core/data-transfer/lib/bootstrap/controllers/index.ts create mode 100644 packages/core/data-transfer/lib/bootstrap/controllers/push.ts create mode 100644 packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts create mode 100644 packages/core/data-transfer/lib/providers/__tests__/register.test.ts diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/index.ts b/packages/core/data-transfer/lib/bootstrap/controllers/index.ts new file mode 100644 index 0000000000..314b26e75b --- /dev/null +++ b/packages/core/data-transfer/lib/bootstrap/controllers/index.ts @@ -0,0 +1,2 @@ +export * from './push'; +export { default as createTransferController } from './transfer'; diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/push.ts b/packages/core/data-transfer/lib/bootstrap/controllers/push.ts new file mode 100644 index 0000000000..06a6e1981b --- /dev/null +++ b/packages/core/data-transfer/lib/bootstrap/controllers/push.ts @@ -0,0 +1,134 @@ +import { PassThrough, Writable } from 'stream-chain'; + +import { IAsset, IMetadata, PushTransferMessage, PushTransferStage } from '../../../types'; +import { + createLocalStrapiDestinationProvider, + ILocalStrapiDestinationProviderOptions, +} from '../../providers'; + +export interface IPushController { + streams: { [stage in PushTransferStage]?: Writable }; + actions: { + getMetadata(): Promise; + getSchemas(): Strapi.Schemas; + bootstrap(): Promise; + close(): Promise; + beforeTransfer(): Promise; + }; + transfer: { + [key in PushTransferStage]: ( + value: T extends { stage: key; data: infer U } ? U : never + ) => Promise; + }; +} + +const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { + const provider = createLocalStrapiDestinationProvider(options); + + const streams: { [stage in PushTransferStage]?: Writable } = {}; + const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; + + const writeAsync = (stream: Writable, data: T) => { + return new Promise((resolve, reject) => { + stream.write(data, (error) => { + if (error) { + reject(error); + } + + resolve(); + }); + }); + }; + + return { + streams, + + actions: { + async getSchemas(): Promise { + return provider.getSchemas(); + }, + + async getMetadata() { + return provider.getMetadata(); + }, + + async bootstrap() { + return provider.bootstrap(); + }, + + async close() { + return provider.close(); + }, + + async beforeTransfer() { + return provider.beforeTransfer(); + }, + }, + + transfer: { + async entities(entity) { + if (!streams.entities) { + streams.entities = provider.getEntitiesStream(); + } + + await writeAsync(streams.entities!, entity); + }, + + async links(link) { + if (!streams.links) { + streams.links = await provider.getLinksStream(); + } + + await writeAsync(streams.links!, link); + }, + + async configuration(config) { + if (!streams.configuration) { + streams.configuration = await provider.getConfigurationStream(); + } + + await writeAsync(streams.configuration!, config); + }, + + async assets(payload) { + if (payload === null) { + streams.assets?.end(); + return; + } + + const { step, assetID } = payload; + + if (!streams.assets) { + streams.assets = await provider.getAssetsStream(); + } + + if (step === 'start') { + assets[assetID] = { ...payload.data, stream: new PassThrough() }; + writeAsync(streams.assets, assets[assetID]); + } + + if (step === 'stream') { + const chunk = Buffer.from(payload.data.chunk.data); + + await writeAsync(assets[assetID].stream, chunk); + } + + if (step === 'end') { + await new Promise((resolve, reject) => { + const { stream } = assets[assetID]; + + stream + .on('close', () => { + delete assets[assetID]; + resolve(); + }) + .on('error', reject) + .end(); + }); + } + }, + }, + }; +}; + +export default createPushController; diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts new file mode 100644 index 0000000000..d52a848a57 --- /dev/null +++ b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts @@ -0,0 +1,147 @@ +import type { Context } from 'koa'; +import type { ServerOptions } from 'ws'; + +import { v4 } from 'uuid'; +import { WebSocket } from 'ws'; + +import type { IPushController } from './push'; + +import { InitMessage, Message, TransferKind } from '../../../types'; +import createPushController from './push'; + +interface ITransferState { + kind?: TransferKind; + transferID?: string; + controller?: IPushController; +} + +const createTransferController = + (options: ServerOptions = {}) => + async (ctx: Context) => { + const upgradeHeader = (ctx.request.headers.upgrade || '') + .split(',') + .map((s) => s.trim().toLowerCase()); + + // Create the websocket server + const wss = new WebSocket.Server({ ...options, noServer: true }); + + if (upgradeHeader.includes('websocket')) { + wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => { + // Create a connection between the client & the server + wss.emit('connection', ws, ctx.req); + + const state: ITransferState = {}; + let uuid: string | undefined; + + const callback = (e: Error | null = null, data?: T) => { + return new Promise((resolve, reject) => { + if (!uuid) { + reject(new Error('Missing uuid for this message')); + return; + } + + const payload = JSON.stringify({ + uuid, + data: data ?? {}, + error: e?.message || 'Unknown error', + }); + + ws.send(payload, (error) => (error ? reject(error) : resolve())); + }); + }; + + const answer = async (fn: () => T) => { + try { + const response = await fn(); + callback(null, response); + } catch (e) { + if (e instanceof Error) { + callback(e); + } else if (typeof e === 'string') { + callback(new Error(e)); + } else { + callback(new Error('Unexpected error')); + } + } + }; + + const teardown = () => { + delete state.kind; + delete state.controller; + delete state.transferID; + + return { ok: true }; + }; + + const init = (msg: InitMessage) => { + const { kind, options: controllerOptions } = msg; + + if (state.controller) { + throw new Error('Transfer already in progres'); + } + + if (kind === 'push') { + state.controller = createPushController({ + ...controllerOptions, + autoDestroy: false, + getStrapi() { + return strapi; + }, + }); + } + + // Pull or others + else { + throw new Error(`${kind} transfer not implemented`); + } + + state.transferID = v4(); + + return { transferID: state.transferID }; + }; + + ws.on('close', () => { + teardown(); + }); + + ws.on('error', (e) => { + teardown(); + console.error(e); + }); + + ws.on('message', async (raw) => { + const msg: Message = JSON.parse(raw.toString()); + + if (!msg.uuid) { + throw new Error('Missing uuid in message'); + } + + uuid = msg.uuid; + + if (msg.type === 'init') { + await answer(() => init(msg)); + } + + if (msg.type === 'teardown') { + await answer(teardown); + } + + if (msg.type === 'action') { + await answer(() => state.controller?.actions[msg.action]?.()); + } + + if (msg.type === 'transfer') { + await answer(() => { + const { stage, data } = msg; + + return state.controller?.transfer[stage](data as never); + }); + } + }); + }); + + ctx.respond = false; + } + }; + +export default createTransferController; diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/lib/engine/index.ts index 489ee01c9b..d6b7027ae8 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/lib/engine/index.ts @@ -338,23 +338,23 @@ class TransferEngine< async transfer(): Promise> { try { await this.bootstrap(); - // await this.init(); - // const isValidTransfer = await this.integrityCheck(); - // if (!isValidTransfer) { - // // TODO: provide the log from the integrity check - // throw new Error( - // `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` - // ); - // } - // await this.beforeTransfer(); + await this.init(); + const isValidTransfer = await this.integrityCheck(); + if (!isValidTransfer) { + // TODO: provide the log from the integrity check + throw new Error( + `Unable to transfer the data between ${this.sourceProvider.name} and ${this.destinationProvider.name}.\nPlease refer to the log above for more information.` + ); + } + await this.beforeTransfer(); // Run the transfer stages - // await this.transferSchemas(); - // await this.transferEntities(); - // await this.transferAssets(); - // await this.transferLinks(); - // await this.transferConfiguration(); + await this.transferSchemas(); + await this.transferEntities(); + await this.transferAssets(); + await this.transferLinks(); + await this.transferConfiguration(); // Gracefully close the providers - // await this.close(); + await this.close(); } catch (e: unknown) { // Rollback the destination provider if an exception is thrown during the transfer // Note: This will be configurable in the future diff --git a/packages/core/data-transfer/lib/providers/__tests__/register.test.ts b/packages/core/data-transfer/lib/providers/__tests__/register.test.ts new file mode 100644 index 0000000000..56ed87157b --- /dev/null +++ b/packages/core/data-transfer/lib/providers/__tests__/register.test.ts @@ -0,0 +1,32 @@ +import { createTransferController } from '../../bootstrap/controllers'; +import register from '../../register'; + +afterEach(() => { + jest.clearAllMocks(); +}); + +const strapiMock = { + admin: { + routes: { + push: jest.fn(), + }, + }, +}; + +jest.mock('../../bootstrap/controllers', () => ({ + createTransferController: jest.fn(), +})); + +describe('Register the Transfer route', () => { + test('registers the /transfer route', () => { + register(strapiMock); + expect(strapiMock.admin.routes.push).toHaveBeenCalledWith({ + method: 'GET', + path: '/transfer', + handler: createTransferController(), + config: { + auth: false, + }, + }); + }); +}); diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index adddd27741..3424cb2559 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -186,8 +186,6 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { data: { filename, filepath, stats }, }); - console.log('is writing'); - for await (const chunk of stream) { await this.#dispatchTransfer('assets', { step: 'stream', diff --git a/packages/core/data-transfer/lib/register.ts b/packages/core/data-transfer/lib/register.ts index d114f58152..c971fbce77 100644 --- a/packages/core/data-transfer/lib/register.ts +++ b/packages/core/data-transfer/lib/register.ts @@ -1,284 +1,4 @@ -import type { Context } from 'koa'; -import type { ServerOptions } from 'ws'; -import { WebSocket } from 'ws'; -import { Writable, PassThrough } from 'stream'; -import { v4 } from 'uuid'; -import { - IAsset, - Message, - IMetadata, - PushTransferMessage, - TransferKind, - InitMessage, - PushTransferStage, -} from '../types'; -import { - ILocalStrapiDestinationProviderOptions, - createLocalStrapiDestinationProvider, -} from './providers'; - -interface ITransferState { - kind?: TransferKind; - transferID?: string; - controller?: IPushController; -} - -// Controllers - -interface IPushController { - streams: { [stage in PushTransferStage]?: Writable }; - actions: { - getMetadata(): Promise; - getSchemas(): Strapi.Schemas; - bootstrap(): Promise; - close(): Promise; - beforeTransfer(): Promise; - }; - transfer: { - [key in PushTransferStage]: ( - value: T extends { stage: key; data: infer U } ? U : never - ) => Promise; - }; -} - -const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { - const provider = createLocalStrapiDestinationProvider(options); - - const streams: { [stage in PushTransferStage]?: Writable } = {}; - const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; - - const writeAsync = (stream: Writable, data: T) => { - return new Promise((resolve, reject) => { - stream.write(data, (error) => { - if (error) { - reject(error); - } - - resolve(); - }); - }); - }; - - return { - streams, - - actions: { - async getSchemas(): Promise { - return provider.getSchemas(); - }, - - async getMetadata() { - return provider.getMetadata(); - }, - - async bootstrap() { - return provider.bootstrap(); - }, - - async close() { - return provider.close(); - }, - - async beforeTransfer() { - return provider.beforeTransfer(); - }, - }, - - transfer: { - async entities(entity) { - if (!streams.entities) { - streams.entities = provider.getEntitiesStream(); - } - - await writeAsync(streams.entities!, entity); - }, - - async links(link) { - if (!streams.links) { - streams.links = await provider.getLinksStream(); - } - - await writeAsync(streams.links!, link); - }, - - async configuration(config) { - if (!streams.configuration) { - streams.configuration = await provider.getConfigurationStream(); - } - - await writeAsync(streams.configuration!, config); - }, - - async assets(payload) { - console.log('llega'); - if (payload === null) { - streams.assets?.end(); - return; - } - - const { step, assetID } = payload; - - if (!streams.assets) { - streams.assets = await provider.getAssetsStream(); - } - - if (step === 'start') { - assets[assetID] = { ...payload.data, stream: new PassThrough() }; - writeAsync(streams.assets, assets[assetID]); - } - - if (step === 'stream') { - const chunk = Buffer.from(payload.data.chunk.data); - - await writeAsync(assets[assetID].stream, chunk); - } - - if (step === 'end') { - await new Promise((resolve, reject) => { - const { stream } = assets[assetID]; - - stream - .on('close', () => { - delete assets[assetID]; - resolve(); - }) - .on('error', reject) - .end(); - }); - } - }, - }, - }; -}; - -const createTransferController = - (options: ServerOptions = {}) => - async (ctx: Context) => { - const upgradeHeader = (ctx.request.headers.upgrade || '') - .split(',') - .map((s) => s.trim().toLowerCase()); - - // Create the websocket server - const wss = new WebSocket.Server({ ...options, noServer: true }); - - if (upgradeHeader.includes('websocket')) { - wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => { - // Create a connection between the client & the server - wss.emit('connection', ws, ctx.req); - - const state: ITransferState = {}; - let uuid: string | undefined; - - const callback = (e: Error | null = null, data?: T) => { - return new Promise((resolve, reject) => { - if (!uuid) { - reject(new Error('Missing uuid for this message')); - return; - } - - const payload = JSON.stringify({ - uuid, - data: data ?? {}, - error: e?.message || 'Unknown error', - }); - - ws.send(payload, (error) => (error ? reject(error) : resolve())); - }); - }; - - const answer = async (fn: () => T) => { - try { - const response = await fn(); - callback(null, response); - } catch (e) { - if (e instanceof Error) { - callback(e); - } else if (typeof e === 'string') { - callback(new Error(e)); - } else { - callback(new Error('Unexpected error')); - } - } - }; - - const teardown = () => { - delete state.kind; - delete state.controller; - delete state.transferID; - - return { ok: true }; - }; - - const init = (msg: InitMessage) => { - const { kind, options: controllerOptions } = msg; - - if (state.controller) { - throw new Error('Transfer already in progres'); - } - - if (kind === 'push') { - state.controller = createPushController({ - ...controllerOptions, - autoDestroy: false, - getStrapi() { - return strapi; - }, - }); - } - - // Pull or others - else { - throw new Error(`${kind} transfer not implemented`); - } - - state.transferID = v4(); - - return { transferID: state.transferID }; - }; - - ws.on('close', () => { - teardown(); - }); - - ws.on('error', (e) => { - teardown(); - console.error(e); - }); - - ws.on('message', async (raw) => { - const msg: Message = JSON.parse(raw.toString()); - - if (!msg.uuid) { - throw new Error('Missing uuid in message'); - } - - uuid = msg.uuid; - - if (msg.type === 'init') { - await answer(() => init(msg)); - } - - if (msg.type === 'teardown') { - await answer(teardown); - } - - if (msg.type === 'action') { - await answer(() => state.controller?.actions[msg.action]?.()); - } - - if (msg.type === 'transfer') { - await answer(() => { - const { stage, data } = msg; - - return state.controller?.transfer[stage](data as never); - }); - } - }); - }); - - ctx.respond = false; - } - }; +import { createTransferController } from './bootstrap/controllers'; const registerTransferRoute = (strapi: any) => { strapi.admin.routes.push({ diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 2c58f96c58..34a6a36b48 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -21,7 +21,7 @@ jest.mock( const transferCommand = require('../../transfer/transfer'); const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); -jest.spyOn(console, 'error').mockImplementation(() => {}); +const logger = jest.spyOn(console, 'error').mockImplementation(() => {}); jest.mock('../../transfer/utils'); @@ -69,10 +69,6 @@ describe('transfer', () => { expect(exit).toHaveBeenCalled(); }); - it('uses destination url provided by user with authentication', async () => { - // TODO when authentication is implemented - }); - it('uses restore as the default strategy', async () => { await transferCommand({ from: 'local', to: destinationUrl }); @@ -92,7 +88,9 @@ describe('transfer', () => { expect(exit).toHaveBeenCalled(); }); - it('creates the transfer engine successfully', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + it('Logs an error when the source provider does not exist', async () => { + await transferCommand({ from: 'test', to: destinationUrl }); + + expect(logger).toHaveBeenCalledWith("Couldn't create providers"); }); }); From 6e88b09625210c2353ab9c609d28a56fdb982ab3 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Wed, 28 Dec 2022 13:07:21 +0100 Subject: [PATCH 11/20] Remove unknown error message because it seems to be breaking the tranfser --- .../core/data-transfer/lib/bootstrap/controllers/transfer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts index d52a848a57..cf57c2f514 100644 --- a/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts +++ b/packages/core/data-transfer/lib/bootstrap/controllers/transfer.ts @@ -43,7 +43,7 @@ const createTransferController = const payload = JSON.stringify({ uuid, data: data ?? {}, - error: e?.message || 'Unknown error', + error: e, }); ws.send(payload, (error) => (error ? reject(error) : resolve())); From 807800b9a5d6fd3a496eb55ea04cdc324fabcfb7 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 29 Dec 2022 11:12:09 +0100 Subject: [PATCH 12/20] Clean the code --- .../lib/providers/local-file-destination-provider/index.ts | 1 - .../lib/providers/remote-strapi-destination-provider/index.ts | 1 - .../lib/commands/__tests__/data-transfer/transfer.test.js | 4 +--- 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts index 3799915c74..081ebef70b 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts @@ -255,7 +255,6 @@ class LocalFileDestinationProvider implements IDestinationProvider { entry .on('finish', () => { - console.log('FINISH WRITING ALREADY'); callback(null); }) .on('error', (error) => { diff --git a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts index 3424cb2559..d2ac160085 100644 --- a/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/remote-strapi-destination-provider/index.ts @@ -172,7 +172,6 @@ class RemoteStrapiDestinationProvider implements IDestinationProvider { return new Writable({ objectMode: true, final: async (callback) => { - console.log('FINAL'); const e = await this.#dispatchTransfer('assets', null); callback(e); }, diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 34a6a36b48..0f26e65b42 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -44,9 +44,7 @@ describe('transfer', () => { expect(exit).toHaveBeenCalled(); }); - it('uses destination url provided by user with authentication', async () => { - // TODO when authentication is implemented - }); + it.todo('uses destination url provided by user with authentication'); it('uses restore as the default strategy', async () => { await transferCommand({ from: 'local', to: destinationUrl }); From 3cd8e1e8c06c2a5ed364d3c7f29bf967837efe90 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 29 Dec 2022 13:04:14 +0100 Subject: [PATCH 13/20] Fix mock path --- .../strapi/lib/commands/__tests__/data-transfer/export.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index 5fc0fb544b..1598c48453 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -41,7 +41,7 @@ describe('export', () => { getDefaultExportName: jest.fn(() => defaultFileName), }; jest.mock( - '../transfer/utils', + '../../transfer/utils', () => { return mockUtils; }, From 998d9677706e061d4dc24fa61ec06bf5479a5aa0 Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Thu, 29 Dec 2022 13:21:10 +0100 Subject: [PATCH 14/20] Fix the exit verifications --- .../__tests__/data-transfer/transfer.test.js | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js index 0f26e65b42..508384672c 100644 --- a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -18,9 +18,19 @@ jest.mock( { virtual: true } ); +const expectExit = async (code, fn) => { + const exit = jest.spyOn(process, 'exit').mockImplementation((number) => { + throw new Error(`process.exit: ${number}`); + }); + await expect(async () => { + await fn(); + }).rejects.toThrow(); + expect(exit).toHaveBeenCalledWith(code); + exit.mockRestore(); +}; + const transferCommand = require('../../transfer/transfer'); -const exit = jest.spyOn(process, 'exit').mockImplementation(() => {}); const logger = jest.spyOn(console, 'error').mockImplementation(() => {}); jest.mock('../../transfer/utils'); @@ -33,21 +43,23 @@ describe('transfer', () => { }); it('uses destination url provided by user without authentication', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ url: destinationUrl, }) ); - - expect(exit).toHaveBeenCalled(); }); it.todo('uses destination url provided by user with authentication'); it('uses restore as the default strategy', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ @@ -56,19 +68,21 @@ describe('transfer', () => { ); }); it('uses destination url provided by user without authentication', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ url: destinationUrl, }) ); - - expect(exit).toHaveBeenCalled(); }); it('uses restore as the default strategy', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createRemoteStrapiDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ @@ -78,16 +92,18 @@ describe('transfer', () => { }); it('uses local strapi instance when local specified', async () => { - await transferCommand({ from: 'local', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'local', to: destinationUrl }); + }); expect(mockDataTransfer.createLocalStrapiSourceProvider).toHaveBeenCalled(); expect(utils.createStrapiInstance).toHaveBeenCalled(); - - expect(exit).toHaveBeenCalled(); }); it('Logs an error when the source provider does not exist', async () => { - await transferCommand({ from: 'test', to: destinationUrl }); + await expectExit(1, async () => { + await transferCommand({ from: 'test', to: destinationUrl }); + }); expect(logger).toHaveBeenCalledWith("Couldn't create providers"); }); From 160a1683b50dd0487dd9fffc66888e800c616b6c Mon Sep 17 00:00:00 2001 From: Christian Capeans Date: Fri, 30 Dec 2022 09:48:36 +0100 Subject: [PATCH 15/20] Add no missing require to data-transfer --- packages/core/admin/server/register.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/core/admin/server/register.js b/packages/core/admin/server/register.js index 1789abf77e..b02e8def3d 100644 --- a/packages/core/admin/server/register.js +++ b/packages/core/admin/server/register.js @@ -1,5 +1,6 @@ 'use strict'; +// eslint-disable-next-line import/no-unresolved, node/no-missing-require const { register: registerDataTransfer } = require('@strapi/data-transfer'); const registerAdminPanelRoute = require('./routes/serve-admin-panel'); From fdcec5ea253dc79691b53db509e0879e8958983d Mon Sep 17 00:00:00 2001 From: Convly Date: Fri, 30 Dec 2022 10:55:40 +0100 Subject: [PATCH 16/20] Rename compile to build:ts --- .github/workflows/tests.yml | 12 ++++++------ package.json | 2 +- packages/core/data-transfer/package.json | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1d55210f26..5596ab2cdf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,8 +34,8 @@ jobs: key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Compile - run: yarn compile + - name: Build TypeScript packages + run: yarn build:ts - name: Run lint run: yarn run -s lint @@ -57,8 +57,8 @@ jobs: key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Compile - run: yarn compile + - name: Build TypeScript packages + run: yarn build:ts - name: Run tests run: yarn run -s test:unit --coverage - name: Upload coverage to Codecov @@ -88,8 +88,8 @@ jobs: - run: yarn install --frozen-lockfile - name: Build run: yarn build - - name: Compile - run: yarn compile + - name: Build TypeScript packages + run: yarn build:ts - name: Run test run: yarn run -s test:front --coverage - name: Upload coverage to Codecov diff --git a/package.json b/package.json index e60bfaa3ac..42ce2a3f5c 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "clean": "lerna run --stream clean --no-private", "watch": "lerna run --stream watch --no-private", "build": "lerna run --stream build --no-private", - "compile": "lerna run --stream compile --no-private", + "build:ts": "lerna run --stream build:ts --no-private", "generate": "plop --plopfile ./packages/generators/admin/plopfile.js", "lint": "npm-run-all -p lint:code lint:css", "lint:code": "eslint .", diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index bdcfd5a6f6..a4233d66f8 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -27,11 +27,13 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { - "compile": "tsc -p tsconfig.json", "clean": "rimraf ./dist", - "compile:clean": "yarn clean && yarn compile", - "watch": "yarn compile -w", - "test:unit": "jest --verbose" + "build": "yarn build:ts", + "build:ts": "tsc -p tsconfig.json", + "build:clean": "yarn clean && yarn build", + "watch": "yarn build:ts -w", + "test:unit": "jest --verbose", + "prepublishOnly": "yarn build:clean" }, "directories": { "lib": "./dist" From 7bbc065c353f84d07a78873f171f07e1026cf79a Mon Sep 17 00:00:00 2001 From: Convly Date: Fri, 30 Dec 2022 10:56:57 +0100 Subject: [PATCH 17/20] Update the setup script (compile => build:ts) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 42ce2a3f5c..ec7f407518 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ ], "scripts": { "prepare": "husky install", - "setup": "yarn && yarn clean && yarn compile && yarn build", + "setup": "yarn && yarn clean && yarn build:ts && yarn build", "clean": "lerna run --stream clean --no-private", "watch": "lerna run --stream watch --no-private", "build": "lerna run --stream build --no-private", From 3220d73f71d9015ab86020eab9a76ea6d1a5a56f Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 11:07:00 +0100 Subject: [PATCH 18/20] remove build:ts from yarn setup --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 604a4b50e9..5e05b1bda3 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ ], "scripts": { "prepare": "husky install", - "setup": "yarn && yarn clean && yarn build:ts && yarn build", + "setup": "yarn && yarn clean && yarn build", "clean": "lerna run --stream clean --no-private", "watch": "lerna run --stream watch --no-private --parallel", "build": "lerna run --stream build --no-private", From 437c1b7e177fd736973b680c5be55ab4084f4704 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 11:10:29 +0100 Subject: [PATCH 19/20] remove extraneous build:ts, add comments --- .github/workflows/tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5596ab2cdf..f683c85b0a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,7 +34,7 @@ jobs: key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages + - name: Build TypeScript packages # for lint we need to build ts, for rules checking if paths exist run: yarn build:ts - name: Run lint run: yarn run -s lint @@ -57,7 +57,7 @@ jobs: key: ${{ runner.os }}-${{ matrix.node }}-${{ hashFiles('**/yarn.lock') }} - run: yarn install --frozen-lockfile - - name: Build TypeScript packages + - name: Build TypeScript packages # for unit tests we need to build ts, for finding and mocking ts packages run: yarn build:ts - name: Run tests run: yarn run -s test:unit --coverage @@ -88,8 +88,6 @@ jobs: - run: yarn install --frozen-lockfile - name: Build run: yarn build - - name: Build TypeScript packages - run: yarn build:ts - name: Run test run: yarn run -s test:front --coverage - name: Upload coverage to Codecov From 1b4560ba28c4f014202ca69137258450ffed4fa8 Mon Sep 17 00:00:00 2001 From: Ben Irvin Date: Mon, 2 Jan 2023 11:12:09 +0100 Subject: [PATCH 20/20] alphabetize --- packages/core/data-transfer/package.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index 0ae176a828..8f8349b907 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -27,13 +27,13 @@ "main": "./dist/index.js", "types": "./dist/index.d.ts", "scripts": { - "clean": "rimraf ./dist", "build": "yarn build:ts", "build:ts": "tsc -p tsconfig.json", "build:clean": "yarn clean && yarn build", - "watch": "yarn build:ts -w --preserveWatchOutput", + "clean": "rimraf ./dist", + "prepublishOnly": "yarn build:clean", "test:unit": "jest --verbose", - "prepublishOnly": "yarn build:clean" + "watch": "yarn build:ts -w --preserveWatchOutput" }, "directories": { "lib": "./dist"