diff --git a/.eslintignore b/.eslintignore index b9cf1235d6..9662e15d5d 100644 --- a/.eslintignore +++ b/.eslintignore @@ -12,6 +12,7 @@ packages/core/helper-plugin/build/** packages/core/helper-plugin/lib/src/old/components/** packages/core/helper-plugin/lib/src/testUtils/** packages/core/helper-plugin/lib/src/utils/** +packages/core/data-transfer/lib .eslintrc.js .eslintrc.front.js .eslintrc.back.js diff --git a/.eslintrc.back.js b/.eslintrc.back.js index 1c079b8326..a4068ece55 100644 --- a/.eslintrc.back.js +++ b/.eslintrc.back.js @@ -31,6 +31,7 @@ module.exports = { 'no-continue': 'warn', 'no-process-exit': 'off', 'no-loop-func': 'off', + 'max-classes-per-file': 'off', 'no-param-reassign': [ 'error', { diff --git a/.eslintrc.back.typescript.js b/.eslintrc.back.typescript.js index 2e4260bd42..0c6543c94e 100644 --- a/.eslintrc.back.typescript.js +++ b/.eslintrc.back.typescript.js @@ -22,6 +22,7 @@ module.exports = { '@typescript-eslint/brace-style': 'off', // TODO: fix conflict with prettier/prettier in data-transfer/engine/index.ts // to be cleaned up throughout codebase (too many to fix at the moment) '@typescript-eslint/no-use-before-define': 'warn', + '@typescript-eslint/comma-dangle': 'off', }, // Disable only for tests overrides: [ diff --git a/packages/core/admin/package.json b/packages/core/admin/package.json index f34884caef..a26d23ebc6 100644 --- a/packages/core/admin/package.json +++ b/packages/core/admin/package.json @@ -46,6 +46,7 @@ "@casl/ability": "^5.4.3", "@fingerprintjs/fingerprintjs": "3.3.3", "@pmmmwh/react-refresh-webpack-plugin": "0.5.7", + "@strapi/data-transfer": "4.5.5", "@strapi/babel-plugin-switch-ee-ce": "4.5.5", "@strapi/design-system": "1.4.1", "@strapi/helper-plugin": "4.5.5", diff --git a/packages/core/admin/server/register.js b/packages/core/admin/server/register.js index 99d33710d9..d99d6326f8 100644 --- a/packages/core/admin/server/register.js +++ b/packages/core/admin/server/register.js @@ -1,5 +1,7 @@ 'use strict'; +const { register: registerDataTransfer } = require('@strapi/data-transfer/lib/strapi'); + const registerAdminPanelRoute = require('./routes/serve-admin-panel'); const adminAuthStrategy = require('./strategies/admin'); const apiTokenAuthStrategy = require('./strategies/api-token'); @@ -14,4 +16,6 @@ module.exports = ({ strapi }) => { if (strapi.config.serveAdminPanel) { registerAdminPanelRoute({ strapi }); } + + registerDataTransfer(strapi); }; diff --git a/packages/core/data-transfer/.gitignore b/packages/core/data-transfer/.gitignore new file mode 100644 index 0000000000..c3af857904 --- /dev/null +++ b/packages/core/data-transfer/.gitignore @@ -0,0 +1 @@ +lib/ diff --git a/packages/core/data-transfer/.npmignore b/packages/core/data-transfer/.npmignore index 5c1b474909..1bf5894edf 100644 --- a/packages/core/data-transfer/.npmignore +++ b/packages/core/data-transfer/.npmignore @@ -1,9 +1,9 @@ ############################# -# DATA TRANSFER X +# DATA TRANSFER ############################ *.js.map -lib/ +src/ types/ tsconfig.json @@ -19,14 +19,12 @@ Icon .Trashes ._* - ############################ # Linux ############################ *~ - ############################ # Windows ############################ @@ -40,7 +38,6 @@ $RECYCLE.BIN/ *.msm *.msp - ############################ # Packages ############################ @@ -69,7 +66,6 @@ $RECYCLE.BIN/ *.out *.pid - ############################ # Logs and databases ############################ @@ -79,7 +75,6 @@ $RECYCLE.BIN/ *.sql *.sqlite - ############################ # Misc. ############################ @@ -89,7 +84,6 @@ $RECYCLE.BIN/ nbproject .vscode/ - ############################ # Node.js ############################ @@ -107,7 +101,6 @@ package-lock.json !docs/package-lock.json *.heapsnapshot - ############################ # Tests ############################ diff --git a/packages/core/data-transfer/lib/index.ts b/packages/core/data-transfer/lib/index.ts deleted file mode 100644 index 95411a583a..0000000000 --- a/packages/core/data-transfer/lib/index.ts +++ /dev/null @@ -1,2 +0,0 @@ -export * from './engine'; -export * from './providers'; diff --git a/packages/core/data-transfer/lib/providers/index.ts b/packages/core/data-transfer/lib/providers/index.ts deleted file mode 100644 index 77df6a4bf7..0000000000 --- a/packages/core/data-transfer/lib/providers/index.ts +++ /dev/null @@ -1,7 +0,0 @@ -// source providers -export * from './local-file-source-provider'; -export * from './local-strapi-source-provider'; - -// destination providers -export * from './local-file-destination-provider'; -export * from './local-strapi-destination-provider'; diff --git a/packages/core/data-transfer/lib/providers/shared/index.ts b/packages/core/data-transfer/lib/providers/shared/index.ts deleted file mode 100644 index d3f3965b83..0000000000 --- a/packages/core/data-transfer/lib/providers/shared/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * as strapi from './strapi'; diff --git a/packages/core/data-transfer/lib/providers/test-utils/index.ts b/packages/core/data-transfer/lib/providers/test-utils/index.ts deleted file mode 100644 index c09362e900..0000000000 --- a/packages/core/data-transfer/lib/providers/test-utils/index.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Readable } from 'stream'; - -/** - * Collect every entity in a Readable stream - */ -export const collect = (stream: Readable): Promise => { - const chunks: T[] = []; - - return new Promise((resolve, reject) => { - stream - .on('data', (chunk) => chunks.push(chunk)) - .on('close', () => resolve(chunks)) - .on('error', reject); - }); -}; - -/** - * Create a "Strapi" like object factory based on the - * given params and cast it to the correct type - */ -export const getStrapiFactory = - < - T extends { - [key in keyof Partial]: unknown; - } - >( - properties?: T - ) => - () => { - return { ...properties } as Strapi.Strapi; - }; - -/** - * Union type used to represent the default content types available - */ -export type ContentType = 'foo' | 'bar'; - -/** - * Factory to get default content types test values - */ -export const getContentTypes = (): { - [key in ContentType]: { uid: key; attributes: { [attribute: string]: unknown } }; -} => ({ - foo: { uid: 'foo', attributes: { title: { type: 'string' } } }, - bar: { uid: 'bar', attributes: { age: { type: 'number' } } }, -}); - -/** - * Create a factory of readable streams (wrapped with a jest mock function) - */ -export const createMockedReadableFactory = (source: { - [ct in T]: Array<{ id: number; [key: string]: unknown }>; -}) => - jest.fn((uid: T) => { - return Readable.from(source[uid] || []); - }); - -/** - * Create a factory of mocked query builders - */ -export const createMockedQueryBuilder = (data: { - [key in T]: unknown[]; -}) => - jest.fn((uid: T) => { - const state: { [key: string]: unknown } = { populate: undefined }; - - return { - populate(populate: unknown) { - state.populate = populate; - return this; - }, - stream() { - return Readable.from(data[uid]); - }, - }; - }); - -/** - * Update the global store with the given strapi value - */ -export const setGlobalStrapi = (strapi: Strapi.Strapi): void => { - (global as unknown as Global).strapi = strapi; -}; diff --git a/packages/core/data-transfer/lib/utils/schema.ts b/packages/core/data-transfer/lib/utils/schema.ts deleted file mode 100644 index d59c12e22a..0000000000 --- a/packages/core/data-transfer/lib/utils/schema.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { Schema } from '@strapi/strapi'; -import { mapValues, pick } from 'lodash/fp'; - -const schemaSelectedKeys = [ - 'collectionName', - 'info', - 'options', - 'pluginOptions', - 'attributes', - 'kind', - 'modelType', - 'modelName', - 'uid', - 'plugin', - 'globalId', -]; - -export const mapSchemasValues = (schemas: Record) => { - return mapValues(pick(schemaSelectedKeys), schemas); -}; diff --git a/packages/core/data-transfer/lib/utils/stream.ts b/packages/core/data-transfer/lib/utils/stream.ts deleted file mode 100644 index 21c1c48da8..0000000000 --- a/packages/core/data-transfer/lib/utils/stream.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { Transform, Readable } from 'stream'; - -type TransformOptions = ConstructorParameters[0]; - -export const filter = ( - predicate: (value: T) => boolean | Promise, - options: TransformOptions = { objectMode: true } -): Transform => { - return new Transform({ - ...options, - - async transform(chunk, _encoding, callback) { - const keep = await predicate(chunk); - - callback(null, keep ? chunk : undefined); - }, - }); -}; - -export const map = ( - predicate: (value: T) => T | Promise, - options: TransformOptions = { objectMode: true } -): Transform => { - return new Transform({ - ...options, - - async transform(chunk, _encoding, callback) { - const mappedValue = await predicate(chunk); - - callback(null, mappedValue); - }, - }); -}; - -/** - * Collect every entity in a Readable stream - */ -export const collect = (stream: Readable): Promise => { - const chunks: T[] = []; - - return new Promise((resolve) => { - stream.on('data', (chunk) => chunks.push(chunk)); - stream.on('end', () => { - stream.destroy(); - resolve(chunks); - }); - }); -}; diff --git a/packages/core/data-transfer/package.json b/packages/core/data-transfer/package.json index 81d3a7e204..0cef69b748 100644 --- a/packages/core/data-transfer/package.json +++ b/packages/core/data-transfer/package.json @@ -24,8 +24,8 @@ "url": "https://strapi.io" } ], - "main": "./dist/index.js", - "types": "./dist/index.d.ts", + "main": "./lib/index.js", + "types": "./lib/index.d.ts", "scripts": { "build": "yarn build:ts", "build:ts": "tsc -p tsconfig.json", @@ -36,7 +36,7 @@ "watch": "yarn build:ts -w --preserveWatchOutput" }, "directories": { - "lib": "./dist" + "lib": "./lib" }, "dependencies": { "@strapi/logger": "4.5.5", @@ -49,7 +49,9 @@ "stream-chain": "2.2.5", "stream-json": "1.7.4", "tar": "6.1.12", - "tar-stream": "2.2.0" + "tar-stream": "2.2.0", + "uuid": "9.0.0", + "ws": "8.11.0" }, "devDependencies": { "@tsconfig/node16": "1.0.3", @@ -60,6 +62,9 @@ "@types/stream-json": "1.7.2", "@types/tar": "6.1.3", "@types/tar-stream": "2.2.2", + "@types/uuid": "9.0.0", + "koa": "2.13.4", + "@types/koa": "2.13.4", "rimraf": "3.0.2", "typescript": "4.6.2" }, diff --git a/packages/core/data-transfer/lib/__tests__/test-utils.ts b/packages/core/data-transfer/src/__tests__/test-utils.ts similarity index 97% rename from packages/core/data-transfer/lib/__tests__/test-utils.ts rename to packages/core/data-transfer/src/__tests__/test-utils.ts index 476f2992fa..8a86e4b3e7 100644 --- a/packages/core/data-transfer/lib/__tests__/test-utils.ts +++ b/packages/core/data-transfer/src/__tests__/test-utils.ts @@ -102,6 +102,13 @@ export const destinationStages = [ 'getSchemasStream', ]; +/** + * Update the global store with the given strapi value + */ +export const setGlobalStrapi = (strapi: Strapi.Strapi): void => { + (global as unknown as Global).strapi = strapi; +}; + /** * Add jest expect helpers */ diff --git a/packages/core/data-transfer/lib/engine/__tests__/engine.test.ts b/packages/core/data-transfer/src/engine/__tests__/engine.test.ts similarity index 100% rename from packages/core/data-transfer/lib/engine/__tests__/engine.test.ts rename to packages/core/data-transfer/src/engine/__tests__/engine.test.ts diff --git a/packages/core/data-transfer/lib/engine/index.ts b/packages/core/data-transfer/src/engine/index.ts similarity index 95% rename from packages/core/data-transfer/lib/engine/index.ts rename to packages/core/data-transfer/src/engine/index.ts index a8b4e4d1de..9e080295dd 100644 --- a/packages/core/data-transfer/lib/engine/index.ts +++ b/packages/core/data-transfer/src/engine/index.ts @@ -19,7 +19,7 @@ import type { } from '../../types'; import type { Diff } from '../utils/json'; -import compareSchemas from '../strategies'; +import { compareSchemas } from './validation/schemas'; import { filter, map } from '../utils/stream'; export const TRANSFER_STAGES: ReadonlyArray = Object.freeze([ @@ -250,7 +250,23 @@ class TransferEngine< const { stage, source, destination, transform, tracker } = options; if (!source || !destination) { + // Wait until source and destination are closed + await Promise.allSettled( + [source, destination].map((stream) => { + // if stream is undefined or already closed, resolve immediately + if (!stream || stream.destroyed) { + return Promise.resolve(); + } + + // Wait until the close event is produced and then destroy the stream and resolve + return new Promise((resolve, reject) => { + stream.on('close', resolve).on('error', reject).destroy(); + }); + }) + ); + this.#emitStageUpdate('skip', stage); + return; } @@ -340,9 +356,7 @@ class TransferEngine< this.#emitTransferUpdate('init'); await this.bootstrap(); await this.init(); - const isValidTransfer = await this.integrityCheck(); - if (!isValidTransfer) { // TODO: provide the log from the integrity check throw new Error( @@ -353,14 +367,12 @@ class TransferEngine< this.#emitTransferUpdate('start'); await this.beforeTransfer(); - // Run the transfer stages await this.transferSchemas(); await this.transferEntities(); await this.transferAssets(); await this.transferLinks(); await this.transferConfiguration(); - // Gracefully close the providers await this.close(); diff --git a/packages/core/data-transfer/src/engine/validation/index.ts b/packages/core/data-transfer/src/engine/validation/index.ts new file mode 100644 index 0000000000..197f1788ef --- /dev/null +++ b/packages/core/data-transfer/src/engine/validation/index.ts @@ -0,0 +1 @@ +export * as schemas from './schemas'; diff --git a/packages/core/data-transfer/lib/strategies/index.ts b/packages/core/data-transfer/src/engine/validation/schemas/index.ts similarity index 87% rename from packages/core/data-transfer/lib/strategies/index.ts rename to packages/core/data-transfer/src/engine/validation/schemas/index.ts index c0b69bb2f2..4d9f096235 100644 --- a/packages/core/data-transfer/lib/strategies/index.ts +++ b/packages/core/data-transfer/src/engine/validation/schemas/index.ts @@ -1,5 +1,5 @@ -import type { Diff } from '../utils/json'; -import * as utils from '../utils'; +import type { Diff } from '../../../utils/json'; +import * as utils from '../../../utils'; const strategies = { // No diffs @@ -32,4 +32,4 @@ const compareSchemas = (a: T, b: P, strategy: keyof typeof strategies) => return strategies[strategy](diffs); }; -export default compareSchemas; +export { compareSchemas }; diff --git a/packages/core/data-transfer/src/file/index.ts b/packages/core/data-transfer/src/file/index.ts new file mode 100644 index 0000000000..15f74296f5 --- /dev/null +++ b/packages/core/data-transfer/src/file/index.ts @@ -0,0 +1 @@ +export * as providers from './providers'; diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts similarity index 97% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts index 85580dece6..245d22070a 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts @@ -6,7 +6,7 @@ jest.mock('fs'); import fs from 'fs-extra'; import { Writable } from 'stream-chain'; import { createLocalFileDestinationProvider, ILocalFileDestinationProviderOptions } from '..'; -import * as encryption from '../../../encryption/encrypt'; +import * as encryption from '../../../../utils/encryption'; import { createFilePathFactory, createTarEntryStream } from '../utils'; fs.createWriteStream = jest.fn().mockReturnValue( @@ -18,14 +18,14 @@ fs.createWriteStream = jest.fn().mockReturnValue( const filePath = './test-file'; -jest.mock('../../../encryption/encrypt', () => { +jest.mock('../../../../utils/encryption', () => { return { __esModule: true, createEncryptionCipher() {}, }; }); -jest.mock('../../local-file-destination-provider/utils'); +jest.mock('../utils'); describe('Local File Destination Provider', () => { (createFilePathFactory as jest.Mock).mockImplementation(jest.fn()); diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts b/packages/core/data-transfer/src/file/providers/destination/__tests__/utils.test.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts rename to packages/core/data-transfer/src/file/providers/destination/__tests__/utils.test.ts diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts b/packages/core/data-transfer/src/file/providers/destination/index.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts rename to packages/core/data-transfer/src/file/providers/destination/index.ts index 081ebef70b..e064b5435b 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts +++ b/packages/core/data-transfer/src/file/providers/destination/index.ts @@ -6,7 +6,7 @@ import { stringer } from 'stream-json/jsonl/Stringer'; import { chain } from 'stream-chain'; import { Readable, Writable } from 'stream'; -import { createEncryptionCipher } from '../../encryption/encrypt'; +import { createEncryptionCipher } from '../../../utils/encryption'; import type { IAsset, IDestinationProvider, @@ -14,7 +14,7 @@ import type { IMetadata, ProviderType, Stream, -} from '../../../types'; +} from '../../../../types'; import { createFilePathFactory, createTarEntryStream } from './utils'; export interface ILocalFileDestinationProviderOptions { diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts b/packages/core/data-transfer/src/file/providers/destination/utils.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts rename to packages/core/data-transfer/src/file/providers/destination/utils.ts diff --git a/packages/core/data-transfer/src/file/providers/index.ts b/packages/core/data-transfer/src/file/providers/index.ts new file mode 100644 index 0000000000..d3c54e88d0 --- /dev/null +++ b/packages/core/data-transfer/src/file/providers/index.ts @@ -0,0 +1,2 @@ +export * from './source'; +export * from './destination'; diff --git a/packages/core/data-transfer/lib/providers/local-file-source-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/file/providers/source/__tests__/index.test.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-file-source-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/file/providers/source/__tests__/index.test.ts diff --git a/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts b/packages/core/data-transfer/src/file/providers/source/index.ts similarity index 90% rename from packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts rename to packages/core/data-transfer/src/file/providers/source/index.ts index ff078043dd..5b7c3e4f93 100644 --- a/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts +++ b/packages/core/data-transfer/src/file/providers/source/index.ts @@ -8,10 +8,10 @@ import { keyBy } from 'lodash/fp'; import { chain } from 'stream-chain'; import { pipeline, PassThrough } from 'stream'; import { parser } from 'stream-json/jsonl/Parser'; -import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../types'; +import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../../types'; -import { createDecryptionCipher } from '../../encryption'; -import * as utils from '../../utils'; +import { createDecryptionCipher } from '../../../utils/encryption'; +import { collect } from '../../../utils/stream'; type StreamItemArray = Parameters[0]; @@ -83,7 +83,7 @@ class LocalFileSourceProvider implements ISourceProvider { } async getSchemas() { - const schemas = await utils.stream.collect(this.streamSchemas()); + const schemas = await collect(this.streamSchemas()); return keyBy('uid', schemas); } @@ -227,20 +227,22 @@ class LocalFileSourceProvider implements ISourceProvider { return entryPath === filePath && entry.type === 'File'; }, - /** - * Whenever an entry passes the filter method, process it - */ async onentry(entry) { // Collect all the content of the entry file const content = await entry.collect(); - // Parse from buffer to string to JSON - const parsedContent = JSON.parse(content.toString()); - // Resolve the Promise with the parsed content - resolve(parsedContent); + try { + // Parse from buffer to string to JSON + const parsedContent = JSON.parse(content.toString()); - // Cleanup (close the stream associated to the entry) - entry.destroy(); + // Resolve the Promise with the parsed content + resolve(parsedContent); + } catch (e) { + reject(e); + } finally { + // Cleanup (close the stream associated to the entry) + entry.destroy(); + } }, }), ], diff --git a/packages/core/data-transfer/src/index.ts b/packages/core/data-transfer/src/index.ts new file mode 100644 index 0000000000..f6c4c97772 --- /dev/null +++ b/packages/core/data-transfer/src/index.ts @@ -0,0 +1,4 @@ +export * as engine from './engine'; +export * as strapi from './strapi'; +export * as file from './file'; +export * as utils from './utils'; diff --git a/packages/core/data-transfer/src/strapi/__tests__/register.test.ts b/packages/core/data-transfer/src/strapi/__tests__/register.test.ts new file mode 100644 index 0000000000..dc739fb52d --- /dev/null +++ b/packages/core/data-transfer/src/strapi/__tests__/register.test.ts @@ -0,0 +1,36 @@ +import { getStrapiFactory } from '../../__tests__/test-utils'; + +import { createTransferHandler } from '../remote/handlers'; +import register from '../register'; + +afterEach(() => { + jest.clearAllMocks(); +}); + +const strapiMockFactory = getStrapiFactory({ + admin: { + routes: { + push: jest.fn(), + }, + }, +}); + +jest.mock('../remote/handlers', () => ({ + createTransferHandler: jest.fn(), +})); + +describe('Register the Transfer route', () => { + test('registers the /transfer route', () => { + const strapi = strapiMockFactory(); + + register(strapi); + expect(strapi.admin.routes.push).toHaveBeenCalledWith({ + method: 'GET', + path: '/transfer', + handler: createTransferHandler(), + config: { + auth: false, + }, + }); + }); +}); diff --git a/packages/core/data-transfer/src/strapi/index.ts b/packages/core/data-transfer/src/strapi/index.ts new file mode 100644 index 0000000000..5866e4cf11 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/index.ts @@ -0,0 +1,5 @@ +export * as providers from './providers'; +export * as queries from './queries'; +export * as remote from './remote'; + +export { default as register } from './register'; diff --git a/packages/core/data-transfer/src/strapi/providers/index.ts b/packages/core/data-transfer/src/strapi/providers/index.ts new file mode 100644 index 0000000000..4f3e079af7 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/providers/index.ts @@ -0,0 +1,6 @@ +// Local +export * from './local-destination'; +export * from './local-source'; + +// Remote +export * from './remote-destination'; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/assets.test.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts similarity index 94% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/assets.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts index 6851617b5c..33ea37edc4 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/assets.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts @@ -1,8 +1,8 @@ import fse from 'fs-extra'; import { Writable, Readable } from 'stream'; -import type { IAsset } from '../../../../types'; +import type { IAsset } from '../../../../../types'; -import { getStrapiFactory } from '../../../__tests__/test-utils'; +import { getStrapiFactory } from '../../../../__tests__/test-utils'; import { createLocalStrapiDestinationProvider } from '../index'; const write = jest.fn((_chunk, _encoding, callback) => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts similarity index 97% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts index 3e4c8ff88f..1be96c02bf 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts @@ -1,6 +1,10 @@ import { createLocalStrapiDestinationProvider } from '../index'; import * as restoreApi from '../strategies/restore'; -import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils'; +import { + getStrapiFactory, + getContentTypes, + setGlobalStrapi, +} from '../../../../__tests__/test-utils'; afterEach(() => { jest.clearAllMocks(); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts similarity index 95% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts index 71b969c461..0776fc6dcc 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts @@ -1,6 +1,10 @@ import { deleteRecords, restoreConfigs } from '../strategies/restore'; -import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils'; -import { IConfiguration } from '../../../../types'; +import { + getStrapiFactory, + getContentTypes, + setGlobalStrapi, +} from '../../../../__tests__/test-utils'; +import { IConfiguration } from '../../../../../types'; const entities = [ { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts similarity index 91% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/index.ts index 78fe4f26f3..0225b83571 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/index.ts @@ -1,16 +1,17 @@ import { Writable } from 'stream'; import path from 'path'; import * as fse from 'fs-extra'; -import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../types'; +import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../../types'; import { restore } from './strategies'; -import * as utils from '../../utils'; +import * as utils from '../../../utils'; export const VALID_CONFLICT_STRATEGIES = ['restore', 'merge']; export const DEFAULT_CONFLICT_STRATEGY = 'restore'; -interface ILocalStrapiDestinationProviderOptions { +export interface ILocalStrapiDestinationProviderOptions { getStrapi(): Strapi.Strapi | Promise; + autoDestroy?: boolean; restore?: restore.IRestoreOptions; strategy: 'restore' | 'merge'; } @@ -24,6 +25,9 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { strapi?: Strapi.Strapi; + /** + * The entities mapper is used to map old entities to their new IDs + */ #entitiesMapper: { [type: string]: { [id: number]: number } }; constructor(options: ILocalStrapiDestinationProviderOptions) { @@ -37,7 +41,12 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { } async close(): Promise { - await this.strapi?.destroy?.(); + const { autoDestroy } = this.options; + + // Basically `!== false` but more deterministic + if (autoDestroy === undefined || autoDestroy === true) { + await this.strapi?.destroy(); + } } #validateOptions() { @@ -60,7 +69,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { } } - getMetadata(): IMetadata | Promise { + getMetadata(): IMetadata { const strapiVersion = strapi.config.get('info.strapi'); const createdAt = new Date().toISOString(); @@ -117,6 +126,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { throw new Error(`Invalid strategy supplied: "${strategy}"`); } + // TODO: Move this logic to the restore strategy async getAssetsStream(): Promise { if (!this.strapi) { throw new Error('Not able to stream Assets. Strapi instance not found'); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/index.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/index.ts diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/configuration.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts similarity index 95% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/configuration.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts index eae954e619..e1cb04b345 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/configuration.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts @@ -1,6 +1,6 @@ import { Writable } from 'stream'; import chalk from 'chalk'; -import { IConfiguration } from '../../../../../types'; +import { IConfiguration } from '../../../../../../types'; const restoreCoreStore = async (strapi: Strapi.Strapi, data: T) => { return strapi.db.query('strapi::core-store').create({ diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts similarity index 71% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts index c120098599..d3eb397a00 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts @@ -3,9 +3,9 @@ import type { SchemaUID } from '@strapi/strapi/lib/types/utils'; import { get } from 'lodash/fp'; import { Writable } from 'stream'; -import type { IEntity } from '../../../../../types'; -import { json } from '../../../../utils'; -import * as shared from '../../../shared'; +import type { IEntity } from '../../../../../../types'; +import { json } from '../../../../../utils'; +import * as queries from '../../../../queries'; interface IEntitiesRestoreStreamOptions { strapi: Strapi.Strapi; @@ -14,7 +14,7 @@ interface IEntitiesRestoreStreamOptions { const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { const { strapi, updateMappingTable } = options; - const query = shared.strapi.entity.createEntityQuery(strapi); + const query = queries.entity.createEntityQuery(strapi); return new Writable({ objectMode: true, @@ -24,13 +24,19 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { const { create, getDeepPopulateComponentLikeQuery } = query(type); const contentType = strapi.getModel(type); - const resolveType = (paths: string[]) => { + /** + * Resolve the component UID of an entity's attribute based + * on a given path (components & dynamic zones only) + */ + const resolveType = (paths: string[]): string | undefined => { let cType = contentType; let value: unknown = data; for (const path of paths) { value = get(path, value); + // Needed when the value of cType should be computed + // based on the next value (eg: dynamic zones) if (typeof cType === 'function') { cType = cType(value); } @@ -48,24 +54,32 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => { } } - return cType.uid; + return cType?.uid; }; try { - // Create the entity const created = await create({ data, populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }), select: 'id', }); + // Compute differences between original & new entities const diffs = json.diff(data, created); updateMappingTable(type, id, created.id); + // For each difference found on an ID attribute, + // update the mapping the table accordingly diffs.forEach((diff) => { if (diff.kind === 'modified' && diff.path.at(-1) === 'id') { const target = resolveType(diff.path); + + // If no type is found for the given path, then ignore the diff + if (!target) { + return; + } + const [oldID, newID] = diff.values as [number, number]; updateMappingTable(target, oldID, newID); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/index.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts similarity index 96% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts index f325c67ecd..90f9505fdb 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts @@ -1,5 +1,5 @@ import type { ContentTypeSchema } from '@strapi/strapi'; -import * as shared from '../../../shared'; +import * as queries from '../../../../queries'; export interface IRestoreOptions { assets?: boolean; @@ -36,7 +36,7 @@ const deleteEntitiesRecord = async ( options: IRestoreOptions = {} ): Promise => { const { entities } = options; - const query = shared.strapi.entity.createEntityQuery(strapi); + const query = queries.entity.createEntityQuery(strapi); const contentTypes = Object.values(strapi.contentTypes); const contentTypesToClear = contentTypes.filter((contentType) => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/links.ts b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts similarity index 88% rename from packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/links.ts rename to packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts index d4258625a4..89e9b0b695 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/links.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts @@ -1,6 +1,6 @@ import { Writable } from 'stream'; -import { ILink } from '../../../../../types'; -import { createLinkQuery } from '../../../shared/strapi/link'; +import { ILink } from '../../../../../../types'; +import { createLinkQuery } from '../../../../queries/link'; export const createLinksWriteStream = ( mapID: (uid: string, id: number) => number | undefined, diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/configuration.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts similarity index 91% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/configuration.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts index 088d92471b..225a886b28 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/configuration.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts @@ -1,6 +1,10 @@ import { Readable } from 'stream'; -import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils'; +import { + collect, + createMockedQueryBuilder, + getStrapiFactory, +} from '../../../../__tests__/test-utils'; import { createConfigurationStream } from '../configuration'; describe('Configuration', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts index ef4c29c15b..e29ab5e4f2 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts @@ -1,12 +1,12 @@ import { Readable, PassThrough } from 'stream'; -import type { IEntity } from '../../../../types'; +import type { IEntity } from '../../../../../types'; import { collect, getStrapiFactory, getContentTypes, createMockedQueryBuilder, -} from '../../../__tests__/test-utils'; +} from '../../../../__tests__/test-utils'; import { createEntitiesStream, createEntitiesTransformStream } from '../entities'; describe('Local Strapi Source Provider - Entities Streaming', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts similarity index 97% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts index 342c5a9ba2..77ecb8e9db 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts @@ -1,7 +1,11 @@ import { Readable } from 'stream'; -import type { IEntity } from '../../../../types'; +import type { IEntity } from '../../../../../types'; -import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils'; +import { + collect, + createMockedQueryBuilder, + getStrapiFactory, +} from '../../../../__tests__/test-utils'; import { createLocalStrapiSourceProvider } from '..'; describe('Local Strapi Source Provider', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts similarity index 99% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts index be6420d615..5c4aec8f70 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts @@ -1,7 +1,7 @@ import { Readable } from 'stream'; import { createLinksStream } from '../links'; -import { collect, getStrapiFactory } from '../../../__tests__/test-utils'; +import { collect, getStrapiFactory } from '../../../../__tests__/test-utils'; // TODO: entityService needs to be replaced with a mocked wrapper of db.connection and provide real metadata describe.skip('Local Strapi Source Provider - Entities Streaming', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/assets.ts b/packages/core/data-transfer/src/strapi/providers/local-source/assets.ts similarity index 95% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/assets.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/assets.ts index 3edc89aaba..8cdbf67699 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/assets.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/assets.ts @@ -2,7 +2,7 @@ import { join } from 'path'; import { readdir, stat, createReadStream } from 'fs-extra'; import { Duplex } from 'stream'; -import type { IAsset } from '../../../types'; +import type { IAsset } from '../../../../types'; const IGNORED_FILES = ['.gitkeep']; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts b/packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts similarity index 54% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts index af9e3161b7..9340d98ded 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts @@ -2,30 +2,29 @@ import { chain } from 'stream-chain'; import { Readable } from 'stream'; import { set } from 'lodash/fp'; -import type { IConfiguration } from '../../../types'; +import type { IConfiguration } from '../../../../types'; /** * Create a readable stream that export the Strapi app configuration */ export const createConfigurationStream = (strapi: Strapi.Strapi): Readable => { - // Core Store - const coreStoreStream = chain([ - strapi.db.queryBuilder('strapi::core-store').stream(), - (data) => set('value', JSON.parse(data.value), data), - wrapConfigurationItem('core-store'), - ]); - - // Webhook - const webhooksStream = chain([ - strapi.db.queryBuilder('webhook').stream(), - wrapConfigurationItem('webhook'), - ]); - - const streams = [coreStoreStream, webhooksStream]; - - // Readable configuration stream return Readable.from( (async function* configurationGenerator(): AsyncGenerator { + // Core Store + const coreStoreStream = chain([ + strapi.db.queryBuilder('strapi::core-store').stream(), + (data) => set('value', JSON.parse(data.value), data), + wrapConfigurationItem('core-store'), + ]); + + // Webhook + const webhooksStream = chain([ + strapi.db.queryBuilder('webhook').stream(), + wrapConfigurationItem('webhook'), + ]); + + const streams = [coreStoreStream, webhooksStream]; + for (const stream of streams) { for await (const item of stream) { yield item; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts b/packages/core/data-transfer/src/strapi/providers/local-source/entities.ts similarity index 85% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/entities.ts index 8f45069d09..7aba49f26f 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/entities.ts @@ -1,8 +1,8 @@ import type { ContentTypeSchema } from '@strapi/strapi'; import { Readable, PassThrough } from 'stream'; -import * as shared from '../shared/strapi'; -import { IEntity } from '../../../types'; +import * as shared from '../../queries'; +import { IEntity } from '../../../../types'; /** * Generate and consume content-types streams in order to stream each entity individually @@ -34,11 +34,15 @@ export const createEntitiesStream = (strapi: Strapi.Strapi): Readable => { contentType: ContentTypeSchema; }> { for await (const { stream, contentType } of contentTypeStreamGenerator()) { - for await (const entity of stream) { - yield { entity, contentType }; + try { + for await (const entity of stream) { + yield { entity, contentType }; + } + } catch { + // ignore + } finally { + stream.destroy(); } - - stream.destroy(); } })() ); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts b/packages/core/data-transfer/src/strapi/providers/local-source/index.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/index.ts index 181eed4c25..f8cfa75232 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/index.ts @@ -1,12 +1,12 @@ import { chain } from 'stream-chain'; import { Readable } from 'stream'; -import type { IMetadata, ISourceProvider, ProviderType } from '../../../types'; +import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types'; import { createEntitiesStream, createEntitiesTransformStream } from './entities'; import { createLinksStream } from './links'; import { createConfigurationStream } from './configuration'; import { createAssetsStream } from './assets'; -import * as utils from '../../utils'; +import * as utils from '../../../utils'; export interface ILocalStrapiSourceProviderOptions { getStrapi(): Strapi.Strapi | Promise; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links.ts b/packages/core/data-transfer/src/strapi/providers/local-source/links.ts similarity index 87% rename from packages/core/data-transfer/lib/providers/local-strapi-source-provider/links.ts rename to packages/core/data-transfer/src/strapi/providers/local-source/links.ts index 5387c45002..91e4d3fd96 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links.ts +++ b/packages/core/data-transfer/src/strapi/providers/local-source/links.ts @@ -1,7 +1,7 @@ import { Readable } from 'stream'; -import type { ILink } from '../../../types'; -import { createLinkQuery } from '../shared/strapi/link'; +import type { ILink } from '../../../../types'; +import { createLinkQuery } from '../../queries/link'; /** * Create a Readable which will stream all the links from a Strapi instance diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts new file mode 100644 index 0000000000..24dbce0012 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts @@ -0,0 +1,54 @@ +import { WebSocket } from 'ws'; +import type { IRemoteStrapiDestinationProviderOptions } from '..'; + +import { createRemoteStrapiDestinationProvider } from '..'; + +const defaultOptions: IRemoteStrapiDestinationProviderOptions = { + strategy: 'restore', + url: '', +}; + +jest.mock('../utils', () => ({ + createDispatcher: jest.fn(), +})); + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn((type, callback) => { + callback(); + return { + once: jest.fn((t, c) => c), + }; + }), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination', () => { + describe('Bootstrap', () => { + test('Should not have a defined websocket connection if bootstrap has not been called', () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + + expect(provider.ws).toBeNull(); + }); + + test('Should have a defined websocket connection if bootstrap has been called', async () => { + const provider = createRemoteStrapiDestinationProvider(defaultOptions); + try { + await provider.bootstrap(); + } catch { + // ignore ws connection error + } + + expect(provider.ws).not.toBeNull(); + expect(provider.ws?.readyState).toBe(WebSocket.CLOSED); + }); + }); +}); diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts new file mode 100644 index 0000000000..8c907c756b --- /dev/null +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts @@ -0,0 +1,45 @@ +import { WebSocket } from 'ws'; +import { CommandMessage } from '../../../../../types/remote/protocol/client'; +import { createDispatcher } from '../utils'; + +jest.mock('ws', () => ({ + WebSocket: jest.fn().mockImplementation(() => { + return { + ...jest.requireActual('ws').WebSocket, + send: jest.fn(), + once: jest.fn(), + }; + }), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe('Remote Strapi Destination Utils', () => { + test('Dispatch method sends payload', () => { + const ws = new WebSocket('ws://test/admin/transfer'); + const message: CommandMessage = { + type: 'command', + command: 'status', + }; + + createDispatcher(ws).dispatch(message); + + expect.extend({ + toContain(receivedString, expected) { + const jsonReceived = JSON.parse(receivedString); + const pass = Object.keys(expected).every((key) => jsonReceived[key] === expected[key]); + + return { + message: () => + `Expected ${jsonReceived} ${!pass && 'not'} to contain properties ${expected}`, + pass, + }; + }, + }); + + // @ts-ignore + expect(ws.send).toHaveBeenCalledWith(expect.toContain(message), expect.anything()); + }); +}); diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts new file mode 100644 index 0000000000..7c88c84b96 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts @@ -0,0 +1,236 @@ +import { WebSocket } from 'ws'; +import { v4 } from 'uuid'; +import { Writable } from 'stream'; + +import { createDispatcher } from './utils'; + +import type { + IDestinationProvider, + IEntity, + ILink, + IMetadata, + ProviderType, + IConfiguration, + IAsset, +} from '../../../../types'; +import type { client, server } from '../../../../types/remote/protocol'; +import type { ILocalStrapiDestinationProviderOptions } from '../local-destination'; + +interface ITokenAuth { + type: 'token'; + token: string; +} + +interface ICredentialsAuth { + type: 'credentials'; + email: string; + password: string; +} + +export interface IRemoteStrapiDestinationProviderOptions + extends Pick { + url: string; + auth?: ITokenAuth | ICredentialsAuth; +} + +class RemoteStrapiDestinationProvider implements IDestinationProvider { + name = 'destination::remote-strapi'; + + type: ProviderType = 'destination'; + + options: IRemoteStrapiDestinationProviderOptions; + + ws: WebSocket | null; + + dispatcher: ReturnType | null; + + constructor(options: IRemoteStrapiDestinationProviderOptions) { + this.options = options; + this.ws = null; + this.dispatcher = null; + } + + async initTransfer(): Promise { + const { strategy, restore } = this.options; + + // Wait for the connection to be made to the server, then init the transfer + return new Promise((resolve, reject) => { + this.ws + ?.once('open', async () => { + const query = this.dispatcher?.dispatchCommand({ + command: 'init', + params: { options: { strategy, restore }, transfer: 'push' }, + }); + + const res = (await query) as server.Payload; + + if (!res?.transferID) { + return reject(new Error('Init failed, invalid response from the server')); + } + + resolve(res.transferID); + }) + .once('error', reject); + }); + } + + async #streamStep( + step: T, + data: client.GetTransferPushStreamData + ) { + try { + await this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data }); + } catch (e) { + if (e instanceof Error) { + return e; + } + + if (typeof e === 'string') { + return new Error(e); + } + + return new Error('Unexpected error'); + } + + return null; + } + + async bootstrap(): Promise { + const { url, auth } = this.options; + + let ws: WebSocket; + + // No auth defined, trying public access for transfer + if (!auth) { + ws = new WebSocket(url); + } + + // Common token auth, this should be the main auth method + else if (auth.type === 'token') { + const headers = { Authentication: `Bearer ${auth.token}` }; + + ws = new WebSocket(this.options.url, { headers }); + } + + // Invalid auth method provided + else { + throw new Error('Auth method not implemented'); + } + + this.ws = ws; + this.dispatcher = createDispatcher(this.ws); + + const transferID = await this.initTransfer(); + + this.dispatcher.setTransferProperties({ id: transferID, kind: 'push' }); + + await this.dispatcher.dispatchTransferAction('bootstrap'); + } + + async close() { + await this.dispatcher?.dispatchTransferAction('close'); + + await new Promise((resolve) => { + const { ws } = this; + + if (!ws || ws.CLOSED) { + resolve(); + return; + } + + ws.on('close', () => resolve()).close(); + }); + } + + getMetadata() { + return this.dispatcher?.dispatchTransferAction('getMetadata') ?? null; + } + + async beforeTransfer() { + await this.dispatcher?.dispatchTransferAction('beforeTransfer'); + } + + getSchemas(): Promise { + if (!this.dispatcher) { + return Promise.resolve(null); + } + + return this.dispatcher.dispatchTransferAction('getSchemas'); + } + + getEntitiesStream(): Writable { + return new Writable({ + objectMode: true, + write: async (entity: IEntity, _encoding, callback) => { + const e = await this.#streamStep('entities', entity); + + callback(e); + }, + }); + } + + getLinksStream(): Writable { + return new Writable({ + objectMode: true, + write: async (link: ILink, _encoding, callback) => { + const e = await this.#streamStep('links', link); + + callback(e); + }, + }); + } + + getConfigurationStream(): Writable { + return new Writable({ + objectMode: true, + write: async (configuration: IConfiguration, _encoding, callback) => { + const e = await this.#streamStep('configuration', configuration); + + callback(e); + }, + }); + } + + getAssetsStream(): Writable | Promise { + return new Writable({ + objectMode: true, + final: async (callback) => { + // TODO: replace this stream call by an end call + const e = await this.#streamStep('assets', null); + + callback(e); + }, + write: async (asset: IAsset, _encoding, callback) => { + const { filename, filepath, stats, stream } = asset; + const assetID = v4(); + + await this.#streamStep('assets', { + action: 'start', + assetID, + data: { filename, filepath, stats }, + }); + + for await (const chunk of stream) { + await this.#streamStep('assets', { + action: 'stream', + assetID, + data: chunk, + }); + } + + await this.#streamStep('assets', { + action: 'end', + assetID, + }); + + callback(); + }, + }); + } +} + +export const createRemoteStrapiDestinationProvider = ( + options: IRemoteStrapiDestinationProviderOptions +) => { + return new RemoteStrapiDestinationProvider(options); +}; diff --git a/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts b/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts new file mode 100644 index 0000000000..0a1931cb4e --- /dev/null +++ b/packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts @@ -0,0 +1,123 @@ +import { set } from 'lodash/fp'; +import { v4 } from 'uuid'; +import { RawData, WebSocket } from 'ws'; + +import type { client, server } from '../../../../types/remote/protocol'; + +interface IDispatcherState { + transfer?: { kind: client.TransferKind; id: string }; +} + +interface IDispatchOptions { + attachTransfer?: boolean; +} + +type Dispatch = Omit; + +const createDispatcher = (ws: WebSocket) => { + const state: IDispatcherState = {}; + + type DispatchMessage = Dispatch; + + const dispatch = async ( + message: DispatchMessage, + options: IDispatchOptions = {} + ): Promise => { + if (!ws) { + throw new Error('No websocket connection found'); + } + + return new Promise((resolve, reject) => { + const uuid = v4(); + const payload = { ...message, uuid }; + + if (options.attachTransfer) { + Object.assign(payload, { transferID: state.transfer?.id }); + } + + const stringifiedPayload = JSON.stringify(payload); + + ws.send(stringifiedPayload, (error) => { + if (error) { + reject(error); + } + }); + + const onResponse = (raw: RawData) => { + const response: server.Message = JSON.parse(raw.toString()); + if (response.uuid === uuid) { + if (response.error) { + return reject(new Error(response.error.message)); + } + + resolve(response.data ?? null); + } else { + ws.once('message', onResponse); + } + }; + + // TODO: What happens if the server sends another message (not a response to this message) + ws.once('message', onResponse); + }); + }; + + const dispatchCommand = ( + payload: { + command: U; + } & ([client.GetCommandParams] extends [never] + ? unknown + : { params: client.GetCommandParams }) + ) => { + return dispatch({ type: 'command', ...payload } as client.CommandMessage); + }; + + const dispatchTransferAction = async (action: client.Action['action']) => { + const payload: Dispatch = { type: 'transfer', kind: 'action', action }; + + return dispatch(payload, { attachTransfer: true }) ?? Promise.resolve(null); + }; + + const dispatchTransferStep = async < + T, + A extends client.TransferPushMessage['action'] = client.TransferPushMessage['action'], + S extends client.TransferPushStep = client.TransferPushStep + >( + payload: { + step: S; + action: A; + } & (A extends 'stream' ? { data: client.GetTransferPushStreamData } : unknown) + ) => { + const message: Dispatch = { + type: 'transfer', + kind: 'step', + ...payload, + }; + + return dispatch(message, { attachTransfer: true }) ?? Promise.resolve(null); + }; + + const setTransferProperties = ( + properties: Exclude + ): void => { + state.transfer = { ...properties }; + }; + + return { + get transferID() { + return state.transfer?.id; + }, + + get transferKind() { + return state.transfer?.kind; + }, + + setTransferProperties, + + dispatch, + dispatchCommand, + dispatchTransferAction, + dispatchTransferStep, + }; +}; + +export { createDispatcher }; diff --git a/packages/core/data-transfer/lib/providers/shared/strapi/entity.ts b/packages/core/data-transfer/src/strapi/queries/entity.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/shared/strapi/entity.ts rename to packages/core/data-transfer/src/strapi/queries/entity.ts diff --git a/packages/core/data-transfer/lib/providers/shared/strapi/index.ts b/packages/core/data-transfer/src/strapi/queries/index.ts similarity index 100% rename from packages/core/data-transfer/lib/providers/shared/strapi/index.ts rename to packages/core/data-transfer/src/strapi/queries/index.ts diff --git a/packages/core/data-transfer/lib/providers/shared/strapi/link.ts b/packages/core/data-transfer/src/strapi/queries/link.ts similarity index 98% rename from packages/core/data-transfer/lib/providers/shared/strapi/link.ts rename to packages/core/data-transfer/src/strapi/queries/link.ts index 55855a4667..ca6f7be345 100644 --- a/packages/core/data-transfer/lib/providers/shared/strapi/link.ts +++ b/packages/core/data-transfer/src/strapi/queries/link.ts @@ -1,5 +1,6 @@ +import { RelationAttribute } from '@strapi/strapi'; import { clone, isNil } from 'lodash/fp'; -import { ILink } from '../../../../types'; +import { ILink } from '../../../types'; // TODO: Remove any types when we'll have types for DB metadata diff --git a/packages/core/data-transfer/src/strapi/register.ts b/packages/core/data-transfer/src/strapi/register.ts new file mode 100644 index 0000000000..0217733e4f --- /dev/null +++ b/packages/core/data-transfer/src/strapi/register.ts @@ -0,0 +1,12 @@ +import { routes } from './remote'; + +/** + * This is intended to be called on Strapi register phase. + * + * It registers a transfer route in the Strapi admin router. + */ +const register = (strapi: Strapi.Strapi) => { + routes.registerAdminTransferRoute(strapi); +}; + +export default register; diff --git a/packages/core/data-transfer/src/strapi/remote/constants.ts b/packages/core/data-transfer/src/strapi/remote/constants.ts new file mode 100644 index 0000000000..6f913eeba6 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/constants.ts @@ -0,0 +1 @@ +export const TRANSFER_URL = '/transfer'; diff --git a/packages/core/data-transfer/src/strapi/remote/controllers/index.ts b/packages/core/data-transfer/src/strapi/remote/controllers/index.ts new file mode 100644 index 0000000000..553009a4ea --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/controllers/index.ts @@ -0,0 +1 @@ +export * from './push'; diff --git a/packages/core/data-transfer/src/strapi/remote/controllers/push.ts b/packages/core/data-transfer/src/strapi/remote/controllers/push.ts new file mode 100644 index 0000000000..3a2487a4d4 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/controllers/push.ts @@ -0,0 +1,142 @@ +import { PassThrough, Writable } from 'stream-chain'; + +import type { IAsset, IMetadata } from '../../../../types'; +import type { + TransferPushMessage, + TransferPushStep, +} from '../../../../types/remote/protocol/client'; +import { + createLocalStrapiDestinationProvider, + ILocalStrapiDestinationProviderOptions, +} from '../../providers'; + +export interface IPushController { + streams: { [stage in TransferPushStep]?: Writable }; + actions: { + getMetadata(): Promise; + getSchemas(): Strapi.Schemas; + bootstrap(): Promise; + close(): Promise; + beforeTransfer(): Promise; + }; + transfer: { + [key in TransferPushStep]: ( + value: T extends { step: key; data: infer U } ? U : never + ) => Promise; + }; +} + +const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => { + const provider = createLocalStrapiDestinationProvider(options); + + const streams: { [stage in TransferPushStep]?: Writable } = {}; + const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {}; + + const writeAsync = (stream: Writable, data: T) => { + return new Promise((resolve, reject) => { + stream.write(data, (error) => { + if (error) { + reject(error); + } + + resolve(); + }); + }); + }; + + return { + streams, + + actions: { + async getSchemas(): Promise { + return provider.getSchemas(); + }, + + async getMetadata() { + return provider.getMetadata(); + }, + + async bootstrap() { + return provider.bootstrap(); + }, + + async close() { + return provider.close(); + }, + + async beforeTransfer() { + return provider.beforeTransfer(); + }, + }, + + transfer: { + async entities(entity) { + if (!streams.entities) { + streams.entities = provider.getEntitiesStream(); + } + + await writeAsync(streams.entities, entity); + }, + + async links(link) { + if (!streams.links) { + streams.links = await provider.getLinksStream(); + } + + await writeAsync(streams.links, link); + }, + + async configuration(config) { + if (!streams.configuration) { + streams.configuration = await provider.getConfigurationStream(); + } + + await writeAsync(streams.configuration, config); + }, + + async assets(payload) { + // TODO: close the stream upong receiving an 'end' event instead + if (payload === null) { + streams.assets?.end(); + return; + } + + const { action, assetID } = payload; + + if (!streams.assets) { + streams.assets = await provider.getAssetsStream(); + } + + if (action === 'start') { + assets[assetID] = { ...payload.data, stream: new PassThrough() }; + writeAsync(streams.assets, assets[assetID]); + } + + if (action === 'stream') { + // The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array } + // We need to transform it back into a Buffer instance + const rawBuffer = payload.data as unknown as { type: 'Buffer'; data: Uint8Array }; + const chunk = Buffer.from(rawBuffer.data); + + await writeAsync(assets[assetID].stream, chunk); + } + + if (action === 'end') { + await new Promise((resolve, reject) => { + const { stream } = assets[assetID]; + + stream + .on('close', () => { + delete assets[assetID]; + resolve(); + }) + .on('error', reject) + .end(); + }); + } + }, + }, + }; +}; + +export default createPushController; diff --git a/packages/core/data-transfer/src/strapi/remote/handlers.ts b/packages/core/data-transfer/src/strapi/remote/handlers.ts new file mode 100644 index 0000000000..f6c9315b21 --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/handlers.ts @@ -0,0 +1,221 @@ +import type { Context } from 'koa'; +import type { ServerOptions } from 'ws'; + +import { randomUUID } from 'crypto'; +import { WebSocket } from 'ws'; + +import type { IPushController } from './controllers/push'; + +import createPushController from './controllers/push'; +import type { client, server } from '../../../types/remote/protocol'; + +interface ITransferState { + transfer?: { id: string; kind: client.TransferKind }; + controller?: IPushController; +} + +export const createTransferHandler = + (options: ServerOptions = {}) => + async (ctx: Context) => { + const upgradeHeader = (ctx.request.headers.upgrade || '') + .split(',') + .map((s) => s.trim().toLowerCase()); + + // Create the websocket server + const wss = new WebSocket.Server({ ...options, noServer: true }); + + if (upgradeHeader.includes('websocket')) { + wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => { + // Create a connection between the client & the server + wss.emit('connection', ws, ctx.req); + + const state: ITransferState = {}; + let uuid: string | undefined; + + /** + * Format error & message to follow the remote transfer protocol + */ + const callback = (e: Error | null = null, data?: T) => { + return new Promise((resolve, reject) => { + if (!uuid) { + reject(new Error('Missing uuid for this message')); + return; + } + + const payload = JSON.stringify({ + uuid, + data: data ?? null, + error: e + ? { + code: 'ERR', + message: e?.message, + } + : null, + }); + + ws.send(payload, (error) => (error ? reject(error) : resolve())); + }); + }; + + /** + * Wrap a function call to catch errors and answer the request with the correct format + */ + const answer = async (fn: () => T) => { + try { + const response = await fn(); + callback(null, response); + } catch (e) { + if (e instanceof Error) { + callback(e); + } else if (typeof e === 'string') { + callback(new Error(e)); + } else { + callback(new Error('Unexpected error')); + } + } + }; + + const teardown = (): server.Payload => { + delete state.controller; + delete state.transfer; + + return { ok: true }; + }; + + const init = (msg: client.InitCommand): server.Payload => { + if (state.controller) { + throw new Error('Transfer already in progres'); + } + + const { transfer } = msg.params; + + // Push transfer + if (transfer === 'push') { + const { options: controllerOptions } = msg.params; + + state.controller = createPushController({ + ...controllerOptions, + autoDestroy: false, + getStrapi: () => strapi, + }); + } + + // Pull or any other string + else { + throw new Error(`Transfer not implemented: "${transfer}"`); + } + + state.transfer = { id: randomUUID(), kind: transfer }; + + return { transferID: state.transfer.id }; + }; + + /** + * On command message (init, end, status, ...) + */ + const onCommand = async (msg: client.CommandMessage) => { + const { command } = msg; + + if (command === 'init') { + await answer(() => init(msg)); + } + + if (command === 'end') { + await answer(teardown); + } + + if (command === 'status') { + await callback(new Error('Command not implemented: "status"')); + } + }; + + const onTransferCommand = async (msg: client.TransferMessage) => { + const { transferID, kind } = msg; + const { controller } = state; + + // TODO: (re)move this check + // It shouldn't be possible to strart a pull transfer for now, so reaching + // this code should be impossible too, but this has been added by security + if (state.transfer?.kind === 'pull') { + return callback(new Error('Pull transfer not implemented')); + } + + if (!controller) { + return callback(new Error("The transfer hasn't been initialized")); + } + + if (!transferID) { + return callback(new Error('Missing transfer ID')); + } + + // Action + if (kind === 'action') { + const { action } = msg; + + if (!(action in controller.actions)) { + return callback(new Error(`Invalid action provided: "${action}"`)); + } + + await answer(() => controller.actions[action as keyof typeof controller.actions]()); + } + + // Transfer + else if (kind === 'step') { + // We can only have push transfer message for the moment + const message = msg as client.TransferPushMessage; + + // TODO: lock transfer process + if (message.action === 'start') { + // console.log('Starting transfer for ', message.step); + } + + // Stream step + else if (message.action === 'stream') { + await answer(() => controller.transfer[message.step]?.(message.data as never)); + } + + // TODO: unlock transfer process + else if (message.action === 'end') { + // console.log('Ending transfer for ', message.step); + } + } + }; + + ws.on('close', () => { + teardown(); + }); + + ws.on('error', (e) => { + teardown(); + console.error(e); + }); + + ws.on('message', async (raw) => { + const msg: client.Message = JSON.parse(raw.toString()); + + if (!msg.uuid) { + throw new Error('Missing uuid in message'); + } + + uuid = msg.uuid; + + // Regular command message (init, end, status) + if (msg.type === 'command') { + await onCommand(msg); + } + + // Transfer message (the transfer must be initialized first) + else if (msg.type === 'transfer') { + await onTransferCommand(msg); + } + + // Invalid messages + else { + await callback(new Error('Bad request')); + } + }); + }); + + ctx.respond = false; + } + }; diff --git a/packages/core/data-transfer/src/strapi/remote/index.ts b/packages/core/data-transfer/src/strapi/remote/index.ts new file mode 100644 index 0000000000..719e07906a --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/index.ts @@ -0,0 +1,3 @@ +export * as controllers from './controllers'; +export * as routes from './routes'; +export * as constants from './constants'; diff --git a/packages/core/data-transfer/src/strapi/remote/routes.ts b/packages/core/data-transfer/src/strapi/remote/routes.ts new file mode 100644 index 0000000000..746bed09bb --- /dev/null +++ b/packages/core/data-transfer/src/strapi/remote/routes.ts @@ -0,0 +1,36 @@ +// eslint-disable-next-line node/no-extraneous-import +import type { Context } from 'koa'; + +import { TRANSFER_URL } from './constants'; +import { createTransferHandler } from './handlers'; + +// Extend Strapi interface type to access the admin routes' API +// TODO: Remove this when the Strapi instances will be better typed +declare module '@strapi/strapi' { + interface Strapi { + admin: { + routes: { + method: string; + path: string; + handler: (ctx: Context) => Promise; + config: unknown; + }[]; + }; + } +} + +/** + * Register a transfer route in the Strapi admin router. + * + * It exposes a WS server that can be used to run and manage transfer processes. + * + * @param strapi - A Strapi instance + */ +export const registerAdminTransferRoute = (strapi: Strapi.Strapi) => { + strapi.admin.routes.push({ + method: 'GET', + path: TRANSFER_URL, + handler: createTransferHandler(), + config: { auth: false }, + }); +}; diff --git a/packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts b/packages/core/data-transfer/src/utils/encryption/__tests__/encrypt.test.ts similarity index 100% rename from packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts rename to packages/core/data-transfer/src/utils/encryption/__tests__/encrypt.test.ts diff --git a/packages/core/data-transfer/lib/encryption/decrypt.ts b/packages/core/data-transfer/src/utils/encryption/decrypt.ts similarity index 86% rename from packages/core/data-transfer/lib/encryption/decrypt.ts rename to packages/core/data-transfer/src/utils/encryption/decrypt.ts index e7a8e4c7a2..3687e9af43 100644 --- a/packages/core/data-transfer/lib/encryption/decrypt.ts +++ b/packages/core/data-transfer/src/utils/encryption/decrypt.ts @@ -1,5 +1,5 @@ import { Cipher, scryptSync, CipherKey, BinaryLike, createDecipheriv } from 'crypto'; -import { EncryptionStrategy, Strategies, Algorithm } from '../../types'; +import { EncryptionStrategy, Strategies, Algorithm } from '../../../types'; // different key values depending on algorithm chosen const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { @@ -33,6 +33,14 @@ const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { return strategies[algorithm]; }; +/** + * It creates a cipher instance used for decryption + * + * @param key - The decryption key + * @param algorithm - The algorithm to use to create the Cipher + * + * @returns A {@link Cipher} instance created with the given key & algorithm + */ export const createDecryptionCipher = ( key: string, algorithm: Algorithm = 'aes-128-ecb' diff --git a/packages/core/data-transfer/lib/encryption/encrypt.ts b/packages/core/data-transfer/src/utils/encryption/encrypt.ts similarity index 86% rename from packages/core/data-transfer/lib/encryption/encrypt.ts rename to packages/core/data-transfer/src/utils/encryption/encrypt.ts index d32e2cd6e9..5fea673d76 100644 --- a/packages/core/data-transfer/lib/encryption/encrypt.ts +++ b/packages/core/data-transfer/src/utils/encryption/encrypt.ts @@ -1,5 +1,5 @@ import { createCipheriv, Cipher, scryptSync, CipherKey, BinaryLike } from 'crypto'; -import { EncryptionStrategy, Strategies, Algorithm } from '../../types'; +import { EncryptionStrategy, Strategies, Algorithm } from '../../../types'; // different key values depending on algorithm chosen const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { @@ -33,6 +33,14 @@ const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { return strategies[algorithm]; }; +/** + * It creates a cipher instance used for encryption + * + * @param key - The encryption key + * @param algorithm - The algorithm to use to create the Cipher + * + * @returns A {@link Cipher} instance created with the given key & algorithm + */ export const createEncryptionCipher = ( key: string, algorithm: Algorithm = 'aes-128-ecb' diff --git a/packages/core/data-transfer/lib/encryption/index.ts b/packages/core/data-transfer/src/utils/encryption/index.ts similarity index 100% rename from packages/core/data-transfer/lib/encryption/index.ts rename to packages/core/data-transfer/src/utils/encryption/index.ts diff --git a/packages/core/data-transfer/lib/utils/index.ts b/packages/core/data-transfer/src/utils/index.ts similarity index 70% rename from packages/core/data-transfer/lib/utils/index.ts rename to packages/core/data-transfer/src/utils/index.ts index 069e3aae93..0203b50860 100644 --- a/packages/core/data-transfer/lib/utils/index.ts +++ b/packages/core/data-transfer/src/utils/index.ts @@ -1,3 +1,4 @@ +export * as encryption from './encryption'; export * as stream from './stream'; export * as json from './json'; export * as schema from './schema'; diff --git a/packages/core/data-transfer/lib/utils/json.ts b/packages/core/data-transfer/src/utils/json.ts similarity index 89% rename from packages/core/data-transfer/lib/utils/json.ts rename to packages/core/data-transfer/src/utils/json.ts index ee14f29c6b..d305ee1cab 100644 --- a/packages/core/data-transfer/lib/utils/json.ts +++ b/packages/core/data-transfer/src/utils/json.ts @@ -2,6 +2,13 @@ import { isArray, isObject, zip, isEqual, uniq } from 'lodash/fp'; const createContext = (): Context => ({ path: [] }); +/** + * Compute differences between two JSON objects and returns them + * + * @param a - First object + * @param b - Second object + * @param ctx - Context used to keep track of the current path during recursion + */ export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Diff[] => { const diffs: Diff[] = []; const { path } = ctx; @@ -70,7 +77,7 @@ export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Di } if (!isEqual(a, b)) { - modified(); + return modified(); } return diffs; diff --git a/packages/core/data-transfer/src/utils/schema.ts b/packages/core/data-transfer/src/utils/schema.ts new file mode 100644 index 0000000000..1ab4c1dfde --- /dev/null +++ b/packages/core/data-transfer/src/utils/schema.ts @@ -0,0 +1,27 @@ +import type { Schema } from '@strapi/strapi'; +import { mapValues, pick } from 'lodash/fp'; + +/** + * List of schema properties that should be kept when sanitizing schemas + */ +const VALID_SCHEMA_PROPERTIES = [ + 'collectionName', + 'info', + 'options', + 'pluginOptions', + 'attributes', + 'kind', + 'modelType', + 'modelName', + 'uid', + 'plugin', + 'globalId', +]; + +/** + * Sanitize a schemas dictionnary by omiting unwanted properties + * The list of allowed properties can be found here: {@link VALID_SCHEMA_PROPERTIES} + */ +export const mapSchemasValues = (schemas: Record) => { + return mapValues(pick(VALID_SCHEMA_PROPERTIES), schemas); +}; diff --git a/packages/core/data-transfer/src/utils/stream.ts b/packages/core/data-transfer/src/utils/stream.ts new file mode 100644 index 0000000000..1715f31122 --- /dev/null +++ b/packages/core/data-transfer/src/utils/stream.ts @@ -0,0 +1,72 @@ +import { Transform, Readable } from 'stream'; + +type TransformOptions = ConstructorParameters[0]; + +/** + * Create a filter stream that discard chunks which doesn't satisfies the given predicate + * + * @param predicate - A filter predicate, takes a stream data chunk as parameter and returns a boolean value + * @param options - Transform stream options + */ +export const filter = ( + predicate: (value: T) => boolean | Promise, + options: TransformOptions = { objectMode: true } +): Transform => { + return new Transform({ + ...options, + + async transform(chunk, _encoding, callback) { + const keep = await predicate(chunk); + + callback(null, keep ? chunk : undefined); + }, + }); +}; + +/** + * Create a map stream that transform chunks using the given predicate + * + * @param predicate - A map predicate, takes a stream data chunk as parameter and returns a mapped value + * @param options - Transform stream options + */ +export const map = ( + predicate: (value: T) => U | Promise, + options: TransformOptions = { objectMode: true } +): Transform => { + return new Transform({ + ...options, + + async transform(chunk, _encoding, callback) { + const mappedValue = await predicate(chunk); + + callback(null, mappedValue); + }, + }); +}; + +/** + * Collect every chunks from a Readable stream. + * + * @param stream - The redable stream to collect data from + * @param options.destroy - If set to true, it automatically calls `destroy()` on the given stream upon receiving the 'end' event + */ +export const collect = ( + stream: Readable, + options: { destroy: boolean } = { destroy: true } +): Promise => { + const chunks: T[] = []; + + return new Promise((resolve, reject) => { + stream + .on('close', () => resolve(chunks)) + .on('error', reject) + .on('data', (chunk) => chunks.push(chunk)) + .on('end', () => { + if (options.destroy) { + stream.destroy(); + } + + resolve(chunks); + }); + }); +}; diff --git a/packages/core/data-transfer/tsconfig.json b/packages/core/data-transfer/tsconfig.json index ff65f30217..cf0ec9c76a 100644 --- a/packages/core/data-transfer/tsconfig.json +++ b/packages/core/data-transfer/tsconfig.json @@ -4,11 +4,11 @@ "strict": true, "lib": ["ESNEXT"], "skipLibCheck": true, - "outDir": "dist", + "outDir": "lib", "declaration": true, "sourceMap": true, "esModuleInterop": true }, - "include": ["types", "lib/**/*.ts"], - "exclude": ["node_modules", "lib/**/__tests__"] + "include": ["types", "src/**/*.ts"], + "exclude": ["node_modules", "src/**/__tests__"] } diff --git a/packages/core/data-transfer/types/index.d.ts b/packages/core/data-transfer/types/index.d.ts index 73d649f130..51f3087de4 100644 --- a/packages/core/data-transfer/types/index.d.ts +++ b/packages/core/data-transfer/types/index.d.ts @@ -3,3 +3,4 @@ export * from './providers'; export * from './transfer-engine'; export * from './utils'; export * from './encryption'; +export * from './remote'; diff --git a/packages/core/data-transfer/types/remote/index.d.ts b/packages/core/data-transfer/types/remote/index.d.ts new file mode 100644 index 0000000000..24b40f78af --- /dev/null +++ b/packages/core/data-transfer/types/remote/index.d.ts @@ -0,0 +1 @@ +export * as protocol from './protocol'; diff --git a/packages/core/data-transfer/types/remote/protocol/client/commands.d.ts b/packages/core/data-transfer/types/remote/protocol/client/commands.d.ts new file mode 100644 index 0000000000..af0153dc4d --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/commands.d.ts @@ -0,0 +1,30 @@ +import type { ILocalStrapiDestinationProviderOptions } from '../../../../src/strapi/providers'; + +export type CommandMessage = { type: 'command' } & (InitCommand | EndCommand | StatusCommand); + +export type Command = CommandMessage['command']; + +export type GetCommandParams = { + [key in Command]: { command: key } & CommandMessage; +}[T] extends { params: infer U } + ? U + : never; + +export type InitCommand = CreateCommand< + 'init', + | { + transfer: 'push'; + options: Pick; + } + | { transfer: 'pull' } +>; +export type TransferKind = InitCommand['params']['transfer']; + +export type EndCommand = CreateCommand<'end', { transferID: string }>; + +export type StatusCommand = CreateCommand<'status'>; + +type CreateCommand = never> = { + type: 'command'; + command: T; +} & ([U] extends [never] ? unknown : { params: U }); diff --git a/packages/core/data-transfer/types/remote/protocol/client/index.d.ts b/packages/core/data-transfer/types/remote/protocol/client/index.d.ts new file mode 100644 index 0000000000..5701cd242e --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/index.d.ts @@ -0,0 +1,8 @@ +import type { CommandMessage } from './commands'; +import type { TransferMessage } from './transfer'; + +export * from './commands'; +export * from './transfer'; + +export type Message = { uuid: string } & (CommandMessage | TransferMessage); +export type MessageType = Message['type']; diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/action.d.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/action.d.ts new file mode 100644 index 0000000000..c729791b8b --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/action.d.ts @@ -0,0 +1,3 @@ +import type { CreateTransferMessage } from './utils'; + +export type Action = CreateTransferMessage<'action', { action: string }>; diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/index.d.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/index.d.ts new file mode 100644 index 0000000000..037b3bc61f --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/index.d.ts @@ -0,0 +1,13 @@ +import type { Action } from './action'; +import type { TransferPullMessage } from './pull'; +import type { TransferPushMessage } from './push'; + +export * from './action'; +export * from './pull'; +export * from './push'; + +export type TransferMessage = { type: 'transfer'; transferID: string } & ( + | Action + | TransferPushMessage + | TransferPullMessage +); diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.d.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.d.ts new file mode 100644 index 0000000000..43b07f62d2 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/pull.d.ts @@ -0,0 +1,8 @@ +import { CreateTransferMessage } from './utils'; + +export type TransferPullMessage = CreateTransferMessage< + 'step', + { + action: 'start' | 'stop'; + } +>; diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/push.d.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/push.d.ts new file mode 100644 index 0000000000..cc5fc9b760 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/push.d.ts @@ -0,0 +1,31 @@ +import type { CreateTransferMessage } from './utils'; +import type { IEntity, ILink, IConfiguration, IAsset } from '../../../../common-entities'; + +export type TransferPushMessage = CreateTransferMessage< + 'step', + | TransferStepCommands<'entities', IEntity> + | TransferStepCommands<'links', ILink> + | TransferStepCommands<'configuration', IConfiguration> + | TransferStepCommands<'assets', TransferAssetFlow | null> +>; + +export type GetTransferPushStreamData = { + [key in TransferPushStep]: { + action: 'stream'; + step: key; + } & TransferPushMessage; +}[T] extends { data: infer U } + ? U + : never; + +export type TransferPushStep = TransferPushMessage['step']; + +type TransferStepCommands = { step: T } & TransferStepFlow; + +type TransferStepFlow = { action: 'start' } | { action: 'stream'; data: U } | { action: 'end' }; + +type TransferAssetFlow = { assetID: string } & ( + | { action: 'start'; data: Omit } + | { action: 'stream'; data: Buffer } + | { action: 'end' } +); diff --git a/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.d.ts b/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.d.ts new file mode 100644 index 0000000000..b9d838c95e --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/client/transfer/utils.d.ts @@ -0,0 +1,5 @@ +export type CreateTransferMessage = { + type: 'transfer'; + kind: T; + transferID: string; +} & U; diff --git a/packages/core/data-transfer/types/remote/protocol/index.d.ts b/packages/core/data-transfer/types/remote/protocol/index.d.ts new file mode 100644 index 0000000000..4cbe5b42a3 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/index.d.ts @@ -0,0 +1,2 @@ +export * as client from './client'; +export * as server from './server'; diff --git a/packages/core/data-transfer/types/remote/protocol/server/error.d.ts b/packages/core/data-transfer/types/remote/protocol/server/error.d.ts new file mode 100644 index 0000000000..5925f89235 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/error.d.ts @@ -0,0 +1,29 @@ +export enum ErrorKind { + // Generic + Unknown = 0, + // Chunk transfer + DiscardChunk = 1, + InvalidChunkFormat = 2, +} + +export class ServerError extends Error { + constructor( + public code: ErrorKind, + public message: string, + public details?: Record | null + ) { + super(message); + } +} + +export class UnknownError extends ServerError { + constructor(message: string, details?: Record | null) { + super(ErrorKind.Unknown, message, details); + } +} + +export class DiscardChunkError extends ServerError { + constructor(message: string, details?: Record | null) { + super(ErrorKind.DiscardChunk, message, details); + } +} diff --git a/packages/core/data-transfer/types/remote/protocol/server/index.d.ts b/packages/core/data-transfer/types/remote/protocol/server/index.d.ts new file mode 100644 index 0000000000..cbe3473c90 --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/index.d.ts @@ -0,0 +1,2 @@ +export * from './messaging'; +export * as error from './error'; diff --git a/packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts b/packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts new file mode 100644 index 0000000000..1c0d5cd71d --- /dev/null +++ b/packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts @@ -0,0 +1,14 @@ +import type { ServerError } from './error'; + +export type Message = { + uuid?: string; + data?: T | null; + error?: ServerError | null; +}; + +// Successful +export type OKMessage = Message<{ ok: true }>; +export type InitMessage = Message<{ transferID: string }>; +export type EndMessage = OKMessage; + +export type Payload = T['data']; diff --git a/packages/core/strapi/bin/strapi.js b/packages/core/strapi/bin/strapi.js index aaae25cfaa..63ceeee992 100755 --- a/packages/core/strapi/bin/strapi.js +++ b/packages/core/strapi/bin/strapi.js @@ -258,6 +258,23 @@ program .option('-s, --silent', `Run the generation silently, without any output`, false) .action(getLocalScript('ts/generate-types')); +// `$ strapi transfer` +program + .command('transfer') + .description('Transfer data from one source to another') + .addOption(new Option('--from ', `URL of remote Strapi instance to get data from.`)) + .addOption(new Option('--to ', `URL of remote Strapi instance to send data to`)) + .hook('preAction', async (thisCommand) => { + const opts = thisCommand.opts(); + + if (!opts.from && !opts.to) { + console.error('At least one source (from) or destination (to) option must be provided'); + process.exit(1); + } + }) + .allowExcessArguments(false) + .action(getLocalScript('transfer/transfer')); + // `$ strapi export` program .command('export') diff --git a/packages/core/strapi/lib/commands/__tests__/export.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js similarity index 62% rename from packages/core/strapi/lib/commands/__tests__/export.test.js rename to packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js index 6a57e78533..cc04449279 100644 --- a/packages/core/strapi/lib/commands/__tests__/export.test.js +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js @@ -1,33 +1,39 @@ 'use strict'; -describe('export', () => { +describe('Export', () => { const defaultFileName = 'defaultFilename'; - // mock @strapi/data-transfer const mockDataTransfer = { - createLocalFileDestinationProvider: jest.fn().mockReturnValue({ name: 'testDest' }), - createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testSource' }), - createTransferEngine() { - return { - transfer: jest.fn().mockReturnValue(Promise.resolve({})), - progress: { - on: jest.fn(), - stream: { + file: { + providers: { + createLocalFileDestinationProvider: jest.fn().mockReturnValue({ name: 'testDest' }), + }, + }, + strapi: { + providers: { + createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testSource' }), + }, + }, + engine: { + createTransferEngine() { + return { + transfer: jest.fn().mockReturnValue(Promise.resolve({})), + progress: { on: jest.fn(), + stream: { + on: jest.fn(), + }, }, - }, - sourceProvider: { name: 'testSource' }, - destinationProvider: { name: 'testDestination' }, - }; + sourceProvider: { name: 'testSource' }, + destinationProvider: { name: 'testDestination' }, + }; + }, }, }; - jest.mock( - '@strapi/data-transfer', - () => { - return mockDataTransfer; - }, - { virtual: true } - ); + + jest.mock('@strapi/data-transfer/lib/engine', () => mockDataTransfer.engine, { virtual: true }); + jest.mock('@strapi/data-transfer/lib/strapi', () => mockDataTransfer.strapi, { virtual: true }); + jest.mock('@strapi/data-transfer/lib/file', () => mockDataTransfer.file, { virtual: true }); // mock utils const mockUtils = { @@ -41,7 +47,7 @@ describe('export', () => { getDefaultExportName: jest.fn(() => defaultFileName), }; jest.mock( - '../transfer/utils', + '../../transfer/utils', () => { return mockUtils; }, @@ -54,7 +60,7 @@ describe('export', () => { jest.spyOn(console, 'error').mockImplementation(() => {}); // Now that everything is mocked, import export command - const exportCommand = require('../transfer/export'); + const exportCommand = require('../../transfer/export'); const expectExit = async (code, fn) => { const exit = jest.spyOn(process, 'exit').mockImplementation((number) => { @@ -76,7 +82,7 @@ describe('export', () => { await exportCommand({ file: filename }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ file: { path: filename }, }) @@ -90,7 +96,7 @@ describe('export', () => { }); expect(mockUtils.getDefaultExportName).toHaveBeenCalledTimes(1); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ file: { path: defaultFileName }, }) @@ -103,7 +109,7 @@ describe('export', () => { await exportCommand({ encrypt }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ encryption: { enabled: encrypt }, }) @@ -117,7 +123,7 @@ describe('export', () => { await exportCommand({ encrypt, key }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ encryption: { enabled: encrypt, key }, }) @@ -129,7 +135,7 @@ describe('export', () => { await exportCommand({ compress: false }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ compression: { enabled: false }, }) @@ -137,7 +143,7 @@ describe('export', () => { await expectExit(1, async () => { await exportCommand({ compress: true }); }); - expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith( + expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith( expect.objectContaining({ compression: { enabled: true }, }) diff --git a/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js new file mode 100644 index 0000000000..3ae7131ffd --- /dev/null +++ b/packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js @@ -0,0 +1,113 @@ +'use strict'; + +const utils = require('../../transfer/utils'); + +const mockDataTransfer = { + strapi: { + providers: { + createRemoteStrapiDestinationProvider: jest.fn(), + createLocalStrapiSourceProvider: jest.fn(), + }, + }, + engine: { + createTransferEngine: jest.fn().mockReturnValue({ + transfer: jest.fn().mockReturnValue(Promise.resolve({})), + }), + }, +}; + +jest.mock('@strapi/data-transfer/lib/engine', () => mockDataTransfer.engine, { virtual: true }); +jest.mock('@strapi/data-transfer/lib/strapi', () => mockDataTransfer.strapi, { virtual: true }); + +const expectExit = async (code, fn) => { + const exit = jest.spyOn(process, 'exit').mockImplementation((number) => { + throw new Error(`process.exit: ${number}`); + }); + await expect(async () => { + await fn(); + }).rejects.toThrow(); + expect(exit).toHaveBeenCalledWith(code); + exit.mockRestore(); +}; + +const transferCommand = require('../../transfer/transfer'); + +jest.spyOn(console, 'error').mockImplementation(() => {}); +jest.spyOn(console, 'warn').mockImplementation(() => {}); +jest.spyOn(console, 'log').mockImplementation(() => {}); + +jest.mock('../../transfer/utils'); + +const destinationUrl = 'ws://strapi.com'; + +describe('Transfer', () => { + beforeEach(() => { + jest.resetAllMocks(); + }); + + it('uses destination url provided by user without authentication', async () => { + await expectExit(1, async () => { + await transferCommand({ from: undefined, to: destinationUrl }); + }); + + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + }); + + it.todo('uses destination url provided by user with authentication'); + + it('uses restore as the default strategy', async () => { + await expectExit(1, async () => { + await transferCommand({ from: undefined, to: destinationUrl }); + }); + + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + it('uses destination url provided by user without authentication', async () => { + await expectExit(1, async () => { + await transferCommand({ from: undefined, to: destinationUrl }); + }); + + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( + expect.objectContaining({ + url: destinationUrl, + }) + ); + }); + + it('uses restore as the default strategy', async () => { + await expectExit(1, async () => { + await transferCommand({ from: undefined, to: destinationUrl }); + }); + + expect( + mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider + ).toHaveBeenCalledWith( + expect.objectContaining({ + strategy: 'restore', + }) + ); + }); + + it('uses local strapi instance when local specified', async () => { + await expectExit(1, async () => { + await transferCommand({ from: undefined, to: destinationUrl }); + }); + + expect(mockDataTransfer.strapi.providers.createLocalStrapiSourceProvider).toHaveBeenCalled(); + expect(utils.createStrapiInstance).toHaveBeenCalled(); + }); +}); diff --git a/packages/core/strapi/lib/commands/transfer/export.js b/packages/core/strapi/lib/commands/transfer/export.js index 795bb713f7..be5d33eb08 100644 --- a/packages/core/strapi/lib/commands/transfer/export.js +++ b/packages/core/strapi/lib/commands/transfer/export.js @@ -1,10 +1,12 @@ 'use strict'; const { - createLocalFileDestinationProvider, - createLocalStrapiSourceProvider, - createTransferEngine, -} = require('@strapi/data-transfer'); + providers: { createLocalFileDestinationProvider }, +} = require('@strapi/data-transfer/lib/file'); +const { + providers: { createLocalStrapiSourceProvider }, +} = require('@strapi/data-transfer/lib/strapi'); +const { createTransferEngine } = require('@strapi/data-transfer/lib/engine'); const { isObject, isString, isFinite, toNumber } = require('lodash/fp'); const fs = require('fs-extra'); const chalk = require('chalk'); diff --git a/packages/core/strapi/lib/commands/transfer/import.js b/packages/core/strapi/lib/commands/transfer/import.js index 531cd87a4e..72fd378d79 100644 --- a/packages/core/strapi/lib/commands/transfer/import.js +++ b/packages/core/strapi/lib/commands/transfer/import.js @@ -1,13 +1,17 @@ 'use strict'; const { - createLocalFileSourceProvider, - createLocalStrapiDestinationProvider, + providers: { createLocalFileSourceProvider }, +} = require('@strapi/data-transfer/lib/file'); +const { + providers: { createLocalStrapiDestinationProvider, DEFAULT_CONFLICT_STRATEGY }, +} = require('@strapi/data-transfer/lib/strapi'); +const { createTransferEngine, DEFAULT_VERSION_STRATEGY, DEFAULT_SCHEMA_STRATEGY, - DEFAULT_CONFLICT_STRATEGY, -} = require('@strapi/data-transfer'); +} = require('@strapi/data-transfer/lib/engine'); + const { isObject } = require('lodash/fp'); const path = require('path'); @@ -43,6 +47,7 @@ module.exports = async (opts) => { async getStrapi() { return strapiInstance; }, + autoDestroy: false, strategy: opts.conflictStrategy || DEFAULT_CONFLICT_STRATEGY, restore: { entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES }, @@ -108,6 +113,8 @@ module.exports = async (opts) => { // Note: Telemetry can't be sent in a finish event, because it runs async after this block but we can't await it, so if process.exit is used it won't send await strapiInstance.telemetry.send('didDEITSProcessFinish', getTelemetryPayload()); + await strapiInstance.destroy(); + process.exit(0); }; diff --git a/packages/core/strapi/lib/commands/transfer/transfer.js b/packages/core/strapi/lib/commands/transfer/transfer.js new file mode 100644 index 0000000000..eacea973d9 --- /dev/null +++ b/packages/core/strapi/lib/commands/transfer/transfer.js @@ -0,0 +1,127 @@ +'use strict'; + +const { createTransferEngine } = require('@strapi/data-transfer/lib/engine'); +const { + providers: { + createRemoteStrapiDestinationProvider, + createLocalStrapiSourceProvider, + createLocalStrapiDestinationProvider, + }, +} = require('@strapi/data-transfer/lib/strapi'); +const { isObject } = require('lodash/fp'); +const chalk = require('chalk'); + +const { + buildTransferTable, + createStrapiInstance, + DEFAULT_IGNORED_CONTENT_TYPES, +} = require('./utils'); + +const logger = console; + +/** + * @typedef TransferCommandOptions Options given to the CLI transfer command + * + * @property {string|undefined} [to] The url of a remote Strapi to use as remote destination + * @property {string|undefined} [from] The url of a remote Strapi to use as remote source + */ + +/** + * Transfer command. + * + * It transfers data from a local file to a local strapi instance + * + * @param {TransferCommandOptions} opts + */ +module.exports = async (opts) => { + // Validate inputs from Commander + if (!isObject(opts)) { + logger.error('Could not parse command arguments'); + process.exit(1); + } + + const strapi = await createStrapiInstance(); + + let source; + let destination; + + if (!opts.from && !opts.to) { + logger.error('At least one source (from) or destination (to) option must be provided'); + process.exit(1); + } + + // if no URL provided, use local Strapi + if (!opts.from) { + source = createLocalStrapiSourceProvider({ + getStrapi: () => strapi, + }); + } + // if URL provided, set up a remote source provider + else { + logger.error(`Remote Strapi source provider not yet implemented`); + process.exit(1); + } + + // if no URL provided, use local Strapi + if (!opts.to) { + destination = createLocalStrapiDestinationProvider({ + getStrapi: () => strapi, + }); + } + // if URL provided, set up a remote destination provider + else { + destination = createRemoteStrapiDestinationProvider({ + url: opts.to, + auth: false, + strategy: 'restore', + restore: { + entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES }, + }, + }); + } + + if (!source || !destination) { + logger.error('Could not create providers'); + process.exit(1); + } + + const engine = createTransferEngine(source, destination, { + versionStrategy: 'ignore', // for an export to file, versionStrategy will always be skipped + schemaStrategy: 'ignore', // for an export to file, schemaStrategy will always be skipped + transforms: { + links: [ + { + filter(link) { + return ( + !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) && + !DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type) + ); + }, + }, + ], + entities: [ + { + filter(entity) { + return !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type); + }, + }, + ], + }, + }); + + try { + logger.log(`Starting transfer...`); + + const results = await engine.transfer(); + + const table = buildTransferTable(results.engine); + logger.log(table.toString()); + + logger.log(`${chalk.bold('Transfer process has been completed successfully!')}`); + process.exit(0); + } catch (e) { + logger.error('Transfer process failed unexpectedly'); + logger.error(e); + process.exit(1); + } +}; diff --git a/packages/core/strapi/package.json b/packages/core/strapi/package.json index d7b2b05a3b..110a2505c7 100644 --- a/packages/core/strapi/package.json +++ b/packages/core/strapi/package.json @@ -83,6 +83,7 @@ "@strapi/admin": "4.5.5", "@strapi/data-transfer": "4.5.5", "@strapi/database": "4.5.5", + "@strapi/data-transfer": "4.5.5", "@strapi/generate-new": "4.5.5", "@strapi/generators": "4.5.5", "@strapi/logger": "4.5.5", diff --git a/packages/plugins/sentry/README.md b/packages/plugins/sentry/README.md index 56ba6d59c2..496e2dee27 100644 --- a/packages/plugins/sentry/README.md +++ b/packages/plugins/sentry/README.md @@ -68,10 +68,7 @@ try { // Your code here } catch (error) { // Either send a simple error - strapi - .plugin('sentry') - .service('sentry') - .sendError(error); + strapi.plugin('sentry').service('sentry').sendError(error); // Or send an error with a customized Sentry scope strapi @@ -92,16 +89,13 @@ Use it if you need direct access to the Sentry instance, which should already al **Example** ```js -const sentryInstance = strapi - .plugin('sentry') - .service('sentry') - .getInstance(); +const sentryInstance = strapi.plugin('sentry').service('sentry').getInstance(); ``` ## Disabling for non-production environments If the `dsn` property is set to a nil value (`null` or `undefined`) while `enabled` is true, the Sentry plugin will be available to use in the running Strapi instance, but the service will not actually send errors to Sentry. That allows you to write code that runs on every environment without additional checks, but only send errors to Sentry in production. - + When you start Strapi with a nil `dsn` config property, the plugin will print a warning: `info: @strapi/plugin-sentry is disabled because no Sentry DSN was provided` @@ -137,7 +131,7 @@ Like every other plugin, you can also disable this plugin in the plugins configu module.exports = ({ env }) => ({ // ... sentry: { - enabled: false + enabled: false, }, // ... }); diff --git a/yarn.lock b/yarn.lock index 8367b9528b..a424b6d1b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6300,6 +6300,20 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/koa@2.13.4": + version "2.13.4" + resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b" + integrity sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw== + dependencies: + "@types/accepts" "*" + "@types/content-disposition" "*" + "@types/cookies" "*" + "@types/http-assert" "*" + "@types/http-errors" "*" + "@types/keygrip" "*" + "@types/koa-compose" "*" + "@types/node" "*" + "@types/koa__cors@^3.0.1": version "3.3.0" resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.0.tgz#2986b320d3d7ddf05c4e2e472b25a321cb16bd3b" @@ -6603,6 +6617,11 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== +"@types/uuid@9.0.0": + version "9.0.0" + resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.0.tgz#53ef263e5239728b56096b0a869595135b7952d2" + integrity sha512-kr90f+ERiQtKWMz5rP32ltJ/BtULDI5RVO0uavn1HQUOwjx0R1h0rnDYNL0CepF1zL5bSY6FISAfd9tOdDhU5Q== + "@types/webpack-env@^1.16.0": version "1.18.0" resolved "https://registry.yarnpkg.com/@types/webpack-env/-/webpack-env-1.18.0.tgz#ed6ecaa8e5ed5dfe8b2b3d00181702c9925f13fb" @@ -18204,6 +18223,8 @@ path-case@^2.1.0: version "2.1.1" resolved "https://registry.yarnpkg.com/path-case/-/path-case-2.1.1.tgz#94b8037c372d3fe2906e465bb45e25d226e8eea5" integrity sha1-lLgDfDctP+KQbkZbtF4l0ibo7qU= + dependencies: + no-case "^2.2.0" path-dirname@^1.0.0: version "1.0.2" @@ -22571,6 +22592,11 @@ uuid@8.0.0: resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c" integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw== +uuid@9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5" + integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg== + uuid@^3.3.2: version "3.4.0" resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -23276,6 +23302,11 @@ write-pkg@^4.0.0: type-fest "^0.4.1" write-json-file "^3.2.0" +ws@8.11.0: + version "8.11.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" + integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== + "ws@^5.2.0 || ^6.0.0 || ^7.0.0", ws@^7.3.1: version "7.5.9" resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"