yujunjun/strapi
1
0
Fork 0
mirror of https://github.com/strapi/strapi.git synced 2025-11-03 03:17:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #15343 from strapi/deits/transfer-push

Browse Source
This commit is contained in:
Jean-Sébastien Herbaux 2023-01-04 16:02:38 +01:00 committed by GitHub
commit 1ebbc5f504
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
94 changed files with 1718 additions and 310 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintignore
View File

@ -12,6 +12,7 @@ packages/core/helper-plugin/build/**
packages/core/helper-plugin/lib/src/old/components/**
packages/core/helper-plugin/lib/src/testUtils/**
packages/core/helper-plugin/lib/src/utils/**
packages/core/data-transfer/lib
.eslintrc.js
.eslintrc.front.js
.eslintrc.back.js

1
.eslintrc.back.js
View File

@ -31,6 +31,7 @@ module.exports = {
'no-continue': 'warn',
'no-process-exit': 'off',
'no-loop-func': 'off',
'max-classes-per-file': 'off',
'no-param-reassign': [
'error',
{

1
.eslintrc.back.typescript.js
View File

@ -22,6 +22,7 @@ module.exports = {
'@typescript-eslint/brace-style': 'off', // TODO: fix conflict with prettier/prettier in data-transfer/engine/index.ts
// to be cleaned up throughout codebase (too many to fix at the moment)
'@typescript-eslint/no-use-before-define': 'warn',
'@typescript-eslint/comma-dangle': 'off',
},
// Disable only for tests
overrides: [

1
packages/core/admin/package.json
View File

@ -46,6 +46,7 @@
"@casl/ability": "^5.4.3",
"@fingerprintjs/fingerprintjs": "3.3.3",
"@pmmmwh/react-refresh-webpack-plugin": "0.5.7",
"@strapi/data-transfer": "4.5.5",
"@strapi/babel-plugin-switch-ee-ce": "4.5.5",
"@strapi/design-system": "1.4.1",
"@strapi/helper-plugin": "4.5.5",

4
packages/core/admin/server/register.js
View File

@ -1,5 +1,7 @@
'use strict';
const { register: registerDataTransfer } = require('@strapi/data-transfer/lib/strapi');
const registerAdminPanelRoute = require('./routes/serve-admin-panel');
const adminAuthStrategy = require('./strategies/admin');
const apiTokenAuthStrategy = require('./strategies/api-token');
@ -14,4 +16,6 @@ module.exports = ({ strapi }) => {
if (strapi.config.serveAdminPanel) {
registerAdminPanelRoute({ strapi });
}
registerDataTransfer(strapi);
};

1
packages/core/data-transfer/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
lib/

11
packages/core/data-transfer/.npmignore
View File

@ -1,9 +1,9 @@
#############################
# DATA TRANSFER X
# DATA TRANSFER
############################
*.js.map
lib/
src/
types/
tsconfig.json
@ -19,14 +19,12 @@ Icon
.Trashes
._*
############################
# Linux
############################
*~
############################
# Windows
############################
@ -40,7 +38,6 @@ $RECYCLE.BIN/
*.msm
*.msp
############################
# Packages
############################
@ -69,7 +66,6 @@ $RECYCLE.BIN/
*.out
*.pid
############################
# Logs and databases
############################
@ -79,7 +75,6 @@ $RECYCLE.BIN/
*.sql
*.sqlite
############################
# Misc.
############################
@ -89,7 +84,6 @@ $RECYCLE.BIN/
nbproject
.vscode/
############################
# Node.js
############################
@ -107,7 +101,6 @@ package-lock.json
!docs/package-lock.json
*.heapsnapshot
############################
# Tests
############################

2
packages/core/data-transfer/lib/index.ts
View File

@ -1,2 +0,0 @@
export * from './engine';
export * from './providers';

7
packages/core/data-transfer/lib/providers/index.ts
View File

@ -1,7 +0,0 @@
// source providers
export * from './local-file-source-provider';
export * from './local-strapi-source-provider';
// destination providers
export * from './local-file-destination-provider';
export * from './local-strapi-destination-provider';

1
packages/core/data-transfer/lib/providers/shared/index.ts
View File

@ -1 +0,0 @@
export * as strapi from './strapi';

83
packages/core/data-transfer/lib/providers/test-utils/index.ts
View File

@ -1,83 +0,0 @@
import { Readable } from 'stream';
/**
* Collect every entity in a Readable stream
*/
export const collect = <T = unknown>(stream: Readable): Promise<T[]> => {
const chunks: T[] = [];
return new Promise((resolve, reject) => {
stream
.on('data', (chunk) => chunks.push(chunk))
.on('close', () => resolve(chunks))
.on('error', reject);
});
};
/**
* Create a "Strapi" like object factory based on the
* given params and cast it to the correct type
*/
export const getStrapiFactory =
<
T extends {
[key in keyof Partial<Strapi.Strapi>]: unknown;
}
>(
properties?: T
) =>
() => {
return { ...properties } as Strapi.Strapi;
};
/**
* Union type used to represent the default content types available
*/
export type ContentType = 'foo' | 'bar';
/**
* Factory to get default content types test values
*/
export const getContentTypes = (): {
[key in ContentType]: { uid: key; attributes: { [attribute: string]: unknown } };
} => ({
foo: { uid: 'foo', attributes: { title: { type: 'string' } } },
bar: { uid: 'bar', attributes: { age: { type: 'number' } } },
});
/**
* Create a factory of readable streams (wrapped with a jest mock function)
*/
export const createMockedReadableFactory = <T extends string = ContentType>(source: {
[ct in T]: Array<{ id: number; [key: string]: unknown }>;
}) =>
jest.fn((uid: T) => {
return Readable.from(source[uid] || []);
});
/**
* Create a factory of mocked query builders
*/
export const createMockedQueryBuilder = <T extends string = ContentType>(data: {
[key in T]: unknown[];
}) =>
jest.fn((uid: T) => {
const state: { [key: string]: unknown } = { populate: undefined };
return {
populate(populate: unknown) {
state.populate = populate;
return this;
},
stream() {
return Readable.from(data[uid]);
},
};
});
/**
* Update the global store with the given strapi value
*/
export const setGlobalStrapi = (strapi: Strapi.Strapi): void => {
(global as unknown as Global).strapi = strapi;
};

20
packages/core/data-transfer/lib/utils/schema.ts
View File

@ -1,20 +0,0 @@
import type { Schema } from '@strapi/strapi';
import { mapValues, pick } from 'lodash/fp';
const schemaSelectedKeys = [
'collectionName',
'info',
'options',
'pluginOptions',
'attributes',
'kind',
'modelType',
'modelName',
'uid',
'plugin',
'globalId',
];
export const mapSchemasValues = (schemas: Record<string, Schema>) => {
return mapValues(pick(schemaSelectedKeys), schemas);
};

48
packages/core/data-transfer/lib/utils/stream.ts
View File

@ -1,48 +0,0 @@
import { Transform, Readable } from 'stream';
type TransformOptions = ConstructorParameters<typeof Transform>[0];
export const filter = <T>(
predicate: (value: T) => boolean | Promise<boolean>,
options: TransformOptions = { objectMode: true }
): Transform => {
return new Transform({
...options,
async transform(chunk, _encoding, callback) {
const keep = await predicate(chunk);
callback(null, keep ? chunk : undefined);
},
});
};
export const map = <T>(
predicate: (value: T) => T | Promise<T>,
options: TransformOptions = { objectMode: true }
): Transform => {
return new Transform({
...options,
async transform(chunk, _encoding, callback) {
const mappedValue = await predicate(chunk);
callback(null, mappedValue);
},
});
};
/**
* Collect every entity in a Readable stream
*/
export const collect = <T = unknown>(stream: Readable): Promise<T[]> => {
const chunks: T[] = [];
return new Promise((resolve) => {
stream.on('data', (chunk) => chunks.push(chunk));
stream.on('end', () => {
stream.destroy();
resolve(chunks);
});
});
};

13
packages/core/data-transfer/package.json
View File

@ -24,8 +24,8 @@
"url": "https://strapi.io"
}
],
"main": "./dist/index.js",
"types": "./dist/index.d.ts",
"main": "./lib/index.js",
"types": "./lib/index.d.ts",
"scripts": {
"build": "yarn build:ts",
"build:ts": "tsc -p tsconfig.json",
@ -36,7 +36,7 @@
"watch": "yarn build:ts -w --preserveWatchOutput"
},
"directories": {
"lib": "./dist"
"lib": "./lib"
},
"dependencies": {
"@strapi/logger": "4.5.5",
@ -49,7 +49,9 @@
"stream-chain": "2.2.5",
"stream-json": "1.7.4",
"tar": "6.1.12",
"tar-stream": "2.2.0"
"tar-stream": "2.2.0",
"uuid": "9.0.0",
"ws": "8.11.0"
},
"devDependencies": {
"@tsconfig/node16": "1.0.3",
@ -60,6 +62,9 @@
"@types/stream-json": "1.7.2",
"@types/tar": "6.1.3",
"@types/tar-stream": "2.2.2",
"@types/uuid": "9.0.0",
"koa": "2.13.4",
"@types/koa": "2.13.4",
"rimraf": "3.0.2",
"typescript": "4.6.2"
},

7
packages/core/data-transfer/lib/__tests__/test-utils.ts → packages/core/data-transfer/src/__tests__/test-utils.ts
View File

@ -102,6 +102,13 @@ export const destinationStages = [
'getSchemasStream',
];
/**
* Update the global store with the given strapi value
*/
export const setGlobalStrapi = (strapi: Strapi.Strapi): void => {
(global as unknown as Global).strapi = strapi;
};
/**
* Add jest expect helpers
*/

0
packages/core/data-transfer/lib/engine/__tests__/engine.test.ts → packages/core/data-transfer/src/engine/__tests__/engine.test.ts
View File

22
packages/core/data-transfer/lib/engine/index.ts → packages/core/data-transfer/src/engine/index.ts
View File

@ -19,7 +19,7 @@ import type {
} from '../../types';
import type { Diff } from '../utils/json';
import compareSchemas from '../strategies';
import { compareSchemas } from './validation/schemas';
import { filter, map } from '../utils/stream';
export const TRANSFER_STAGES: ReadonlyArray<TransferStage> = Object.freeze([
@ -250,7 +250,23 @@ class TransferEngine<
const { stage, source, destination, transform, tracker } = options;
if (!source || !destination) {
// Wait until source and destination are closed
await Promise.allSettled(
[source, destination].map((stream) => {
// if stream is undefined or already closed, resolve immediately
if (!stream || stream.destroyed) {
return Promise.resolve();
}
// Wait until the close event is produced and then destroy the stream and resolve
return new Promise((resolve, reject) => {
stream.on('close', resolve).on('error', reject).destroy();
});
})
);
this.#emitStageUpdate('skip', stage);
return;
}
@ -340,9 +356,7 @@ class TransferEngine<
this.#emitTransferUpdate('init');
await this.bootstrap();
await this.init();
const isValidTransfer = await this.integrityCheck();
if (!isValidTransfer) {
// TODO: provide the log from the integrity check
throw new Error(
@ -353,14 +367,12 @@ class TransferEngine<
this.#emitTransferUpdate('start');
await this.beforeTransfer();
// Run the transfer stages
await this.transferSchemas();
await this.transferEntities();
await this.transferAssets();
await this.transferLinks();
await this.transferConfiguration();
// Gracefully close the providers
await this.close();

1
packages/core/data-transfer/src/engine/validation/index.ts Normal file
View File

@ -0,0 +1 @@
export * as schemas from './schemas';

6
packages/core/data-transfer/lib/strategies/index.ts → packages/core/data-transfer/src/engine/validation/schemas/index.ts
View File

@ -1,5 +1,5 @@
import type { Diff } from '../utils/json';
import * as utils from '../utils';
import type { Diff } from '../../../utils/json';
import * as utils from '../../../utils';
const strategies = {
// No diffs
@ -32,4 +32,4 @@ const compareSchemas = <T, P>(a: T, b: P, strategy: keyof typeof strategies) =>
return strategies[strategy](diffs);
};
export default compareSchemas;
export { compareSchemas };

1
packages/core/data-transfer/src/file/index.ts Normal file
View File

@ -0,0 +1 @@
export * as providers from './providers';

6
packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/index.test.ts → packages/core/data-transfer/src/file/providers/destination/__tests__/index.test.ts
View File

@ -6,7 +6,7 @@ jest.mock('fs');
import fs from 'fs-extra';
import { Writable } from 'stream-chain';
import { createLocalFileDestinationProvider, ILocalFileDestinationProviderOptions } from '..';
import * as encryption from '../../../encryption/encrypt';
import * as encryption from '../../../../utils/encryption';
import { createFilePathFactory, createTarEntryStream } from '../utils';
fs.createWriteStream = jest.fn().mockReturnValue(
@ -18,14 +18,14 @@ fs.createWriteStream = jest.fn().mockReturnValue(
const filePath = './test-file';
jest.mock('../../../encryption/encrypt', () => {
jest.mock('../../../../utils/encryption', () => {
return {
__esModule: true,
createEncryptionCipher() {},
};
});
jest.mock('../../local-file-destination-provider/utils');
jest.mock('../utils');
describe('Local File Destination Provider', () => {
(createFilePathFactory as jest.Mock).mockImplementation(jest.fn());

0
packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts → packages/core/data-transfer/src/file/providers/destination/__tests__/utils.test.ts
View File

4
packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts → packages/core/data-transfer/src/file/providers/destination/index.ts
View File

@ -6,7 +6,7 @@ import { stringer } from 'stream-json/jsonl/Stringer';
import { chain } from 'stream-chain';
import { Readable, Writable } from 'stream';
import { createEncryptionCipher } from '../../encryption/encrypt';
import { createEncryptionCipher } from '../../../utils/encryption';
import type {
IAsset,
IDestinationProvider,
@ -14,7 +14,7 @@ import type {
IMetadata,
ProviderType,
Stream,
} from '../../../types';
} from '../../../../types';
import { createFilePathFactory, createTarEntryStream } from './utils';
export interface ILocalFileDestinationProviderOptions {

0
packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts → packages/core/data-transfer/src/file/providers/destination/utils.ts
View File

2
packages/core/data-transfer/src/file/providers/index.ts Normal file
View File

@ -0,0 +1,2 @@
export * from './source';
export * from './destination';

0
packages/core/data-transfer/lib/providers/local-file-source-provider/__tests__/index.test.ts → packages/core/data-transfer/src/file/providers/source/__tests__/index.test.ts
View File

28
packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts → packages/core/data-transfer/src/file/providers/source/index.ts
View File

@ -8,10 +8,10 @@ import { keyBy } from 'lodash/fp';
import { chain } from 'stream-chain';
import { pipeline, PassThrough } from 'stream';
import { parser } from 'stream-json/jsonl/Parser';
import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../types';
import type { IAsset, IMetadata, ISourceProvider, ProviderType } from '../../../../types';
import { createDecryptionCipher } from '../../encryption';
import * as utils from '../../utils';
import { createDecryptionCipher } from '../../../utils/encryption';
import { collect } from '../../../utils/stream';
type StreamItemArray = Parameters<typeof chain>[0];
@ -83,7 +83,7 @@ class LocalFileSourceProvider implements ISourceProvider {
}
async getSchemas() {
const schemas = await utils.stream.collect(this.streamSchemas());
const schemas = await collect(this.streamSchemas());
return keyBy('uid', schemas);
}
@ -227,20 +227,22 @@ class LocalFileSourceProvider implements ISourceProvider {
return entryPath === filePath && entry.type === 'File';
},
/**
* Whenever an entry passes the filter method, process it
*/
async onentry(entry) {
// Collect all the content of the entry file
const content = await entry.collect();
// Parse from buffer to string to JSON
const parsedContent = JSON.parse(content.toString());
// Resolve the Promise with the parsed content
resolve(parsedContent);
try {
// Parse from buffer to string to JSON
const parsedContent = JSON.parse(content.toString());
// Cleanup (close the stream associated to the entry)
entry.destroy();
// Resolve the Promise with the parsed content
resolve(parsedContent);
} catch (e) {
reject(e);
} finally {
// Cleanup (close the stream associated to the entry)
entry.destroy();
}
},
}),
],

4
packages/core/data-transfer/src/index.ts Normal file
View File

@ -0,0 +1,4 @@
export * as engine from './engine';
export * as strapi from './strapi';
export * as file from './file';
export * as utils from './utils';

36
packages/core/data-transfer/src/strapi/__tests__/register.test.ts Normal file
View File

@ -0,0 +1,36 @@
import { getStrapiFactory } from '../../__tests__/test-utils';
import { createTransferHandler } from '../remote/handlers';
import register from '../register';
afterEach(() => {
jest.clearAllMocks();
});
const strapiMockFactory = getStrapiFactory({
admin: {
routes: {
push: jest.fn(),
},
},
});
jest.mock('../remote/handlers', () => ({
createTransferHandler: jest.fn(),
}));
describe('Register the Transfer route', () => {
test('registers the /transfer route', () => {
const strapi = strapiMockFactory();
register(strapi);
expect(strapi.admin.routes.push).toHaveBeenCalledWith({
method: 'GET',
path: '/transfer',
handler: createTransferHandler(),
config: {
auth: false,
},
});
});
});

5
packages/core/data-transfer/src/strapi/index.ts Normal file
View File

@ -0,0 +1,5 @@
export * as providers from './providers';
export * as queries from './queries';
export * as remote from './remote';
export { default as register } from './register';

6
packages/core/data-transfer/src/strapi/providers/index.ts Normal file
View File

@ -0,0 +1,6 @@
// Local
export * from './local-destination';
export * from './local-source';
// Remote
export * from './remote-destination';

4
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/assets.test.ts → packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/assets.test.ts
View File

@ -1,8 +1,8 @@
import fse from 'fs-extra';
import { Writable, Readable } from 'stream';
import type { IAsset } from '../../../../types';
import type { IAsset } from '../../../../../types';
import { getStrapiFactory } from '../../../__tests__/test-utils';
import { getStrapiFactory } from '../../../../__tests__/test-utils';
import { createLocalStrapiDestinationProvider } from '../index';
const write = jest.fn((_chunk, _encoding, callback) => {

6
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/index.test.ts → packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/index.test.ts
View File

@ -1,6 +1,10 @@
import { createLocalStrapiDestinationProvider } from '../index';
import * as restoreApi from '../strategies/restore';
import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils';
import {
getStrapiFactory,
getContentTypes,
setGlobalStrapi,
} from '../../../../__tests__/test-utils';
afterEach(() => {
jest.clearAllMocks();

8
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts → packages/core/data-transfer/src/strapi/providers/local-destination/__tests__/restore.test.ts
View File

@ -1,6 +1,10 @@
import { deleteRecords, restoreConfigs } from '../strategies/restore';
import { getStrapiFactory, getContentTypes, setGlobalStrapi } from '../../test-utils';
import { IConfiguration } from '../../../../types';
import {
getStrapiFactory,
getContentTypes,
setGlobalStrapi,
} from '../../../../__tests__/test-utils';
import { IConfiguration } from '../../../../../types';
const entities = [
{

20
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts → packages/core/data-transfer/src/strapi/providers/local-destination/index.ts
View File

@ -1,16 +1,17 @@
import { Writable } from 'stream';
import path from 'path';
import * as fse from 'fs-extra';
import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../types';
import type { IAsset, IDestinationProvider, IMetadata, ProviderType } from '../../../../types';
import { restore } from './strategies';
import * as utils from '../../utils';
import * as utils from '../../../utils';
export const VALID_CONFLICT_STRATEGIES = ['restore', 'merge'];
export const DEFAULT_CONFLICT_STRATEGY = 'restore';
interface ILocalStrapiDestinationProviderOptions {
export interface ILocalStrapiDestinationProviderOptions {
getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>;
autoDestroy?: boolean;
restore?: restore.IRestoreOptions;
strategy: 'restore' | 'merge';
}
@ -24,6 +25,9 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
strapi?: Strapi.Strapi;
/**
* The entities mapper is used to map old entities to their new IDs
*/
#entitiesMapper: { [type: string]: { [id: number]: number } };
constructor(options: ILocalStrapiDestinationProviderOptions) {
@ -37,7 +41,12 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
}
async close(): Promise<void> {
await this.strapi?.destroy?.();
const { autoDestroy } = this.options;
// Basically `!== false` but more deterministic
if (autoDestroy === undefined || autoDestroy === true) {
await this.strapi?.destroy();
}
}
#validateOptions() {
@ -60,7 +69,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
}
}
getMetadata(): IMetadata | Promise<IMetadata> {
getMetadata(): IMetadata {
const strapiVersion = strapi.config.get('info.strapi');
const createdAt = new Date().toISOString();
@ -117,6 +126,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider {
throw new Error(`Invalid strategy supplied: "${strategy}"`);
}
// TODO: Move this logic to the restore strategy
async getAssetsStream(): Promise<Writable> {
if (!this.strapi) {
throw new Error('Not able to stream Assets. Strapi instance not found');

0
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/index.ts → packages/core/data-transfer/src/strapi/providers/local-destination/strategies/index.ts
View File

2
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/configuration.ts → packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/configuration.ts
View File

@ -1,6 +1,6 @@
import { Writable } from 'stream';
import chalk from 'chalk';
import { IConfiguration } from '../../../../../types';
import { IConfiguration } from '../../../../../../types';
const restoreCoreStore = async <T extends { value: unknown }>(strapi: Strapi.Strapi, data: T) => {
return strapi.db.query('strapi::core-store').create({

28
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/entities.ts → packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/entities.ts
View File

@ -3,9 +3,9 @@ import type { SchemaUID } from '@strapi/strapi/lib/types/utils';
import { get } from 'lodash/fp';
import { Writable } from 'stream';
import type { IEntity } from '../../../../../types';
import { json } from '../../../../utils';
import * as shared from '../../../shared';
import type { IEntity } from '../../../../../../types';
import { json } from '../../../../../utils';
import * as queries from '../../../../queries';
interface IEntitiesRestoreStreamOptions {
strapi: Strapi.Strapi;
@ -14,7 +14,7 @@ interface IEntitiesRestoreStreamOptions {
const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
const { strapi, updateMappingTable } = options;
const query = shared.strapi.entity.createEntityQuery(strapi);
const query = queries.entity.createEntityQuery(strapi);
return new Writable({
objectMode: true,
@ -24,13 +24,19 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
const { create, getDeepPopulateComponentLikeQuery } = query(type);
const contentType = strapi.getModel(type);
const resolveType = (paths: string[]) => {
/**
* Resolve the component UID of an entity's attribute based
* on a given path (components & dynamic zones only)
*/
const resolveType = (paths: string[]): string | undefined => {
let cType = contentType;
let value: unknown = data;
for (const path of paths) {
value = get(path, value);
// Needed when the value of cType should be computed
// based on the next value (eg: dynamic zones)
if (typeof cType === 'function') {
cType = cType(value);
}
@ -48,24 +54,32 @@ const createEntitiesWriteStream = (options: IEntitiesRestoreStreamOptions) => {
}
}
return cType.uid;
return cType?.uid;
};
try {
// Create the entity
const created = await create({
data,
populate: getDeepPopulateComponentLikeQuery(contentType, { select: 'id' }),
select: 'id',
});
// Compute differences between original & new entities
const diffs = json.diff(data, created);
updateMappingTable(type, id, created.id);
// For each difference found on an ID attribute,
// update the mapping the table accordingly
diffs.forEach((diff) => {
if (diff.kind === 'modified' && diff.path.at(-1) === 'id') {
const target = resolveType(diff.path);
// If no type is found for the given path, then ignore the diff
if (!target) {
return;
}
const [oldID, newID] = diff.values as [number, number];
updateMappingTable(target, oldID, newID);

4
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/index.ts → packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/index.ts
View File

@ -1,5 +1,5 @@
import type { ContentTypeSchema } from '@strapi/strapi';
import * as shared from '../../../shared';
import * as queries from '../../../../queries';
export interface IRestoreOptions {
assets?: boolean;
@ -36,7 +36,7 @@ const deleteEntitiesRecord = async (
options: IRestoreOptions = {}
): Promise<IDeleteResults> => {
const { entities } = options;
const query = shared.strapi.entity.createEntityQuery(strapi);
const query = queries.entity.createEntityQuery(strapi);
const contentTypes = Object.values<ContentTypeSchema>(strapi.contentTypes);
const contentTypesToClear = contentTypes.filter((contentType) => {

4
packages/core/data-transfer/lib/providers/local-strapi-destination-provider/strategies/restore/links.ts → packages/core/data-transfer/src/strapi/providers/local-destination/strategies/restore/links.ts
View File

@ -1,6 +1,6 @@
import { Writable } from 'stream';
import { ILink } from '../../../../../types';
import { createLinkQuery } from '../../../shared/strapi/link';
import { ILink } from '../../../../../../types';
import { createLinkQuery } from '../../../../queries/link';
export const createLinksWriteStream = (
mapID: (uid: string, id: number) => number | undefined,

6
packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/configuration.test.ts → packages/core/data-transfer/src/strapi/providers/local-source/__tests__/configuration.test.ts
View File

@ -1,6 +1,10 @@
import { Readable } from 'stream';
import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils';
import {
collect,
createMockedQueryBuilder,
getStrapiFactory,
} from '../../../../__tests__/test-utils';
import { createConfigurationStream } from '../configuration';
describe('Configuration', () => {

4
packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts → packages/core/data-transfer/src/strapi/providers/local-source/__tests__/entities.test.ts
View File

@ -1,12 +1,12 @@
import { Readable, PassThrough } from 'stream';
import type { IEntity } from '../../../../types';
import type { IEntity } from '../../../../../types';
import {
collect,
getStrapiFactory,
getContentTypes,
createMockedQueryBuilder,
} from '../../../__tests__/test-utils';
} from '../../../../__tests__/test-utils';
import { createEntitiesStream, createEntitiesTransformStream } from '../entities';
describe('Local Strapi Source Provider - Entities Streaming', () => {

8
packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts → packages/core/data-transfer/src/strapi/providers/local-source/__tests__/index.test.ts
View File

@ -1,7 +1,11 @@
import { Readable } from 'stream';
import type { IEntity } from '../../../../types';
import type { IEntity } from '../../../../../types';
import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils';
import {
collect,
createMockedQueryBuilder,
getStrapiFactory,
} from '../../../../__tests__/test-utils';
import { createLocalStrapiSourceProvider } from '..';
describe('Local Strapi Source Provider', () => {

2
packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts → packages/core/data-transfer/src/strapi/providers/local-source/__tests__/links.test.ts
View File

@ -1,7 +1,7 @@
import { Readable } from 'stream';
import { createLinksStream } from '../links';
import { collect, getStrapiFactory } from '../../../__tests__/test-utils';
import { collect, getStrapiFactory } from '../../../../__tests__/test-utils';
// TODO: entityService needs to be replaced with a mocked wrapper of db.connection and provide real metadata
describe.skip('Local Strapi Source Provider - Entities Streaming', () => {

2
packages/core/data-transfer/lib/providers/local-strapi-source-provider/assets.ts → packages/core/data-transfer/src/strapi/providers/local-source/assets.ts
View File

@ -2,7 +2,7 @@ import { join } from 'path';
import { readdir, stat, createReadStream } from 'fs-extra';
import { Duplex } from 'stream';
import type { IAsset } from '../../../types';
import type { IAsset } from '../../../../types';
const IGNORED_FILES = ['.gitkeep'];

33
packages/core/data-transfer/lib/providers/local-strapi-source-provider/configuration.ts → packages/core/data-transfer/src/strapi/providers/local-source/configuration.ts
View File

@ -2,30 +2,29 @@ import { chain } from 'stream-chain';
import { Readable } from 'stream';
import { set } from 'lodash/fp';
import type { IConfiguration } from '../../../types';
import type { IConfiguration } from '../../../../types';
/**
* Create a readable stream that export the Strapi app configuration
*/
export const createConfigurationStream = (strapi: Strapi.Strapi): Readable => {
// Core Store
const coreStoreStream = chain([
strapi.db.queryBuilder('strapi::core-store').stream(),
(data) => set('value', JSON.parse(data.value), data),
wrapConfigurationItem('core-store'),
]);
// Webhook
const webhooksStream = chain([
strapi.db.queryBuilder('webhook').stream(),
wrapConfigurationItem('webhook'),
]);
const streams = [coreStoreStream, webhooksStream];
// Readable configuration stream
return Readable.from(
(async function* configurationGenerator(): AsyncGenerator<IConfiguration> {
// Core Store
const coreStoreStream = chain([
strapi.db.queryBuilder('strapi::core-store').stream(),
(data) => set('value', JSON.parse(data.value), data),
wrapConfigurationItem('core-store'),
]);
// Webhook
const webhooksStream = chain([
strapi.db.queryBuilder('webhook').stream(),
wrapConfigurationItem('webhook'),
]);
const streams = [coreStoreStream, webhooksStream];
for (const stream of streams) {
for await (const item of stream) {
yield item;

16
packages/core/data-transfer/lib/providers/local-strapi-source-provider/entities.ts → packages/core/data-transfer/src/strapi/providers/local-source/entities.ts
View File

@ -1,8 +1,8 @@
import type { ContentTypeSchema } from '@strapi/strapi';
import { Readable, PassThrough } from 'stream';
import * as shared from '../shared/strapi';
import { IEntity } from '../../../types';
import * as shared from '../../queries';
import { IEntity } from '../../../../types';
/**
* Generate and consume content-types streams in order to stream each entity individually
@ -34,11 +34,15 @@ export const createEntitiesStream = (strapi: Strapi.Strapi): Readable => {
contentType: ContentTypeSchema;
}> {
for await (const { stream, contentType } of contentTypeStreamGenerator()) {
for await (const entity of stream) {
yield { entity, contentType };
try {
for await (const entity of stream) {
yield { entity, contentType };
}
} catch {
// ignore
} finally {
stream.destroy();
}
stream.destroy();
}
})()
);

4
packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts → packages/core/data-transfer/src/strapi/providers/local-source/index.ts
View File

@ -1,12 +1,12 @@
import { chain } from 'stream-chain';
import { Readable } from 'stream';
import type { IMetadata, ISourceProvider, ProviderType } from '../../../types';
import type { IMetadata, ISourceProvider, ProviderType } from '../../../../types';
import { createEntitiesStream, createEntitiesTransformStream } from './entities';
import { createLinksStream } from './links';
import { createConfigurationStream } from './configuration';
import { createAssetsStream } from './assets';
import * as utils from '../../utils';
import * as utils from '../../../utils';
export interface ILocalStrapiSourceProviderOptions {
getStrapi(): Strapi.Strapi | Promise<Strapi.Strapi>;

4
packages/core/data-transfer/lib/providers/local-strapi-source-provider/links.ts → packages/core/data-transfer/src/strapi/providers/local-source/links.ts
View File

@ -1,7 +1,7 @@
import { Readable } from 'stream';
import type { ILink } from '../../../types';
import { createLinkQuery } from '../shared/strapi/link';
import type { ILink } from '../../../../types';
import { createLinkQuery } from '../../queries/link';
/**
* Create a Readable which will stream all the links from a Strapi instance

54
packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/index.test.ts Normal file
View File

@ -0,0 +1,54 @@
import { WebSocket } from 'ws';
import type { IRemoteStrapiDestinationProviderOptions } from '..';
import { createRemoteStrapiDestinationProvider } from '..';
const defaultOptions: IRemoteStrapiDestinationProviderOptions = {
strategy: 'restore',
url: '<some_url>',
};
jest.mock('../utils', () => ({
createDispatcher: jest.fn(),
}));
jest.mock('ws', () => ({
WebSocket: jest.fn().mockImplementation(() => {
return {
...jest.requireActual('ws').WebSocket,
send: jest.fn(),
once: jest.fn((type, callback) => {
callback();
return {
once: jest.fn((t, c) => c),
};
}),
};
}),
}));
afterEach(() => {
jest.clearAllMocks();
});
describe('Remote Strapi Destination', () => {
describe('Bootstrap', () => {
test('Should not have a defined websocket connection if bootstrap has not been called', () => {
const provider = createRemoteStrapiDestinationProvider(defaultOptions);
expect(provider.ws).toBeNull();
});
test('Should have a defined websocket connection if bootstrap has been called', async () => {
const provider = createRemoteStrapiDestinationProvider(defaultOptions);
try {
await provider.bootstrap();
} catch {
// ignore ws connection error
}
expect(provider.ws).not.toBeNull();
expect(provider.ws?.readyState).toBe(WebSocket.CLOSED);
});
});
});

45
packages/core/data-transfer/src/strapi/providers/remote-destination/__tests__/utils.test.ts Normal file
View File

@ -0,0 +1,45 @@
import { WebSocket } from 'ws';
import { CommandMessage } from '../../../../../types/remote/protocol/client';
import { createDispatcher } from '../utils';
jest.mock('ws', () => ({
WebSocket: jest.fn().mockImplementation(() => {
return {
...jest.requireActual('ws').WebSocket,
send: jest.fn(),
once: jest.fn(),
};
}),
}));
afterEach(() => {
jest.clearAllMocks();
});
describe('Remote Strapi Destination Utils', () => {
test('Dispatch method sends payload', () => {
const ws = new WebSocket('ws://test/admin/transfer');
const message: CommandMessage = {
type: 'command',
command: 'status',
};
createDispatcher(ws).dispatch(message);
expect.extend({
toContain(receivedString, expected) {
const jsonReceived = JSON.parse(receivedString);
const pass = Object.keys(expected).every((key) => jsonReceived[key] === expected[key]);
return {
message: () =>
`Expected ${jsonReceived} ${!pass && 'not'} to contain properties ${expected}`,
pass,
};
},
});
// @ts-ignore
expect(ws.send).toHaveBeenCalledWith(expect.toContain(message), expect.anything());
});
});

236
packages/core/data-transfer/src/strapi/providers/remote-destination/index.ts Normal file
View File

@ -0,0 +1,236 @@
import { WebSocket } from 'ws';
import { v4 } from 'uuid';
import { Writable } from 'stream';
import { createDispatcher } from './utils';
import type {
IDestinationProvider,
IEntity,
ILink,
IMetadata,
ProviderType,
IConfiguration,
IAsset,
} from '../../../../types';
import type { client, server } from '../../../../types/remote/protocol';
import type { ILocalStrapiDestinationProviderOptions } from '../local-destination';
interface ITokenAuth {
type: 'token';
token: string;
}
interface ICredentialsAuth {
type: 'credentials';
email: string;
password: string;
}
export interface IRemoteStrapiDestinationProviderOptions
extends Pick<ILocalStrapiDestinationProviderOptions, 'restore' | 'strategy'> {
url: string;
auth?: ITokenAuth | ICredentialsAuth;
}
class RemoteStrapiDestinationProvider implements IDestinationProvider {
name = 'destination::remote-strapi';
type: ProviderType = 'destination';
options: IRemoteStrapiDestinationProviderOptions;
ws: WebSocket | null;
dispatcher: ReturnType<typeof createDispatcher> | null;
constructor(options: IRemoteStrapiDestinationProviderOptions) {
this.options = options;
this.ws = null;
this.dispatcher = null;
}
async initTransfer(): Promise<string> {
const { strategy, restore } = this.options;
// Wait for the connection to be made to the server, then init the transfer
return new Promise<string>((resolve, reject) => {
this.ws
?.once('open', async () => {
const query = this.dispatcher?.dispatchCommand({
command: 'init',
params: { options: { strategy, restore }, transfer: 'push' },
});
const res = (await query) as server.Payload<server.InitMessage>;
if (!res?.transferID) {
return reject(new Error('Init failed, invalid response from the server'));
}
resolve(res.transferID);
})
.once('error', reject);
});
}
async #streamStep<T extends client.TransferPushStep>(
step: T,
data: client.GetTransferPushStreamData<T>
) {
try {
await this.dispatcher?.dispatchTransferStep({ action: 'stream', step, data });
} catch (e) {
if (e instanceof Error) {
return e;
}
if (typeof e === 'string') {
return new Error(e);
}
return new Error('Unexpected error');
}
return null;
}
async bootstrap(): Promise<void> {
const { url, auth } = this.options;
let ws: WebSocket;
// No auth defined, trying public access for transfer
if (!auth) {
ws = new WebSocket(url);
}
// Common token auth, this should be the main auth method
else if (auth.type === 'token') {
const headers = { Authentication: `Bearer ${auth.token}` };
ws = new WebSocket(this.options.url, { headers });
}
// Invalid auth method provided
else {
throw new Error('Auth method not implemented');
}
this.ws = ws;
this.dispatcher = createDispatcher(this.ws);
const transferID = await this.initTransfer();
this.dispatcher.setTransferProperties({ id: transferID, kind: 'push' });
await this.dispatcher.dispatchTransferAction('bootstrap');
}
async close() {
await this.dispatcher?.dispatchTransferAction('close');
await new Promise<void>((resolve) => {
const { ws } = this;
if (!ws || ws.CLOSED) {
resolve();
return;
}
ws.on('close', () => resolve()).close();
});
}
getMetadata() {
return this.dispatcher?.dispatchTransferAction<IMetadata>('getMetadata') ?? null;
}
async beforeTransfer() {
await this.dispatcher?.dispatchTransferAction('beforeTransfer');
}
getSchemas(): Promise<Strapi.Schemas | null> {
if (!this.dispatcher) {
return Promise.resolve(null);
}
return this.dispatcher.dispatchTransferAction<Strapi.Schemas>('getSchemas');
}
getEntitiesStream(): Writable {
return new Writable({
objectMode: true,
write: async (entity: IEntity, _encoding, callback) => {
const e = await this.#streamStep('entities', entity);
callback(e);
},
});
}
getLinksStream(): Writable {
return new Writable({
objectMode: true,
write: async (link: ILink, _encoding, callback) => {
const e = await this.#streamStep('links', link);
callback(e);
},
});
}
getConfigurationStream(): Writable {
return new Writable({
objectMode: true,
write: async (configuration: IConfiguration, _encoding, callback) => {
const e = await this.#streamStep('configuration', configuration);
callback(e);
},
});
}
getAssetsStream(): Writable | Promise<Writable> {
return new Writable({
objectMode: true,
final: async (callback) => {
// TODO: replace this stream call by an end call
const e = await this.#streamStep('assets', null);
callback(e);
},
write: async (asset: IAsset, _encoding, callback) => {
const { filename, filepath, stats, stream } = asset;
const assetID = v4();
await this.#streamStep('assets', {
action: 'start',
assetID,
data: { filename, filepath, stats },
});
for await (const chunk of stream) {
await this.#streamStep('assets', {
action: 'stream',
assetID,
data: chunk,
});
}
await this.#streamStep('assets', {
action: 'end',
assetID,
});
callback();
},
});
}
}
export const createRemoteStrapiDestinationProvider = (
options: IRemoteStrapiDestinationProviderOptions
) => {
return new RemoteStrapiDestinationProvider(options);
};

123
packages/core/data-transfer/src/strapi/providers/remote-destination/utils.ts Normal file
View File

@ -0,0 +1,123 @@
import { set } from 'lodash/fp';
import { v4 } from 'uuid';
import { RawData, WebSocket } from 'ws';
import type { client, server } from '../../../../types/remote/protocol';
interface IDispatcherState {
transfer?: { kind: client.TransferKind; id: string };
}
interface IDispatchOptions {
attachTransfer?: boolean;
}
type Dispatch<T> = Omit<T, 'transferID' | 'uuid'>;
const createDispatcher = (ws: WebSocket) => {
const state: IDispatcherState = {};
type DispatchMessage = Dispatch<client.Message>;
const dispatch = async <U = null>(
message: DispatchMessage,
options: IDispatchOptions = {}
): Promise<U | null> => {
if (!ws) {
throw new Error('No websocket connection found');
}
return new Promise<U | null>((resolve, reject) => {
const uuid = v4();
const payload = { ...message, uuid };
if (options.attachTransfer) {
Object.assign(payload, { transferID: state.transfer?.id });
}
const stringifiedPayload = JSON.stringify(payload);
ws.send(stringifiedPayload, (error) => {
if (error) {
reject(error);
}
});
const onResponse = (raw: RawData) => {
const response: server.Message<U> = JSON.parse(raw.toString());
if (response.uuid === uuid) {
if (response.error) {
return reject(new Error(response.error.message));
}
resolve(response.data ?? null);
} else {
ws.once('message', onResponse);
}
};
// TODO: What happens if the server sends another message (not a response to this message)
ws.once('message', onResponse);
});
};
const dispatchCommand = <U extends client.Command>(
payload: {
command: U;
} & ([client.GetCommandParams<U>] extends [never]
? unknown
: { params: client.GetCommandParams<U> })
) => {
return dispatch({ type: 'command', ...payload } as client.CommandMessage);
};
const dispatchTransferAction = async <T>(action: client.Action['action']) => {
const payload: Dispatch<client.Action> = { type: 'transfer', kind: 'action', action };
return dispatch<T>(payload, { attachTransfer: true }) ?? Promise.resolve(null);
};
const dispatchTransferStep = async <
T,
A extends client.TransferPushMessage['action'] = client.TransferPushMessage['action'],
S extends client.TransferPushStep = client.TransferPushStep
>(
payload: {
step: S;
action: A;
} & (A extends 'stream' ? { data: client.GetTransferPushStreamData<S> } : unknown)
) => {
const message: Dispatch<client.TransferPushMessage> = {
type: 'transfer',
kind: 'step',
...payload,
};
return dispatch<T>(message, { attachTransfer: true }) ?? Promise.resolve(null);
};
const setTransferProperties = (
properties: Exclude<IDispatcherState['transfer'], undefined>
): void => {
state.transfer = { ...properties };
};
return {
get transferID() {
return state.transfer?.id;
},
get transferKind() {
return state.transfer?.kind;
},
setTransferProperties,
dispatch,
dispatchCommand,
dispatchTransferAction,
dispatchTransferStep,
};
};
export { createDispatcher };

0
packages/core/data-transfer/lib/providers/shared/strapi/entity.ts → packages/core/data-transfer/src/strapi/queries/entity.ts
View File

0
packages/core/data-transfer/lib/providers/shared/strapi/index.ts → packages/core/data-transfer/src/strapi/queries/index.ts
View File

3
packages/core/data-transfer/lib/providers/shared/strapi/link.ts → packages/core/data-transfer/src/strapi/queries/link.ts
View File

@ -1,5 +1,6 @@
import { RelationAttribute } from '@strapi/strapi';
import { clone, isNil } from 'lodash/fp';
import { ILink } from '../../../../types';
import { ILink } from '../../../types';
// TODO: Remove any types when we'll have types for DB metadata

12
packages/core/data-transfer/src/strapi/register.ts Normal file
View File

@ -0,0 +1,12 @@
import { routes } from './remote';
/**
* This is intended to be called on Strapi register phase.
*
* It registers a transfer route in the Strapi admin router.
*/
const register = (strapi: Strapi.Strapi) => {
routes.registerAdminTransferRoute(strapi);
};
export default register;

1
packages/core/data-transfer/src/strapi/remote/constants.ts Normal file
View File

@ -0,0 +1 @@
export const TRANSFER_URL = '/transfer';

1
packages/core/data-transfer/src/strapi/remote/controllers/index.ts Normal file
View File

@ -0,0 +1 @@
export * from './push';

142
packages/core/data-transfer/src/strapi/remote/controllers/push.ts Normal file
View File

@ -0,0 +1,142 @@
import { PassThrough, Writable } from 'stream-chain';
import type { IAsset, IMetadata } from '../../../../types';
import type {
TransferPushMessage,
TransferPushStep,
} from '../../../../types/remote/protocol/client';
import {
createLocalStrapiDestinationProvider,
ILocalStrapiDestinationProviderOptions,
} from '../../providers';
export interface IPushController {
streams: { [stage in TransferPushStep]?: Writable };
actions: {
getMetadata(): Promise<IMetadata>;
getSchemas(): Strapi.Schemas;
bootstrap(): Promise<void>;
close(): Promise<void>;
beforeTransfer(): Promise<void>;
};
transfer: {
[key in TransferPushStep]: <T extends TransferPushMessage>(
value: T extends { step: key; data: infer U } ? U : never
) => Promise<void>;
};
}
const createPushController = (options: ILocalStrapiDestinationProviderOptions): IPushController => {
const provider = createLocalStrapiDestinationProvider(options);
const streams: { [stage in TransferPushStep]?: Writable } = {};
const assets: { [filepath: string]: IAsset & { stream: PassThrough } } = {};
const writeAsync = <T>(stream: Writable, data: T) => {
return new Promise<void>((resolve, reject) => {
stream.write(data, (error) => {
if (error) {
reject(error);
}
resolve();
});
});
};
return {
streams,
actions: {
async getSchemas(): Promise<Strapi.Schemas> {
return provider.getSchemas();
},
async getMetadata() {
return provider.getMetadata();
},
async bootstrap() {
return provider.bootstrap();
},
async close() {
return provider.close();
},
async beforeTransfer() {
return provider.beforeTransfer();
},
},
transfer: {
async entities(entity) {
if (!streams.entities) {
streams.entities = provider.getEntitiesStream();
}
await writeAsync(streams.entities, entity);
},
async links(link) {
if (!streams.links) {
streams.links = await provider.getLinksStream();
}
await writeAsync(streams.links, link);
},
async configuration(config) {
if (!streams.configuration) {
streams.configuration = await provider.getConfigurationStream();
}
await writeAsync(streams.configuration, config);
},
async assets(payload) {
// TODO: close the stream upong receiving an 'end' event instead
if (payload === null) {
streams.assets?.end();
return;
}
const { action, assetID } = payload;
if (!streams.assets) {
streams.assets = await provider.getAssetsStream();
}
if (action === 'start') {
assets[assetID] = { ...payload.data, stream: new PassThrough() };
writeAsync(streams.assets, assets[assetID]);
}
if (action === 'stream') {
// The buffer has gone through JSON operations and is now of shape { type: "Buffer"; data: UInt8Array }
// We need to transform it back into a Buffer instance
const rawBuffer = payload.data as unknown as { type: 'Buffer'; data: Uint8Array };
const chunk = Buffer.from(rawBuffer.data);
await writeAsync(assets[assetID].stream, chunk);
}
if (action === 'end') {
await new Promise<void>((resolve, reject) => {
const { stream } = assets[assetID];
stream
.on('close', () => {
delete assets[assetID];
resolve();
})
.on('error', reject)
.end();
});
}
},
},
};
};
export default createPushController;

221
packages/core/data-transfer/src/strapi/remote/handlers.ts Normal file
View File

@ -0,0 +1,221 @@
import type { Context } from 'koa';
import type { ServerOptions } from 'ws';
import { randomUUID } from 'crypto';
import { WebSocket } from 'ws';
import type { IPushController } from './controllers/push';
import createPushController from './controllers/push';
import type { client, server } from '../../../types/remote/protocol';
interface ITransferState {
transfer?: { id: string; kind: client.TransferKind };
controller?: IPushController;
}
export const createTransferHandler =
(options: ServerOptions = {}) =>
async (ctx: Context) => {
const upgradeHeader = (ctx.request.headers.upgrade || '')
.split(',')
.map((s) => s.trim().toLowerCase());
// Create the websocket server
const wss = new WebSocket.Server({ ...options, noServer: true });
if (upgradeHeader.includes('websocket')) {
wss.handleUpgrade(ctx.req, ctx.request.socket, Buffer.alloc(0), (ws) => {
// Create a connection between the client & the server
wss.emit('connection', ws, ctx.req);
const state: ITransferState = {};
let uuid: string | undefined;
/**
* Format error & message to follow the remote transfer protocol
*/
const callback = <T = unknown>(e: Error | null = null, data?: T) => {
return new Promise<void>((resolve, reject) => {
if (!uuid) {
reject(new Error('Missing uuid for this message'));
return;
}
const payload = JSON.stringify({
uuid,
data: data ?? null,
error: e
? {
code: 'ERR',
message: e?.message,
}
: null,
});
ws.send(payload, (error) => (error ? reject(error) : resolve()));
});
};
/**
* Wrap a function call to catch errors and answer the request with the correct format
*/
const answer = async <T = unknown>(fn: () => T) => {
try {
const response = await fn();
callback(null, response);
} catch (e) {
if (e instanceof Error) {
callback(e);
} else if (typeof e === 'string') {
callback(new Error(e));
} else {
callback(new Error('Unexpected error'));
}
}
};
const teardown = (): server.Payload<server.EndMessage> => {
delete state.controller;
delete state.transfer;
return { ok: true };
};
const init = (msg: client.InitCommand): server.Payload<server.InitMessage> => {
if (state.controller) {
throw new Error('Transfer already in progres');
}
const { transfer } = msg.params;
// Push transfer
if (transfer === 'push') {
const { options: controllerOptions } = msg.params;
state.controller = createPushController({
...controllerOptions,
autoDestroy: false,
getStrapi: () => strapi,
});
}
// Pull or any other string
else {
throw new Error(`Transfer not implemented: "${transfer}"`);
}
state.transfer = { id: randomUUID(), kind: transfer };
return { transferID: state.transfer.id };
};
/**
* On command message (init, end, status, ...)
*/
const onCommand = async (msg: client.CommandMessage) => {
const { command } = msg;
if (command === 'init') {
await answer(() => init(msg));
}
if (command === 'end') {
await answer(teardown);
}
if (command === 'status') {
await callback(new Error('Command not implemented: "status"'));
}
};
const onTransferCommand = async (msg: client.TransferMessage) => {
const { transferID, kind } = msg;
const { controller } = state;
// TODO: (re)move this check
// It shouldn't be possible to strart a pull transfer for now, so reaching
// this code should be impossible too, but this has been added by security
if (state.transfer?.kind === 'pull') {
return callback(new Error('Pull transfer not implemented'));
}
if (!controller) {
return callback(new Error("The transfer hasn't been initialized"));
}
if (!transferID) {
return callback(new Error('Missing transfer ID'));
}
// Action
if (kind === 'action') {
const { action } = msg;
if (!(action in controller.actions)) {
return callback(new Error(`Invalid action provided: "${action}"`));
}
await answer(() => controller.actions[action as keyof typeof controller.actions]());
}
// Transfer
else if (kind === 'step') {
// We can only have push transfer message for the moment
const message = msg as client.TransferPushMessage;
// TODO: lock transfer process
if (message.action === 'start') {
// console.log('Starting transfer for ', message.step);
}
// Stream step
else if (message.action === 'stream') {
await answer(() => controller.transfer[message.step]?.(message.data as never));
}
// TODO: unlock transfer process
else if (message.action === 'end') {
// console.log('Ending transfer for ', message.step);
}
}
};
ws.on('close', () => {
teardown();
});
ws.on('error', (e) => {
teardown();
console.error(e);
});
ws.on('message', async (raw) => {
const msg: client.Message = JSON.parse(raw.toString());
if (!msg.uuid) {
throw new Error('Missing uuid in message');
}
uuid = msg.uuid;
// Regular command message (init, end, status)
if (msg.type === 'command') {
await onCommand(msg);
}
// Transfer message (the transfer must be initialized first)
else if (msg.type === 'transfer') {
await onTransferCommand(msg);
}
// Invalid messages
else {
await callback(new Error('Bad request'));
}
});
});
ctx.respond = false;
}
};

3
packages/core/data-transfer/src/strapi/remote/index.ts Normal file
View File

@ -0,0 +1,3 @@
export * as controllers from './controllers';
export * as routes from './routes';
export * as constants from './constants';

36
packages/core/data-transfer/src/strapi/remote/routes.ts Normal file
View File

@ -0,0 +1,36 @@
// eslint-disable-next-line node/no-extraneous-import
import type { Context } from 'koa';
import { TRANSFER_URL } from './constants';
import { createTransferHandler } from './handlers';
// Extend Strapi interface type to access the admin routes' API
// TODO: Remove this when the Strapi instances will be better typed
declare module '@strapi/strapi' {
interface Strapi {
admin: {
routes: {
method: string;
path: string;
handler: (ctx: Context) => Promise<void>;
config: unknown;
}[];
};
}
}
/**
* Register a transfer route in the Strapi admin router.
*
* It exposes a WS server that can be used to run and manage transfer processes.
*
* @param strapi - A Strapi instance
*/
export const registerAdminTransferRoute = (strapi: Strapi.Strapi) => {
strapi.admin.routes.push({
method: 'GET',
path: TRANSFER_URL,
handler: createTransferHandler(),
config: { auth: false },
});
};

0
packages/core/data-transfer/lib/encryption/__tests__/encrypt.test.ts → packages/core/data-transfer/src/utils/encryption/__tests__/encrypt.test.ts
View File

10
packages/core/data-transfer/lib/encryption/decrypt.ts → packages/core/data-transfer/src/utils/encryption/decrypt.ts
View File

@ -1,5 +1,5 @@
import { Cipher, scryptSync, CipherKey, BinaryLike, createDecipheriv } from 'crypto';
import { EncryptionStrategy, Strategies, Algorithm } from '../../types';
import { EncryptionStrategy, Strategies, Algorithm } from '../../../types';
// different key values depending on algorithm chosen
const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
@ -33,6 +33,14 @@ const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
return strategies[algorithm];
};
/**
* It creates a cipher instance used for decryption
*
* @param key - The decryption key
* @param algorithm - The algorithm to use to create the Cipher
*
* @returns A {@link Cipher} instance created with the given key & algorithm
*/
export const createDecryptionCipher = (
key: string,
algorithm: Algorithm = 'aes-128-ecb'

10
packages/core/data-transfer/lib/encryption/encrypt.ts → packages/core/data-transfer/src/utils/encryption/encrypt.ts
View File

@ -1,5 +1,5 @@
import { createCipheriv, Cipher, scryptSync, CipherKey, BinaryLike } from 'crypto';
import { EncryptionStrategy, Strategies, Algorithm } from '../../types';
import { EncryptionStrategy, Strategies, Algorithm } from '../../../types';
// different key values depending on algorithm chosen
const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
@ -33,6 +33,14 @@ const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => {
return strategies[algorithm];
};
/**
* It creates a cipher instance used for encryption
*
* @param key - The encryption key
* @param algorithm - The algorithm to use to create the Cipher
*
* @returns A {@link Cipher} instance created with the given key & algorithm
*/
export const createEncryptionCipher = (
key: string,
algorithm: Algorithm = 'aes-128-ecb'

0
packages/core/data-transfer/lib/encryption/index.ts → packages/core/data-transfer/src/utils/encryption/index.ts
View File

1
packages/core/data-transfer/lib/utils/index.ts → packages/core/data-transfer/src/utils/index.ts
View File

@ -1,3 +1,4 @@
export * as encryption from './encryption';
export * as stream from './stream';
export * as json from './json';
export * as schema from './schema';

9
packages/core/data-transfer/lib/utils/json.ts → packages/core/data-transfer/src/utils/json.ts
View File

@ -2,6 +2,13 @@ import { isArray, isObject, zip, isEqual, uniq } from 'lodash/fp';
const createContext = (): Context => ({ path: [] });
/**
* Compute differences between two JSON objects and returns them
*
* @param a - First object
* @param b - Second object
* @param ctx - Context used to keep track of the current path during recursion
*/
export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Diff[] => {
const diffs: Diff[] = [];
const { path } = ctx;
@ -70,7 +77,7 @@ export const diff = (a: unknown, b: unknown, ctx: Context = createContext()): Di
}
if (!isEqual(a, b)) {
modified();
return modified();
}
return diffs;

27
packages/core/data-transfer/src/utils/schema.ts Normal file
View File

@ -0,0 +1,27 @@
import type { Schema } from '@strapi/strapi';
import { mapValues, pick } from 'lodash/fp';
/**
* List of schema properties that should be kept when sanitizing schemas
*/
const VALID_SCHEMA_PROPERTIES = [
'collectionName',
'info',
'options',
'pluginOptions',
'attributes',
'kind',
'modelType',
'modelName',
'uid',
'plugin',
'globalId',
];
/**
* Sanitize a schemas dictionnary by omiting unwanted properties
* The list of allowed properties can be found here: {@link VALID_SCHEMA_PROPERTIES}
*/
export const mapSchemasValues = (schemas: Record<string, Schema>) => {
return mapValues(pick(VALID_SCHEMA_PROPERTIES), schemas);
};

72
packages/core/data-transfer/src/utils/stream.ts Normal file
View File

@ -0,0 +1,72 @@
import { Transform, Readable } from 'stream';
type TransformOptions = ConstructorParameters<typeof Transform>[0];
/**
* Create a filter stream that discard chunks which doesn't satisfies the given predicate
*
* @param predicate - A filter predicate, takes a stream data chunk as parameter and returns a boolean value
* @param options - Transform stream options
*/
export const filter = <T>(
predicate: (value: T) => boolean | Promise<boolean>,
options: TransformOptions = { objectMode: true }
): Transform => {
return new Transform({
...options,
async transform(chunk, _encoding, callback) {
const keep = await predicate(chunk);
callback(null, keep ? chunk : undefined);
},
});
};
/**
* Create a map stream that transform chunks using the given predicate
*
* @param predicate - A map predicate, takes a stream data chunk as parameter and returns a mapped value
* @param options - Transform stream options
*/
export const map = <T, U = T>(
predicate: (value: T) => U | Promise<U>,
options: TransformOptions = { objectMode: true }
): Transform => {
return new Transform({
...options,
async transform(chunk, _encoding, callback) {
const mappedValue = await predicate(chunk);
callback(null, mappedValue);
},
});
};
/**
* Collect every chunks from a Readable stream.
*
* @param stream - The redable stream to collect data from
* @param options.destroy - If set to true, it automatically calls `destroy()` on the given stream upon receiving the 'end' event
*/
export const collect = <T = unknown>(
stream: Readable,
options: { destroy: boolean } = { destroy: true }
): Promise<T[]> => {
const chunks: T[] = [];
return new Promise((resolve, reject) => {
stream
.on('close', () => resolve(chunks))
.on('error', reject)
.on('data', (chunk) => chunks.push(chunk))
.on('end', () => {
if (options.destroy) {
stream.destroy();
}
resolve(chunks);
});
});
};

6
packages/core/data-transfer/tsconfig.json
View File

@ -4,11 +4,11 @@
"strict": true,
"lib": ["ESNEXT"],
"skipLibCheck": true,
"outDir": "dist",
"outDir": "lib",
"declaration": true,
"sourceMap": true,
"esModuleInterop": true
},
"include": ["types", "lib/**/*.ts"],
"exclude": ["node_modules", "lib/**/__tests__"]
"include": ["types", "src/**/*.ts"],
"exclude": ["node_modules", "src/**/__tests__"]
}

1
packages/core/data-transfer/types/index.d.ts vendored
View File

@ -3,3 +3,4 @@ export * from './providers';
export * from './transfer-engine';
export * from './utils';
export * from './encryption';
export * from './remote';

1
packages/core/data-transfer/types/remote/index.d.ts vendored Normal file
View File

@ -0,0 +1 @@
export * as protocol from './protocol';

30
packages/core/data-transfer/types/remote/protocol/client/commands.d.ts vendored Normal file
View File

@ -0,0 +1,30 @@
import type { ILocalStrapiDestinationProviderOptions } from '../../../../src/strapi/providers';
export type CommandMessage = { type: 'command' } & (InitCommand | EndCommand | StatusCommand);
export type Command = CommandMessage['command'];
export type GetCommandParams<T extends Command> = {
[key in Command]: { command: key } & CommandMessage;
}[T] extends { params: infer U }
? U
: never;
export type InitCommand = CreateCommand<
'init',
| {
transfer: 'push';
options: Pick<ILocalStrapiDestinationProviderOptions, 'strategy' | 'restore'>;
}
| { transfer: 'pull' }
>;
export type TransferKind = InitCommand['params']['transfer'];
export type EndCommand = CreateCommand<'end', { transferID: string }>;
export type StatusCommand = CreateCommand<'status'>;
type CreateCommand<T extends string, U extends Record<string, unknown> = never> = {
type: 'command';
command: T;
} & ([U] extends [never] ? unknown : { params: U });

8
packages/core/data-transfer/types/remote/protocol/client/index.d.ts vendored Normal file
View File

@ -0,0 +1,8 @@
import type { CommandMessage } from './commands';
import type { TransferMessage } from './transfer';
export * from './commands';
export * from './transfer';
export type Message = { uuid: string } & (CommandMessage | TransferMessage);
export type MessageType = Message['type'];

3
packages/core/data-transfer/types/remote/protocol/client/transfer/action.d.ts vendored Normal file
View File

@ -0,0 +1,3 @@
import type { CreateTransferMessage } from './utils';
export type Action = CreateTransferMessage<'action', { action: string }>;

13
packages/core/data-transfer/types/remote/protocol/client/transfer/index.d.ts vendored Normal file
View File

@ -0,0 +1,13 @@
import type { Action } from './action';
import type { TransferPullMessage } from './pull';
import type { TransferPushMessage } from './push';
export * from './action';
export * from './pull';
export * from './push';
export type TransferMessage = { type: 'transfer'; transferID: string } & (
| Action
| TransferPushMessage
| TransferPullMessage
);

8
packages/core/data-transfer/types/remote/protocol/client/transfer/pull.d.ts vendored Normal file
View File

@ -0,0 +1,8 @@
import { CreateTransferMessage } from './utils';
export type TransferPullMessage = CreateTransferMessage<
'step',
{
action: 'start' | 'stop';
}
>;

31
packages/core/data-transfer/types/remote/protocol/client/transfer/push.d.ts vendored Normal file
View File

@ -0,0 +1,31 @@
import type { CreateTransferMessage } from './utils';
import type { IEntity, ILink, IConfiguration, IAsset } from '../../../../common-entities';
export type TransferPushMessage = CreateTransferMessage<
'step',
| TransferStepCommands<'entities', IEntity>
| TransferStepCommands<'links', ILink>
| TransferStepCommands<'configuration', IConfiguration>
| TransferStepCommands<'assets', TransferAssetFlow | null>
>;
export type GetTransferPushStreamData<T extends TransferPushStep> = {
[key in TransferPushStep]: {
action: 'stream';
step: key;
} & TransferPushMessage;
}[T] extends { data: infer U }
? U
: never;
export type TransferPushStep = TransferPushMessage['step'];
type TransferStepCommands<T extends string, U> = { step: T } & TransferStepFlow<U>;
type TransferStepFlow<U> = { action: 'start' } | { action: 'stream'; data: U } | { action: 'end' };
type TransferAssetFlow = { assetID: string } & (
| { action: 'start'; data: Omit<IAsset, 'stream'> }
| { action: 'stream'; data: Buffer }
| { action: 'end' }
);

5
packages/core/data-transfer/types/remote/protocol/client/transfer/utils.d.ts vendored Normal file
View File

@ -0,0 +1,5 @@
export type CreateTransferMessage<T extends string, U = unknown> = {
type: 'transfer';
kind: T;
transferID: string;
} & U;

2
packages/core/data-transfer/types/remote/protocol/index.d.ts vendored Normal file
View File

@ -0,0 +1,2 @@
export * as client from './client';
export * as server from './server';

29
packages/core/data-transfer/types/remote/protocol/server/error.d.ts vendored Normal file
View File

@ -0,0 +1,29 @@
export enum ErrorKind {
// Generic
Unknown = 0,
// Chunk transfer
DiscardChunk = 1,
InvalidChunkFormat = 2,
}
export class ServerError extends Error {
constructor(
public code: ErrorKind,
public message: string,
public details?: Record<string, unknown> | null
) {
super(message);
}
}
export class UnknownError extends ServerError {
constructor(message: string, details?: Record<string, unknown> | null) {
super(ErrorKind.Unknown, message, details);
}
}
export class DiscardChunkError extends ServerError {
constructor(message: string, details?: Record<string, unknown> | null) {
super(ErrorKind.DiscardChunk, message, details);
}
}

2
packages/core/data-transfer/types/remote/protocol/server/index.d.ts vendored Normal file
View File

@ -0,0 +1,2 @@
export * from './messaging';
export * as error from './error';

14
packages/core/data-transfer/types/remote/protocol/server/messaging.d.ts vendored Normal file
View File

@ -0,0 +1,14 @@
import type { ServerError } from './error';
export type Message<T = unknown> = {
uuid?: string;
data?: T | null;
error?: ServerError | null;
};
// Successful
export type OKMessage = Message<{ ok: true }>;
export type InitMessage = Message<{ transferID: string }>;
export type EndMessage = OKMessage;
export type Payload<T extends Message> = T['data'];

17
packages/core/strapi/bin/strapi.js
View File

@ -258,6 +258,23 @@ program
.option('-s, --silent', `Run the generation silently, without any output`, false)
.action(getLocalScript('ts/generate-types'));
// `$ strapi transfer`
program
.command('transfer')
.description('Transfer data from one source to another')
.addOption(new Option('--from <sourceURL>', `URL of remote Strapi instance to get data from.`))
.addOption(new Option('--to <destinationURL>', `URL of remote Strapi instance to send data to`))
.hook('preAction', async (thisCommand) => {
const opts = thisCommand.opts();
if (!opts.from && !opts.to) {
console.error('At least one source (from) or destination (to) option must be provided');
process.exit(1);
}
})
.allowExcessArguments(false)
.action(getLocalScript('transfer/transfer'));
// `$ strapi export`
program
.command('export')

64
packages/core/strapi/lib/commands/__tests__/export.test.js → packages/core/strapi/lib/commands/__tests__/data-transfer/export.test.js
View File

@ -1,33 +1,39 @@
'use strict';
describe('export', () => {
describe('Export', () => {
const defaultFileName = 'defaultFilename';
// mock @strapi/data-transfer
const mockDataTransfer = {
createLocalFileDestinationProvider: jest.fn().mockReturnValue({ name: 'testDest' }),
createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testSource' }),
createTransferEngine() {
return {
transfer: jest.fn().mockReturnValue(Promise.resolve({})),
progress: {
on: jest.fn(),
stream: {
file: {
providers: {
createLocalFileDestinationProvider: jest.fn().mockReturnValue({ name: 'testDest' }),
},
},
strapi: {
providers: {
createLocalStrapiSourceProvider: jest.fn().mockReturnValue({ name: 'testSource' }),
},
},
engine: {
createTransferEngine() {
return {
transfer: jest.fn().mockReturnValue(Promise.resolve({})),
progress: {
on: jest.fn(),
stream: {
on: jest.fn(),
},
},
},
sourceProvider: { name: 'testSource' },
destinationProvider: { name: 'testDestination' },
};
sourceProvider: { name: 'testSource' },
destinationProvider: { name: 'testDestination' },
};
},
},
};
jest.mock(
'@strapi/data-transfer',
() => {
return mockDataTransfer;
},
{ virtual: true }
);
jest.mock('@strapi/data-transfer/lib/engine', () => mockDataTransfer.engine, { virtual: true });
jest.mock('@strapi/data-transfer/lib/strapi', () => mockDataTransfer.strapi, { virtual: true });
jest.mock('@strapi/data-transfer/lib/file', () => mockDataTransfer.file, { virtual: true });
// mock utils
const mockUtils = {
@ -41,7 +47,7 @@ describe('export', () => {
getDefaultExportName: jest.fn(() => defaultFileName),
};
jest.mock(
'../transfer/utils',
'../../transfer/utils',
() => {
return mockUtils;
},
@ -54,7 +60,7 @@ describe('export', () => {
jest.spyOn(console, 'error').mockImplementation(() => {});
// Now that everything is mocked, import export command
const exportCommand = require('../transfer/export');
const exportCommand = require('../../transfer/export');
const expectExit = async (code, fn) => {
const exit = jest.spyOn(process, 'exit').mockImplementation((number) => {
@ -76,7 +82,7 @@ describe('export', () => {
await exportCommand({ file: filename });
});
expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect.objectContaining({
file: { path: filename },
})
@ -90,7 +96,7 @@ describe('export', () => {
});
expect(mockUtils.getDefaultExportName).toHaveBeenCalledTimes(1);
expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect.objectContaining({
file: { path: defaultFileName },
})
@ -103,7 +109,7 @@ describe('export', () => {
await exportCommand({ encrypt });
});
expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect.objectContaining({
encryption: { enabled: encrypt },
})
@ -117,7 +123,7 @@ describe('export', () => {
await exportCommand({ encrypt, key });
});
expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect.objectContaining({
encryption: { enabled: encrypt, key },
})
@ -129,7 +135,7 @@ describe('export', () => {
await exportCommand({ compress: false });
});
expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect.objectContaining({
compression: { enabled: false },
})
@ -137,7 +143,7 @@ describe('export', () => {
await expectExit(1, async () => {
await exportCommand({ compress: true });
});
expect(mockDataTransfer.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect(mockDataTransfer.file.providers.createLocalFileDestinationProvider).toHaveBeenCalledWith(
expect.objectContaining({
compression: { enabled: true },
})

113
packages/core/strapi/lib/commands/__tests__/data-transfer/transfer.test.js Normal file
View File

@ -0,0 +1,113 @@
'use strict';
const utils = require('../../transfer/utils');
const mockDataTransfer = {
strapi: {
providers: {
createRemoteStrapiDestinationProvider: jest.fn(),
createLocalStrapiSourceProvider: jest.fn(),
},
},
engine: {
createTransferEngine: jest.fn().mockReturnValue({
transfer: jest.fn().mockReturnValue(Promise.resolve({})),
}),
},
};
jest.mock('@strapi/data-transfer/lib/engine', () => mockDataTransfer.engine, { virtual: true });
jest.mock('@strapi/data-transfer/lib/strapi', () => mockDataTransfer.strapi, { virtual: true });
const expectExit = async (code, fn) => {
const exit = jest.spyOn(process, 'exit').mockImplementation((number) => {
throw new Error(`process.exit: ${number}`);
});
await expect(async () => {
await fn();
}).rejects.toThrow();
expect(exit).toHaveBeenCalledWith(code);
exit.mockRestore();
};
const transferCommand = require('../../transfer/transfer');
jest.spyOn(console, 'error').mockImplementation(() => {});
jest.spyOn(console, 'warn').mockImplementation(() => {});
jest.spyOn(console, 'log').mockImplementation(() => {});
jest.mock('../../transfer/utils');
const destinationUrl = 'ws://strapi.com';
describe('Transfer', () => {
beforeEach(() => {
jest.resetAllMocks();
});
it('uses destination url provided by user without authentication', async () => {
await expectExit(1, async () => {
await transferCommand({ from: undefined, to: destinationUrl });
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
).toHaveBeenCalledWith(
expect.objectContaining({
url: destinationUrl,
})
);
});
it.todo('uses destination url provided by user with authentication');
it('uses restore as the default strategy', async () => {
await expectExit(1, async () => {
await transferCommand({ from: undefined, to: destinationUrl });
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
).toHaveBeenCalledWith(
expect.objectContaining({
strategy: 'restore',
})
);
});
it('uses destination url provided by user without authentication', async () => {
await expectExit(1, async () => {
await transferCommand({ from: undefined, to: destinationUrl });
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
).toHaveBeenCalledWith(
expect.objectContaining({
url: destinationUrl,
})
);
});
it('uses restore as the default strategy', async () => {
await expectExit(1, async () => {
await transferCommand({ from: undefined, to: destinationUrl });
});
expect(
mockDataTransfer.strapi.providers.createRemoteStrapiDestinationProvider
).toHaveBeenCalledWith(
expect.objectContaining({
strategy: 'restore',
})
);
});
it('uses local strapi instance when local specified', async () => {
await expectExit(1, async () => {
await transferCommand({ from: undefined, to: destinationUrl });
});
expect(mockDataTransfer.strapi.providers.createLocalStrapiSourceProvider).toHaveBeenCalled();
expect(utils.createStrapiInstance).toHaveBeenCalled();
});
});

10
packages/core/strapi/lib/commands/transfer/export.js
View File

@ -1,10 +1,12 @@
'use strict';
const {
createLocalFileDestinationProvider,
createLocalStrapiSourceProvider,
createTransferEngine,
} = require('@strapi/data-transfer');
providers: { createLocalFileDestinationProvider },
} = require('@strapi/data-transfer/lib/file');
const {
providers: { createLocalStrapiSourceProvider },
} = require('@strapi/data-transfer/lib/strapi');
const { createTransferEngine } = require('@strapi/data-transfer/lib/engine');
const { isObject, isString, isFinite, toNumber } = require('lodash/fp');
const fs = require('fs-extra');
const chalk = require('chalk');

15
packages/core/strapi/lib/commands/transfer/import.js
View File

@ -1,13 +1,17 @@
'use strict';
const {
createLocalFileSourceProvider,
createLocalStrapiDestinationProvider,
providers: { createLocalFileSourceProvider },
} = require('@strapi/data-transfer/lib/file');
const {
providers: { createLocalStrapiDestinationProvider, DEFAULT_CONFLICT_STRATEGY },
} = require('@strapi/data-transfer/lib/strapi');
const {
createTransferEngine,
DEFAULT_VERSION_STRATEGY,
DEFAULT_SCHEMA_STRATEGY,
DEFAULT_CONFLICT_STRATEGY,
} = require('@strapi/data-transfer');
} = require('@strapi/data-transfer/lib/engine');
const { isObject } = require('lodash/fp');
const path = require('path');
@ -43,6 +47,7 @@ module.exports = async (opts) => {
async getStrapi() {
return strapiInstance;
},
autoDestroy: false,
strategy: opts.conflictStrategy || DEFAULT_CONFLICT_STRATEGY,
restore: {
entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES },
@ -108,6 +113,8 @@ module.exports = async (opts) => {
// Note: Telemetry can't be sent in a finish event, because it runs async after this block but we can't await it, so if process.exit is used it won't send
await strapiInstance.telemetry.send('didDEITSProcessFinish', getTelemetryPayload());
await strapiInstance.destroy();
process.exit(0);
};

127
packages/core/strapi/lib/commands/transfer/transfer.js Normal file
View File

@ -0,0 +1,127 @@
'use strict';
const { createTransferEngine } = require('@strapi/data-transfer/lib/engine');
const {
providers: {
createRemoteStrapiDestinationProvider,
createLocalStrapiSourceProvider,
createLocalStrapiDestinationProvider,
},
} = require('@strapi/data-transfer/lib/strapi');
const { isObject } = require('lodash/fp');
const chalk = require('chalk');
const {
buildTransferTable,
createStrapiInstance,
DEFAULT_IGNORED_CONTENT_TYPES,
} = require('./utils');
const logger = console;
/**
* @typedef TransferCommandOptions Options given to the CLI transfer command
*
* @property {string|undefined} [to] The url of a remote Strapi to use as remote destination
* @property {string|undefined} [from] The url of a remote Strapi to use as remote source
*/
/**
* Transfer command.
*
* It transfers data from a local file to a local strapi instance
*
* @param {TransferCommandOptions} opts
*/
module.exports = async (opts) => {
// Validate inputs from Commander
if (!isObject(opts)) {
logger.error('Could not parse command arguments');
process.exit(1);
}
const strapi = await createStrapiInstance();
let source;
let destination;
if (!opts.from && !opts.to) {
logger.error('At least one source (from) or destination (to) option must be provided');
process.exit(1);
}
// if no URL provided, use local Strapi
if (!opts.from) {
source = createLocalStrapiSourceProvider({
getStrapi: () => strapi,
});
}
// if URL provided, set up a remote source provider
else {
logger.error(`Remote Strapi source provider not yet implemented`);
process.exit(1);
}
// if no URL provided, use local Strapi
if (!opts.to) {
destination = createLocalStrapiDestinationProvider({
getStrapi: () => strapi,
});
}
// if URL provided, set up a remote destination provider
else {
destination = createRemoteStrapiDestinationProvider({
url: opts.to,
auth: false,
strategy: 'restore',
restore: {
entities: { exclude: DEFAULT_IGNORED_CONTENT_TYPES },
},
});
}
if (!source || !destination) {
logger.error('Could not create providers');
process.exit(1);
}
const engine = createTransferEngine(source, destination, {
versionStrategy: 'ignore', // for an export to file, versionStrategy will always be skipped
schemaStrategy: 'ignore', // for an export to file, schemaStrategy will always be skipped
transforms: {
links: [
{
filter(link) {
return (
!DEFAULT_IGNORED_CONTENT_TYPES.includes(link.left.type) &&
!DEFAULT_IGNORED_CONTENT_TYPES.includes(link.right.type)
);
},
},
],
entities: [
{
filter(entity) {
return !DEFAULT_IGNORED_CONTENT_TYPES.includes(entity.type);
},
},
],
},
});
try {
logger.log(`Starting transfer...`);
const results = await engine.transfer();
const table = buildTransferTable(results.engine);
logger.log(table.toString());
logger.log(`${chalk.bold('Transfer process has been completed successfully!')}`);
process.exit(0);
} catch (e) {
logger.error('Transfer process failed unexpectedly');
logger.error(e);
process.exit(1);
}
};

1
packages/core/strapi/package.json
View File

@ -83,6 +83,7 @@
"@strapi/admin": "4.5.5",
"@strapi/data-transfer": "4.5.5",
"@strapi/database": "4.5.5",
"@strapi/data-transfer": "4.5.5",
"@strapi/generate-new": "4.5.5",
"@strapi/generators": "4.5.5",
"@strapi/logger": "4.5.5",

14
packages/plugins/sentry/README.md
View File

@ -68,10 +68,7 @@ try {
// Your code here
} catch (error) {
// Either send a simple error
strapi
.plugin('sentry')
.service('sentry')
.sendError(error);
strapi.plugin('sentry').service('sentry').sendError(error);
// Or send an error with a customized Sentry scope
strapi
@ -92,16 +89,13 @@ Use it if you need direct access to the Sentry instance, which should already al
**Example**
```js
const sentryInstance = strapi
.plugin('sentry')
.service('sentry')
.getInstance();
const sentryInstance = strapi.plugin('sentry').service('sentry').getInstance();
```
## Disabling for non-production environments
If the `dsn` property is set to a nil value (`null` or `undefined`) while `enabled` is true, the Sentry plugin will be available to use in the running Strapi instance, but the service will not actually send errors to Sentry. That allows you to write code that runs on every environment without additional checks, but only send errors to Sentry in production.
When you start Strapi with a nil `dsn` config property, the plugin will print a warning:
`info: @strapi/plugin-sentry is disabled because no Sentry DSN was provided`
@ -137,7 +131,7 @@ Like every other plugin, you can also disable this plugin in the plugins configu
module.exports = ({ env }) => ({
// ...
sentry: {
enabled: false
enabled: false,
},
// ...
});

31
yarn.lock
View File

@ -6300,6 +6300,20 @@
"@types/koa-compose" "*"
"@types/node" "*"
"@types/koa@2.13.4":
version "2.13.4"
resolved "https://registry.yarnpkg.com/@types/koa/-/koa-2.13.4.tgz#10620b3f24a8027ef5cbae88b393d1b31205726b"
integrity sha512-dfHYMfU+z/vKtQB7NUrthdAEiSvnLebvBjwHtfFmpZmB7em2N3WVQdHgnFq+xvyVgxW5jKDmjWfLD3lw4g4uTw==
dependencies:
"@types/accepts" "*"
"@types/content-disposition" "*"
"@types/cookies" "*"
"@types/http-assert" "*"
"@types/http-errors" "*"
"@types/keygrip" "*"
"@types/koa-compose" "*"
"@types/node" "*"
"@types/koa__cors@^3.0.1":
version "3.3.0"
resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.0.tgz#2986b320d3d7ddf05c4e2e472b25a321cb16bd3b"
@ -6603,6 +6617,11 @@
resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d"
integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ==
"@types/uuid@9.0.0":
version "9.0.0"
resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.0.tgz#53ef263e5239728b56096b0a869595135b7952d2"
integrity sha512-kr90f+ERiQtKWMz5rP32ltJ/BtULDI5RVO0uavn1HQUOwjx0R1h0rnDYNL0CepF1zL5bSY6FISAfd9tOdDhU5Q==
"@types/webpack-env@^1.16.0":
version "1.18.0"
resolved "https://registry.yarnpkg.com/@types/webpack-env/-/webpack-env-1.18.0.tgz#ed6ecaa8e5ed5dfe8b2b3d00181702c9925f13fb"
@ -18204,6 +18223,8 @@ path-case@^2.1.0:
version "2.1.1"
resolved "https://registry.yarnpkg.com/path-case/-/path-case-2.1.1.tgz#94b8037c372d3fe2906e465bb45e25d226e8eea5"
integrity sha1-lLgDfDctP+KQbkZbtF4l0ibo7qU=
dependencies:
no-case "^2.2.0"
path-dirname@^1.0.0:
version "1.0.2"
@ -22571,6 +22592,11 @@ uuid@8.0.0:
resolved "https://registry.yarnpkg.com/uuid/-/uuid-8.0.0.tgz#bc6ccf91b5ff0ac07bbcdbf1c7c4e150db4dbb6c"
integrity sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==
uuid@9.0.0:
version "9.0.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.0.tgz#592f550650024a38ceb0c562f2f6aa435761efb5"
integrity sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==
uuid@^3.3.2:
version "3.4.0"
resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
@ -23276,6 +23302,11 @@ write-pkg@^4.0.0:
type-fest "^0.4.1"
write-json-file "^3.2.0"
ws@8.11.0:
version "8.11.0"
resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143"
integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==
"ws@^5.2.0 || ^6.0.0 || ^7.0.0", ws@^7.3.1:
version "7.5.9"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"