diff --git a/.eslintrc.back.type-definitions.js b/.eslintrc.back.type-definitions.js new file mode 100644 index 0000000000..3cfff0036c --- /dev/null +++ b/.eslintrc.back.type-definitions.js @@ -0,0 +1,19 @@ +'use strict'; + +module.exports = { + parser: '@typescript-eslint/parser', + parserOptions: { + project: './tsconfig.eslint.json', + }, + plugins: ['@typescript-eslint'], + // TODO: This should be turned on but causes hundreds of violations in .d.ts files throughout Strapi that would need to be fixed + // extends: ['@strapi/eslint-config/typescript'], + globals: { + strapi: false, + }, + // Instead of extending (which includes values that interfere with this configuration), only take the rules field + rules: { + ...require('./.eslintrc.back.js').rules, + 'import/no-extraneous-dependencies': ['error', { devDependencies: ['**/*.test.ts'] }], + }, +}; diff --git a/.eslintrc.back.typescript.js b/.eslintrc.back.typescript.js index 78ba988d96..2e4260bd42 100644 --- a/.eslintrc.back.typescript.js +++ b/.eslintrc.back.typescript.js @@ -2,11 +2,11 @@ module.exports = { parser: '@typescript-eslint/parser', + parserOptions: { + project: './tsconfig.eslint.json', + }, plugins: ['@typescript-eslint'], - /** - * TODO: this should extend @strapi/eslint-config/typescript but doing so requires configuring parserOption.project, which requires tsconfig.json configuration - */ - // extends: ['plugin:@typescript-eslint/recommended'], + extends: ['@strapi/eslint-config/typescript'], globals: { strapi: false, }, @@ -14,5 +14,22 @@ module.exports = { rules: { ...require('./.eslintrc.back.js').rules, 'import/no-extraneous-dependencies': ['error', { devDependencies: ['**/*.test.ts'] }], + // TODO: The following rules from @strapi/eslint-config/typescript are disabled because they're causing problems we need to solve or fix + // to be solved in configuration + 'node/no-unsupported-features/es-syntax': 'off', + 'import/prefer-default-export': 'off', + 'node/no-missing-import': 'off', + '@typescript-eslint/brace-style': 'off', // TODO: fix conflict with prettier/prettier in data-transfer/engine/index.ts + // to be cleaned up throughout codebase (too many to fix at the moment) + '@typescript-eslint/no-use-before-define': 'warn', }, + // Disable only for tests + overrides: [ + { + files: ['**.test.ts'], + rules: { + '@typescript-eslint/ban-ts-comment': 'warn', // as long as javascript is allowed in our codebase, we want to test erroneous typescript usage + }, + }, + ], }; diff --git a/.eslintrc.js b/.eslintrc.js index 7feb7e4725..e2371ccd58 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -22,10 +22,17 @@ module.exports = { // Backend typescript { files: ['packages/**/*.ts', 'test/**/*.ts', 'scripts/**/*.ts', 'jest.*.ts'], - excludedFiles: frontPaths, + excludedFiles: [...frontPaths, '**/*.d.ts'], ...require('./.eslintrc.back.typescript.js'), }, + // Type definitions + { + files: ['packages/**/*.d.ts', 'test/**/*.d.ts', 'scripts/**/*.d.ts'], + excludedFiles: frontPaths, + ...require('./.eslintrc.back.type-definitions.js'), + }, + // Frontend { files: frontPaths, diff --git a/package.json b/package.json index ab848b2fac..9de3b599ba 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,8 @@ ], "scripts": { "prepare": "husky install", - "setup": "yarn && yarn build", + "setup": "yarn clean && yarn && yarn build", + "clean": "lerna run --stream clean --no-private", "watch": "lerna run --stream watch --no-private", "build": "lerna run --stream build --no-private", "generate": "plop --plopfile ./packages/generators/admin/plopfile.js", diff --git a/packages/core/data-transfer/lib/__tests__/test-utils.ts b/packages/core/data-transfer/lib/__tests__/test-utils.ts index a08ece0c30..a7f6fa38a9 100644 --- a/packages/core/data-transfer/lib/__tests__/test-utils.ts +++ b/packages/core/data-transfer/lib/__tests__/test-utils.ts @@ -130,7 +130,7 @@ export const extendExpectForDataTransferTests = () => { } return { pass: true, - message: () => `Expected engine not to be valid`, + message: () => 'Expected engine not to be valid', }; }, toHaveSourceStagesCalledTimes(provider: ISourceProvider, times: number) { @@ -145,6 +145,7 @@ export const extendExpectForDataTransferTests = () => { return true; } } + return false; }); if (missing.length) { @@ -156,7 +157,7 @@ export const extendExpectForDataTransferTests = () => { } return { pass: true, - message: () => `Expected source provider not to have all stages called`, + message: () => 'Expected source provider not to have all stages called', }; }, toHaveDestinationStagesCalledTimes(provider: IDestinationProvider, times: number) { @@ -170,6 +171,8 @@ export const extendExpectForDataTransferTests = () => { return true; } } + + return false; }); if (missing.length) { @@ -183,7 +186,7 @@ export const extendExpectForDataTransferTests = () => { } return { pass: true, - message: () => `Expected destination provider not to have all stages called`, + message: () => 'Expected destination provider not to have all stages called', }; }, toBeValidSourceProvider(provider: ISourceProvider) { @@ -199,7 +202,7 @@ export const extendExpectForDataTransferTests = () => { } return { pass: true, - message: () => `Expected source provider not to be valid`, + message: () => 'Expected source provider not to be valid', }; }, toBeValidDestinationProvider(provider: IDestinationProvider) { @@ -215,7 +218,7 @@ export const extendExpectForDataTransferTests = () => { } return { pass: true, - message: () => `Expected destination provider not to be valid`, + message: () => 'Expected destination provider not to be valid', }; }, }); diff --git a/packages/core/data-transfer/lib/encryption/decrypt.ts b/packages/core/data-transfer/lib/encryption/decrypt.ts index 2babd377e0..e7a8e4c7a2 100644 --- a/packages/core/data-transfer/lib/encryption/decrypt.ts +++ b/packages/core/data-transfer/lib/encryption/decrypt.ts @@ -4,25 +4,25 @@ import { EncryptionStrategy, Strategies, Algorithm } from '../../types'; // different key values depending on algorithm chosen const getDecryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { const strategies: Strategies = { - 'aes-128-ecb': (key: string): Cipher => { + 'aes-128-ecb'(key: string): Cipher { const hashedKey = scryptSync(key, '', 16); const initVector: BinaryLike | null = null; const securityKey: CipherKey = hashedKey; return createDecipheriv(algorithm, securityKey, initVector); }, - aes128: (key: string): Cipher => { + aes128(key: string): Cipher { const hashedKey = scryptSync(key, '', 32); const initVector: BinaryLike | null = hashedKey.slice(16); const securityKey: CipherKey = hashedKey.slice(0, 16); return createDecipheriv(algorithm, securityKey, initVector); }, - aes192: (key: string): Cipher => { + aes192(key: string): Cipher { const hashedKey = scryptSync(key, '', 40); const initVector: BinaryLike | null = hashedKey.slice(24); const securityKey: CipherKey = hashedKey.slice(0, 24); return createDecipheriv(algorithm, securityKey, initVector); }, - aes256: (key: string): Cipher => { + aes256(key: string): Cipher { const hashedKey = scryptSync(key, '', 48); const initVector: BinaryLike | null = hashedKey.slice(32); const securityKey: CipherKey = hashedKey.slice(0, 32); diff --git a/packages/core/data-transfer/lib/encryption/encrypt.ts b/packages/core/data-transfer/lib/encryption/encrypt.ts index 523fecf537..d32e2cd6e9 100644 --- a/packages/core/data-transfer/lib/encryption/encrypt.ts +++ b/packages/core/data-transfer/lib/encryption/encrypt.ts @@ -4,25 +4,25 @@ import { EncryptionStrategy, Strategies, Algorithm } from '../../types'; // different key values depending on algorithm chosen const getEncryptionStrategy = (algorithm: Algorithm): EncryptionStrategy => { const strategies: Strategies = { - 'aes-128-ecb': (key: string): Cipher => { + 'aes-128-ecb'(key: string): Cipher { const hashedKey = scryptSync(key, '', 16); const initVector: BinaryLike | null = null; const securityKey: CipherKey = hashedKey; return createCipheriv(algorithm, securityKey, initVector); }, - aes128: (key: string): Cipher => { + aes128(key: string): Cipher { const hashedKey = scryptSync(key, '', 32); const initVector: BinaryLike | null = hashedKey.slice(16); const securityKey: CipherKey = hashedKey.slice(0, 16); return createCipheriv(algorithm, securityKey, initVector); }, - aes192: (key: string): Cipher => { + aes192(key: string): Cipher { const hashedKey = scryptSync(key, '', 40); const initVector: BinaryLike | null = hashedKey.slice(24); const securityKey: CipherKey = hashedKey.slice(0, 24); return createCipheriv(algorithm, securityKey, initVector); }, - aes256: (key: string): Cipher => { + aes256(key: string): Cipher { const hashedKey = scryptSync(key, '', 48); const initVector: BinaryLike | null = hashedKey.slice(32); const securityKey: CipherKey = hashedKey.slice(0, 32); diff --git a/packages/core/data-transfer/lib/engine/__tests__/engine.test.ts b/packages/core/data-transfer/lib/engine/__tests__/engine.test.ts index 2ef830d1a9..e7c817e61e 100644 --- a/packages/core/data-transfer/lib/engine/__tests__/engine.test.ts +++ b/packages/core/data-transfer/lib/engine/__tests__/engine.test.ts @@ -9,7 +9,6 @@ import type { IEntity, ILink, ISourceProvider, - ITransferEngine, ITransferEngineOptions, } from '../../../types'; import { @@ -393,7 +392,7 @@ describe('Transfer engine', () => { }); describe('progressStream', () => { - test(`emits 'progress' events`, async () => { + test("emits 'progress' events", async () => { const source = createSource(); const engine = createTransferEngine(source, completeDestination, defaultOptions); @@ -412,8 +411,8 @@ describe('Transfer engine', () => { }); // TODO: to implement these, the mocked streams need to be improved - test.todo(`emits 'start' events`); - test.todo(`emits 'complete' events`); + test.todo("emits 'start' events"); + test.todo("emits 'complete' events"); }); describe('integrity checks', () => { diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts index 974778991d..84091320c3 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider/__tests__/utils.test.ts @@ -48,7 +48,7 @@ describe('Local File Destination Provider - Utils', () => { const tarEntryStream = createTarEntryStream(archive, pathFactory, maxSize); const write = async () => - await new Promise((resolve, reject) => { + new Promise((resolve, reject) => { tarEntryStream.on('finish', resolve); tarEntryStream.on('error', reject); tarEntryStream.write(chunk); @@ -64,13 +64,13 @@ describe('Local File Destination Provider - Utils', () => { const tarEntryStream = createTarEntryStream(archive, pathFactory, maxSize); const write = async () => - await new Promise((resolve, reject) => { + new Promise((resolve, reject) => { tarEntryStream.on('finish', resolve); tarEntryStream.on('error', reject); tarEntryStream.write(chunk); }); - expect(write).resolves; + expect(write()).resolves.not.toBe(null); }); }); }); diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts index e34df30cf1..994e5b20d6 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider/index.ts @@ -1,12 +1,3 @@ -import type { - IAsset, - IDestinationProvider, - IDestinationProviderTransferResults, - IMetadata, - ProviderType, - Stream, -} from '../../../types'; - import fs from 'fs-extra'; import tar from 'tar-stream'; import path from 'path'; @@ -16,20 +7,26 @@ import { stringer } from 'stream-json/jsonl/Stringer'; import { chain, Writable } from 'stream-chain'; import { createEncryptionCipher } from '../../encryption/encrypt'; +import type { + IAsset, + IDestinationProvider, + IDestinationProviderTransferResults, + IMetadata, + ProviderType, + Stream, +} from '../../../types'; import { createFilePathFactory, createTarEntryStream } from './utils'; + export interface ILocalFileDestinationProviderOptions { - // Encryption encryption: { enabled: boolean; key?: string; }; - // Compression compression: { enabled: boolean; }; - // File file: { path: string; maxSize?: number; @@ -51,12 +48,16 @@ export const createLocalFileDestinationProvider = ( }; class LocalFileDestinationProvider implements IDestinationProvider { - name: string = 'destination::local-file'; + name = 'destination::local-file'; + type: ProviderType = 'destination'; + options: ILocalFileDestinationProviderOptions; + results: ILocalFileDestinationProviderTransferResults = {}; #providersMetadata: { source?: IMetadata; destination?: IMetadata } = {}; + #archive: { stream?: tar.Pack; pipeline?: Stream } = {}; constructor(options: ILocalFileDestinationProviderOptions) { @@ -66,17 +67,17 @@ class LocalFileDestinationProvider implements IDestinationProvider { get #archivePath() { const { encryption, compression, file } = this.options; - let path = `${file.path}.tar`; + let filePath = `${file.path}.tar`; if (compression.enabled) { - path += '.gz'; + filePath += '.gz'; } if (encryption.enabled) { - path += '.enc'; + filePath += '.enc'; } - return path; + return filePath; } setMetadata(target: ProviderType, metadata: IMetadata): IDestinationProvider { diff --git a/packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts b/packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts index 2920087f32..631a1056f7 100644 --- a/packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts +++ b/packages/core/data-transfer/lib/providers/local-file-destination-provider/utils.ts @@ -8,7 +8,7 @@ import tar from 'tar-stream'; */ export const createFilePathFactory = (type: string) => - (fileIndex: number = 0): string => { + (fileIndex = 0): string => { return path.join( // "{type}" directory type, @@ -20,7 +20,7 @@ export const createFilePathFactory = export const createTarEntryStream = ( archive: tar.Pack, pathFactory: (index?: number) => string, - maxSize: number = 2.56e8 + maxSize = 2.56e8 ) => { let fileIndex = 0; let buffer = ''; @@ -30,7 +30,8 @@ export const createTarEntryStream = ( return; } - const name = pathFactory(fileIndex++); + fileIndex += 1; + const name = pathFactory(fileIndex); const size = buffer.length; await new Promise((resolve, reject) => { diff --git a/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts b/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts index a9288e8195..22cdccc259 100644 --- a/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-file-source-provider/index.ts @@ -1,18 +1,17 @@ import type { Readable } from 'stream'; -import type { IMetadata, ISourceProvider, ProviderType } from '../../../types'; - import fs from 'fs'; import zip from 'zlib'; import tar from 'tar'; import path from 'path'; import { keyBy } from 'lodash/fp'; import { chain } from 'stream-chain'; -import { pipeline, PassThrough, Readable as ReadStream } from 'stream'; +import { pipeline, PassThrough } from 'stream'; import { parser } from 'stream-json/jsonl/Parser'; +import type { IMetadata, ISourceProvider, ProviderType } from '../../../types'; -import { createDecryptionCipher } from '../../encryption'; import { collect } from '../../utils'; +import { createDecryptionCipher } from '../../encryption'; type StreamItemArray = Parameters[0]; @@ -25,25 +24,18 @@ const METADATA_FILE_PATH = 'metadata.json'; * Provider options */ export interface ILocalFileSourceProviderOptions { - /** - * Path to the backup archive - */ - backupFilePath: string; + file: { + path: string; + }; - /** - * Whether the backup data is encrypted or not - */ - encrypted?: boolean; + encryption: { + enabled: boolean; + key?: string; + }; - /** - * Encryption key used to decrypt the encrypted data (if necessary) - */ - encryptionKey?: string; - - /** - * Whether the backup data is compressed or not - */ - compressed?: boolean; + compression: { + enabled: boolean; + }; } export const createLocalFileSourceProvider = (options: ILocalFileSourceProviderOptions) => { @@ -52,14 +44,17 @@ export const createLocalFileSourceProvider = (options: ILocalFileSourceProviderO class LocalFileSourceProvider implements ISourceProvider { type: ProviderType = 'source'; - name: string = 'source::local-file'; + + name = 'source::local-file'; options: ILocalFileSourceProviderOptions; constructor(options: ILocalFileSourceProviderOptions) { this.options = options; - if (this.options.encrypted && this.options.encryptionKey === undefined) { + const { encryption } = this.options; + + if (encryption.enabled && encryption.key === undefined) { throw new Error('Missing encryption key'); } } @@ -68,13 +63,13 @@ class LocalFileSourceProvider implements ISourceProvider { * Pre flight checks regarding the provided options (making sure that the provided path is correct, etc...) */ bootstrap() { - const path = this.options.backupFilePath; - const isValidBackupPath = fs.existsSync(path); + const { path: filePath } = this.options.file; + const isValidBackupPath = fs.existsSync(filePath); // Check if the provided path exists if (!isValidBackupPath) { throw new Error( - `Invalid backup file path provided. "${path}" does not exist on the filesystem.` + `Invalid backup file path provided. "${filePath}" does not exist on the filesystem.` ); } } @@ -117,13 +112,13 @@ class LocalFileSourceProvider implements ISourceProvider { [ inStream, new tar.Parse({ - filter(path, entry) { + filter(filePath, entry) { if (entry.type !== 'File') { return false; } - const parts = path.split('/'); - return parts[0] === 'assets' && parts[1] == 'uploads'; + const parts = filePath.split('/'); + return parts[0] === 'assets' && parts[1] === 'uploads'; }, onentry(entry) { const { path: filePath, size } = entry; @@ -138,13 +133,17 @@ class LocalFileSourceProvider implements ISourceProvider { return outStream; } - #getBackupStream(decompress: boolean = true) { - const path = this.options.backupFilePath; - const readStream = fs.createReadStream(path); - const streams: StreamItemArray = [readStream]; + #getBackupStream() { + const { file, encryption, compression } = this.options; - // Handle decompression - if (decompress) { + const fileStream = fs.createReadStream(file.path); + const streams: StreamItemArray = [fileStream]; + + if (encryption.enabled && encryption.key) { + streams.push(createDecryptionCipher(encryption.key)); + } + + if (compression.enabled) { streams.push(zip.createGunzip()); } @@ -152,7 +151,6 @@ class LocalFileSourceProvider implements ISourceProvider { } #streamJsonlDirectory(directory: string) { - const options = this.options; const inStream = this.#getBackupStream(); const outStream = new PassThrough({ objectMode: true }); @@ -161,12 +159,12 @@ class LocalFileSourceProvider implements ISourceProvider { [ inStream, new tar.Parse({ - filter(path, entry) { + filter(filePath, entry) { if (entry.type !== 'File') { return false; } - const parts = path.split('/'); + const parts = filePath.split('/'); if (parts.length !== 2) { return false; @@ -176,22 +174,12 @@ class LocalFileSourceProvider implements ISourceProvider { }, onentry(entry) { - const transforms = []; - - if (options.encrypted) { - transforms.push(createDecryptionCipher(options.encryptionKey!)); - } - - if (options.compressed) { - transforms.push(zip.createGunzip()); - } - - transforms.push( + const transforms = [ // JSONL parser to read the data chunks one by one (line by line) parser(), // The JSONL parser returns each line as key/value - (line: { key: string; value: any }) => line.value - ); + (line: { key: string; value: any }) => line.value, + ]; entry // Pipe transforms @@ -213,7 +201,7 @@ class LocalFileSourceProvider implements ISourceProvider { return outStream; } - async #parseJSONFile( + async #parseJSONFile = any>( fileStream: NodeJS.ReadableStream, filePath: string ): Promise { @@ -226,8 +214,8 @@ class LocalFileSourceProvider implements ISourceProvider { /** * Filter the parsed entries to only keep the one that matches the given filepath */ - filter(path, entry) { - return path === filePath && entry.type === 'File'; + filter(entryPath, entry) { + return entryPath === filePath && entry.type === 'File'; }, /** @@ -250,7 +238,7 @@ class LocalFileSourceProvider implements ISourceProvider { () => { // If the promise hasn't been resolved and we've parsed all // the archive entries, then the file doesn't exist - reject(`${filePath} not found in the archive stream`); + reject(new Error(`File "${filePath}" not found`)); } ); }); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts index 3008fa3c5d..6942d94717 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/__tests__/restore.test.ts @@ -1,4 +1,4 @@ -import { deleteAllRecords } from '../restore'; +import { deleteAllRecords, restoreConfigs } from '../restore'; import { getStrapiFactory, getContentTypes } from '../../test-utils'; const entities = [ @@ -32,6 +32,10 @@ const entities = [ }, ]; +afterEach(() => { + jest.clearAllMocks(); +}); + const deleteMany = jest.fn(async (uid: string) => ({ count: entities.filter((entity) => entity.contentType.uid === uid).length, })); @@ -41,6 +45,7 @@ const query = jest.fn(() => { deleteMany: jest.fn(() => ({ count: 0, })), + create: jest.fn((data) => data), }; }); @@ -72,8 +77,66 @@ describe('Restore ', () => { const { count } = await deleteAllRecords(strapi, { /* @ts-ignore: disable-next-line */ - contentTypes: [getContentTypes()['foo']], + contentTypes: [getContentTypes().foo], }); expect(count).toBe(3); }); + + test('Should add core store data', async () => { + const strapi = getStrapiFactory({ + contentTypes: getContentTypes(), + db: { + query, + }, + })(); + const config = { + type: 'core-store', + value: { + key: 'test-key', + type: 'test-type', + environment: null, + tag: 'tag', + value: {}, + }, + }; + const result = await restoreConfigs(strapi, config); + + expect(strapi.db.query).toBeCalledTimes(1); + expect(strapi.db.query).toBeCalledWith('strapi::core-store'); + expect(result.data).toMatchObject(config.value); + }); + + test('Should add webhook data', async () => { + const strapi = getStrapiFactory({ + contentTypes: getContentTypes(), + db: { + query, + }, + })(); + const config = { + type: 'webhook', + value: { + id: 4, + name: 'christian', + url: 'https://facebook.com', + headers: { null: '' }, + events: [ + 'entry.create', + 'entry.update', + 'entry.delete', + 'entry.publish', + 'entry.unpublish', + 'media.create', + 'media.update', + 'media.delete', + ], + enabled: true, + }, + }; + const result = await restoreConfigs(strapi, config); + + expect(strapi.db.query).toBeCalledTimes(1); + expect(strapi.db.query).toBeCalledWith('webhook'); + expect(result.data).toMatchObject(config.value); + }); }); diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts index ff2d9e6572..a183cbfacb 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/index.ts @@ -1,11 +1,12 @@ // import { createLogger } from '@strapi/logger'; -import type { IDestinationProvider, IMetadata, ProviderType } from '../../../types'; -import { deleteAllRecords, DeleteOptions } from './restore'; +import chalk from 'chalk'; import { Writable } from 'stream'; import path from 'path'; import * as fse from 'fs-extra'; +import type { IConfiguration, IDestinationProvider, IMetadata, ProviderType } from '../../../types'; +import { deleteAllRecords, DeleteOptions, restoreConfigs } from './restore'; import { mapSchemasValues } from '../../utils'; export const VALID_STRATEGIES = ['restore', 'merge']; @@ -16,10 +17,6 @@ interface ILocalStrapiDestinationProviderOptions { strategy: 'restore' | 'merge'; } -// TODO: getting some type errors with @strapi/logger that need to be resolved first -// const log = createLogger(); -const log = console; - export const createLocalStrapiDestinationProvider = ( options: ILocalStrapiDestinationProviderOptions ) => { @@ -27,10 +24,12 @@ export const createLocalStrapiDestinationProvider = ( }; class LocalStrapiDestinationProvider implements IDestinationProvider { - name: string = 'destination::local-strapi'; + name = 'destination::local-strapi'; + type: ProviderType = 'destination'; options: ILocalStrapiDestinationProviderOptions; + strapi?: Strapi.Strapi; constructor(options: ILocalStrapiDestinationProviderOptions) { @@ -48,7 +47,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { #validateOptions() { if (!VALID_STRATEGIES.includes(this.options.strategy)) { - throw new Error('Invalid stategy ' + this.options.strategy); + throw new Error(`Invalid stategy ${this.options.strategy}`); } } @@ -56,7 +55,7 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { if (!this.strapi) { throw new Error('Strapi instance not found'); } - return await deleteAllRecords(this.strapi, this.options.restore); + return deleteAllRecords(this.strapi, this.options.restore); } async beforeTransfer() { @@ -117,4 +116,30 @@ class LocalStrapiDestinationProvider implements IDestinationProvider { }, }); } + + async getConfigurationStream(): Promise { + if (!this.strapi) { + throw new Error('Not able to stream Configurations. Strapi instance not found'); + } + + return new Writable({ + objectMode: true, + write: async (config: IConfiguration, _encoding, callback) => { + try { + if (this.options.strategy === 'restore' && this.strapi) { + await restoreConfigs(this.strapi, config); + } + callback(); + } catch (error) { + callback( + new Error( + `Failed to import ${chalk.yellowBright(config.type)} (${chalk.greenBright( + config.value.id + )}` + ) + ); + } + }, + }); + } } diff --git a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/restore.ts b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/restore.ts index 5202168fbf..fc62bea4fd 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/restore.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-destination-provider/restore.ts @@ -1,4 +1,5 @@ import type { ContentTypeSchema } from '@strapi/strapi'; +import type { IConfiguration } from '../../../types'; export type DeleteOptions = { contentTypes?: ContentTypeSchema[]; @@ -33,3 +34,28 @@ export const deleteAllRecords = async (strapi: Strapi.Strapi, deleteOptions?: De return { count }; }; + +const restoreCoreStore = async (strapi: Strapi.Strapi, data: any) => { + return strapi.db.query('strapi::core-store').create({ + data: { + ...data, + value: JSON.stringify(data.value), + }, + }); +}; + +const restoreWebhooks = async (strapi: Strapi.Strapi, data: any) => { + return strapi.db.query('webhook').create({ + data, + }); +}; + +export const restoreConfigs = async (strapi: Strapi.Strapi, config: IConfiguration) => { + if (config.type === 'core-store') { + return restoreCoreStore(strapi, config.value); + } + + if (config.type === 'webhook') { + return restoreWebhooks(strapi, config.value); + } +}; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts index 2885f10b33..44308acaf0 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/entities.test.ts @@ -1,6 +1,5 @@ -import type { IEntity } from '../../../../types'; - import { Readable, PassThrough } from 'stream'; +import type { IEntity } from '../../../../types'; import { collect, diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts index 528032c9f0..ecac09f538 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/index.test.ts @@ -1,9 +1,8 @@ +import { Readable } from 'stream'; import type { IEntity } from '../../../../types'; -import { Readable } from 'stream'; - import { collect, createMockedQueryBuilder, getStrapiFactory } from '../../../__tests__/test-utils'; -import { createLocalStrapiSourceProvider } from '../'; +import { createLocalStrapiSourceProvider } from '..'; describe('Local Strapi Source Provider', () => { describe('Bootstrap', () => { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts index 09ef3250a7..eca2568afc 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/__tests__/links.test.ts @@ -347,7 +347,7 @@ describe('Local Strapi Source Provider - Entities Streaming', () => { expect(populate).toEqual({}); }); - test(`Should return an empty object if there are components or dynamic zones but they doesn't contain relations`, () => { + test("Should return an empty object if there are components or dynamic zones but they doesn't contain relations", () => { const strapi = getStrapiFactory({ contentTypes: { blog: { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts index bb3b893f80..25da25251a 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/index.ts @@ -1,5 +1,4 @@ import { chain } from 'stream-chain'; -import { mapValues, pick } from 'lodash/fp'; import { Readable } from 'stream'; import type { IMetadata, ISourceProvider, ProviderType } from '../../../types'; @@ -20,10 +19,12 @@ export const createLocalStrapiSourceProvider = (options: ILocalStrapiSourceProvi }; class LocalStrapiSourceProvider implements ISourceProvider { - name: string = 'source::local-strapi'; + name = 'source::local-strapi'; + type: ProviderType = 'source'; options: ILocalStrapiSourceProviderOptions; + strapi?: Strapi.Strapi; constructor(options: ILocalStrapiSourceProviderOptions) { diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/index.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/index.ts index 7713b30b39..706d9cd048 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/index.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/index.ts @@ -1,4 +1,4 @@ -import type { ContentTypeSchema, GetAttributesValues, RelationsType } from '@strapi/strapi'; +import type { ContentTypeSchema } from '@strapi/strapi'; import { Readable } from 'stream'; import { castArray } from 'lodash/fp'; diff --git a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/utils.ts b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/utils.ts index ed2fcbe2ff..ad108c9575 100644 --- a/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/utils.ts +++ b/packages/core/data-transfer/lib/providers/local-strapi-source-provider/links/utils.ts @@ -1,7 +1,6 @@ import type { RelationsType } from '@strapi/strapi'; -import type { ILink } from '../../../../types'; - import { concat, set, isEmpty } from 'lodash/fp'; +import type { ILink } from '../../../../types'; // TODO: Fix any typings when we'll have better Strapi types @@ -49,7 +48,7 @@ export const parseEntityLinks = (entity: any, populate: any, schema: any, strapi .map(({ __component, ...item }: any) => parseEntityLinks(item, subPopulate.populate, strapi.components[__component], strapi) ) - .reduce((acc: any, links: any) => acc.concat(...links), []); + .reduce((acc: any, rlinks: any) => acc.concat(...rlinks), []); links.push(...dzLinks); } @@ -90,6 +89,7 @@ export const parseRelationLinks = ({ entity, schema, fieldName, value }: any): I const isMorphRelation = relation.startsWith('morph'); const isCircularRelation = !isMorphRelation && target === schema.uid; + // eslint-disable-next-line no-nested-ternary const kind: ILink['kind'] = isMorphRelation ? // Polymorphic relations 'relation.morph' diff --git a/packages/core/data-transfer/lib/strategies/index.ts b/packages/core/data-transfer/lib/strategies/index.ts index 8566f7f811..512cbbb676 100644 --- a/packages/core/data-transfer/lib/strategies/index.ts +++ b/packages/core/data-transfer/lib/strategies/index.ts @@ -4,12 +4,12 @@ import { jsonDiffs } from '../utils'; const strategies = { // No diffs - exact: (diffs: Diff[]) => { + exact(diffs: Diff[]) { return diffs; }, // Diffs allowed on specific attributes properties - strict: (diffs: Diff[]) => { + strict(diffs: Diff[]) { const isIgnorableDiff = ({ path }: Diff) => { return ( path.length === 3 && diff --git a/packages/core/data-transfer/lib/utils.ts b/packages/core/data-transfer/lib/utils.ts index 874eff14b1..dc3a953940 100644 --- a/packages/core/data-transfer/lib/utils.ts +++ b/packages/core/data-transfer/lib/utils.ts @@ -1,7 +1,7 @@ import type { Readable } from 'stream'; -import type { Context, Diff } from '../types'; import { isArray, isObject, zip, isEqual, uniq, mapValues, pick } from 'lodash/fp'; +import type { Context, Diff } from '../types'; /** * Collect every entity in a Readable stream @@ -24,6 +24,9 @@ export const jsonDiffs = (a: unknown, b: unknown, ctx: Context = createContext() const diffs: Diff[] = []; const { path } = ctx; + const aType = typeof a; + const bType = typeof b; + // Define helpers const added = () => { @@ -46,9 +49,6 @@ export const jsonDiffs = (a: unknown, b: unknown, ctx: Context = createContext() return diffs; }; - const aType = typeof a; - const bType = typeof b; - if (aType === 'undefined') { return added(); } @@ -66,7 +66,7 @@ export const jsonDiffs = (a: unknown, b: unknown, ctx: Context = createContext() diffs.push(...kDiffs); - k++; + k += 1; } return diffs; diff --git a/packages/core/data-transfer/types/common-entities.d.ts b/packages/core/data-transfer/types/common-entities.d.ts index fae5394ce3..f7e97db34d 100644 --- a/packages/core/data-transfer/types/common-entities.d.ts +++ b/packages/core/data-transfer/types/common-entities.d.ts @@ -122,6 +122,13 @@ interface ICircularLink extends IDefaultLink { kind: 'relation.circular'; } +/** + * Strapi configurations + */ +interface IConfiguration { + type: 'core-store' | 'webhook'; + value: T; +} export interface IAsset { filename: string; filepath: string; diff --git a/packages/core/strapi/bin/strapi.js b/packages/core/strapi/bin/strapi.js index d1c8c1ecce..97bfadfffa 100755 --- a/packages/core/strapi/bin/strapi.js +++ b/packages/core/strapi/bin/strapi.js @@ -5,6 +5,7 @@ // FIXME /* eslint-disable import/extensions */ const _ = require('lodash'); +const path = require('path'); const resolveCwd = require('resolve-cwd'); const { yellow } = require('chalk'); const { Command, Option } = require('commander'); @@ -316,14 +317,15 @@ program 'path and filename to the Strapi export file you want to import' ) .addOption( - new Option('--key ', 'Provide encryption key in command instead of using a prompt') + new Option('-k, --key ', 'Provide encryption key in command instead of using a prompt') ) .allowExcessArguments(false) .hook('preAction', async (thisCommand) => { const opts = thisCommand.opts(); + const ext = path.extname(String(opts.file)); // check extension to guess if we should prompt for key - if (String(opts.file).endsWith('.enc')) { + if (ext === '.enc') { if (!opts.key) { const answers = await inquirer.prompt([ { diff --git a/packages/core/strapi/lib/commands/transfer/import.js b/packages/core/strapi/lib/commands/transfer/import.js index 44e75b27ca..d0413a91a4 100644 --- a/packages/core/strapi/lib/commands/transfer/import.js +++ b/packages/core/strapi/lib/commands/transfer/import.js @@ -8,10 +8,15 @@ const { // eslint-disable-next-line import/no-unresolved, node/no-missing-require } = require('@strapi/data-transfer'); const { isObject } = require('lodash/fp'); +const path = require('path'); const strapi = require('../../index'); const { buildTransferTable } = require('./utils'); +/** + * @typedef {import('@strapi/data-transfer').ILocalFileSourceProviderOptions} ILocalFileSourceProviderOptions + */ + const logger = console; module.exports = async (opts) => { @@ -20,14 +25,12 @@ module.exports = async (opts) => { logger.error('Could not parse arguments'); process.exit(1); } - const filename = opts.file; /** * From strapi backup file */ - const sourceOptions = { - backupFilePath: filename, - }; + const sourceOptions = getLocalFileSourceOptions(opts); + const source = createLocalFileSourceProvider(sourceOptions); /** @@ -69,16 +72,50 @@ module.exports = async (opts) => { const engine = createTransferEngine(source, destination, engineOptions); try { - logger.log('Starting import...'); + logger.info('Starting import...'); const results = await engine.transfer(); const table = buildTransferTable(results.engine); - logger.log(table.toString()); + logger.info(table.toString()); - logger.log('Import process has been completed successfully!'); + logger.info('Import process has been completed successfully!'); process.exit(0); } catch (e) { - logger.log(`Import process failed unexpectedly: ${e.message}`); + logger.error(`Import process failed unexpectedly: ${e.message}`); process.exit(1); } }; + +/** + * Infer local file source provider options based on a given filename + * + * @param {{ file: string; key?: string }} opts + * + * @return {ILocalFileSourceProviderOptions} + */ +const getLocalFileSourceOptions = (opts) => { + /** + * @type {ILocalFileSourceProviderOptions} + */ + const options = { + file: { path: opts.file }, + compression: { enabled: false }, + encryption: { enabled: false }, + }; + + const { extname, parse } = path; + + let file = options.file.path; + + if (extname(file) === '.enc') { + file = parse(file).name; + options.encryption = { enabled: true, key: opts.key }; + } + + if (extname(file) === '.gz') { + file = parse(file).name; + options.compression = { enabled: true }; + } + + return options; +}; diff --git a/tsconfig.eslint.json b/tsconfig.eslint.json new file mode 100644 index 0000000000..a9eb489a89 --- /dev/null +++ b/tsconfig.eslint.json @@ -0,0 +1,8 @@ +{ + "compilerOptions": { + "composite": true, + "declaration": true, + "declarationMap": true, + "incremental": true + } +}