From 34026fd88c8dac104e77db78be110f307f00a09b Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Tue, 5 May 2020 14:21:23 +0200 Subject: [PATCH 1/8] Init Signed-off-by: Alexandre Bodin --- packages/strapi/bin/strapi.js | 5 ++++ packages/strapi/lib/Strapi.js | 15 ++++++----- .../strapi/lib/commands/configurationDump.js | 25 +++++++++++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 packages/strapi/lib/commands/configurationDump.js diff --git a/packages/strapi/bin/strapi.js b/packages/strapi/bin/strapi.js index cefc542648..5226900af8 100755 --- a/packages/strapi/bin/strapi.js +++ b/packages/strapi/bin/strapi.js @@ -205,6 +205,11 @@ program .description('Starts the admin dev server') .action(getLocalScript('watchAdmin')); +program + .command('configuration:dump') + .option('-f, --file ', 'file to output to') + .action(getLocalScript('configurationDump')); + /** * Normalize help argument */ diff --git a/packages/strapi/lib/Strapi.js b/packages/strapi/lib/Strapi.js index 0d6a5bf6e4..8b9acb5e95 100644 --- a/packages/strapi/lib/Strapi.js +++ b/packages/strapi/lib/Strapi.js @@ -53,6 +53,7 @@ class Strapi { this.admin = {}; this.plugins = {}; this.config = loadConfiguration(this.dir, opts); + this.isLoaded = false; // internal services. this.fs = createStrapiFs(this); @@ -127,12 +128,9 @@ class Strapi { async start(cb) { try { - await this.load(); - - // Run bootstrap function. - await this.runBootstrapFunctions(); - // Freeze object. - await this.freeze(); + if (!this.isLoaded) { + await this.load(); + } this.app.use(this.router.routes()).use(this.router.allowedMethods()); @@ -236,6 +234,8 @@ class Strapi { } async load() { + this.isLoaded = true; + this.app.use(async (ctx, next) => { if (ctx.request.url === '/_health' && ctx.request.method === 'HEAD') { ctx.set('strapi', 'You are so French!'); @@ -294,6 +294,9 @@ class Strapi { // Initialize hooks and middlewares. await initializeMiddlewares.call(this); await initializeHooks.call(this); + + await this.runBootstrapFunctions(); + await this.freeze(); } async startWebhooks() { diff --git a/packages/strapi/lib/commands/configurationDump.js b/packages/strapi/lib/commands/configurationDump.js new file mode 100644 index 0000000000..c9480b0907 --- /dev/null +++ b/packages/strapi/lib/commands/configurationDump.js @@ -0,0 +1,25 @@ +'use strict'; + +const fs = require('fs'); +const { logger } = require('strapi-utils'); +const loadConfiguration = require('../core/app-configuration'); +const strapi = require('../index'); + +module.exports = async function({ file }) { + const output = file ? fs.createWriteStream(file) : process.stdout; + + output.write('this is a test'); + + const app = strapi(); + + await app.load(); + + const confs = await app.query('core_store').find({ + key_contains: 'plugin', + }); + + console.log(confs); + + output.write('\n'); + output.end(); +}; From 7da81fef9efc5a0d76093d76ab6a69371d2c31f7 Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Wed, 6 May 2020 18:49:36 +0200 Subject: [PATCH 2/8] Add configuration:dump & configuration:restore commands Signed-off-by: Alexandre Bodin --- packages/strapi/bin/strapi.js | 19 ++- .../strapi/lib/commands/configurationDump.js | 43 +++++-- .../lib/commands/configurationRestore.js | 113 ++++++++++++++++++ 3 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 packages/strapi/lib/commands/configurationRestore.js diff --git a/packages/strapi/bin/strapi.js b/packages/strapi/bin/strapi.js index 5226900af8..524183ee42 100755 --- a/packages/strapi/bin/strapi.js +++ b/packages/strapi/bin/strapi.js @@ -49,7 +49,16 @@ const getLocalScript = name => (...args) => { process.exit(1); } - return require(cmdPath)(...args); + const script = require(cmdPath); + + Promise.resolve() + .then(() => { + return script(...args); + }) + .catch(error => { + console.error(`Error while running command ${name}: ${error.message}`); + process.exit(1); + }); }; /** @@ -207,9 +216,15 @@ program program .command('configuration:dump') - .option('-f, --file ', 'file to output to') + .option('-f, --file ', 'Output file, default output is stdout') .action(getLocalScript('configurationDump')); +program + .command('configuration:restore') + .option('-f, --file ', 'Input file, default input is stdin') + .option('-s, --strategy ', 'Strategy name, one of: "replace", "merge", "keep"') + .action(getLocalScript('configurationRestore')); + /** * Normalize help argument */ diff --git a/packages/strapi/lib/commands/configurationDump.js b/packages/strapi/lib/commands/configurationDump.js index c9480b0907..cd144d0d4f 100644 --- a/packages/strapi/lib/commands/configurationDump.js +++ b/packages/strapi/lib/commands/configurationDump.js @@ -1,25 +1,52 @@ 'use strict'; const fs = require('fs'); -const { logger } = require('strapi-utils'); -const loadConfiguration = require('../core/app-configuration'); const strapi = require('../index'); +const CHUNK_SIZE = 100; + +/** + * Will dump configurations to a file or stdout + * @param {string} file filepath to use as output + */ module.exports = async function({ file }) { const output = file ? fs.createWriteStream(file) : process.stdout; - output.write('this is a test'); - const app = strapi(); await app.load(); - const confs = await app.query('core_store').find({ - key_contains: 'plugin', - }); + const count = await app.query('core_store').count(); - console.log(confs); + const exportData = []; + const pageCount = Math.ceil(count / 100); + + for (let page = 0; page < pageCount; page++) { + const results = await app + .query('core_store') + .find({ _limit: CHUNK_SIZE, _start: page * CHUNK_SIZE }); + + results + .filter(result => result.key.startsWith('plugin_')) + .forEach(result => { + exportData.push({ + key: result.key, + value: result.value, + type: result.type, + environment: result.environment, + tag: result.tag, + }); + }); + } + + output.write(JSON.stringify(exportData)); output.write('\n'); output.end(); + + // log success only when writting to file + if (file) { + console.log(`Successfully exported ${exportData.length} configuration entries`); + } + process.exit(0); }; diff --git a/packages/strapi/lib/commands/configurationRestore.js b/packages/strapi/lib/commands/configurationRestore.js new file mode 100644 index 0000000000..1e94717fb0 --- /dev/null +++ b/packages/strapi/lib/commands/configurationRestore.js @@ -0,0 +1,113 @@ +'use strict'; + +const _ = require('lodash'); +const fs = require('fs'); +const strapi = require('../index'); + +/** + * Will restore configurations. It reads from a file or stdin + * @param {string} file filepath to use as input + * @param {string} strategy import strategy. one of (replace, merge, keep, default: replace) + */ +module.exports = async function({ file, strategy = 'replace' }) { + const input = file ? fs.readFileSync(file) : await readStdin(process.stdin); + + const app = strapi(); + await app.load(); + + let dataToImport; + try { + dataToImport = JSON.parse(input); + + if (!Array.isArray(dataToImport)) { + throw new Error(`Invalid input data. Expected a valid JSON array.`); + } + } catch (error) { + throw new Error(`Invalid input data: ${error.message}. Expected a valid JSON array.`); + } + + const importer = createImporter(app.db, strategy); + + for (const config of dataToImport) { + await importer.import(config); + } + + console.log(`Successfully imported ${dataToImport.length} configuration entries`); + process.exit(0); +}; + +const readStdin = () => { + const { stdin } = process; + let result = ''; + + if (stdin.isTTY) return Promise.resolve(result); + + return new Promise((resolve, reject) => { + stdin.setEncoding('utf8'); + stdin.on('readable', () => { + let chunk; + while ((chunk = stdin.read())) { + result += chunk; + } + }); + + stdin.on('end', () => { + resolve(result); + }); + + stdin.on('error', reject); + }); +}; + +const createImporter = (db, strategy) => { + switch (strategy) { + case 'replace': + return new ReplaceImporter(db); + case 'merge': + return new MergeImporter(db); + case 'keep': + return new KeepImporter(db); + default: + throw new Error(`No importer available for strategy "${strategy}"`); + } +}; + +function ReplaceImporter(db) { + return { + async import(conf) { + const matching = await db.query('core_store').count({ key: conf.key }); + if (matching > 0) { + await db.query('core_store').update({ key: conf.key }, conf); + } else { + await db.query('core_store').create(conf); + } + }, + }; +} + +function MergeImporter(db) { + return { + async import(conf) { + const existingConf = await db.query('core_store').find({ key: conf.key }); + if (existingConf) { + await db.query('core_store').update({ key: conf.key }, _.merge(existingConf, conf)); + } else { + await db.query('core_store').create(conf); + } + }, + }; +} + +function KeepImporter(db) { + return { + async import(conf) { + const matching = await db.query('core_store').count({ key: conf.key }); + if (matching > 0) { + // if configuration already exists do not overwrite it + return; + } + + await db.query('core_store').create(conf); + }, + }; +} From 937e3baca818a2dc984644df288a6bc94d9336ba Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Thu, 7 May 2020 18:49:44 +0200 Subject: [PATCH 3/8] Make strapi.load chainable Signed-off-by: Alexandre Bodin --- packages/strapi/lib/Strapi.js | 1 + packages/strapi/lib/commands/configurationDump.js | 4 +--- packages/strapi/lib/commands/configurationRestore.js | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/packages/strapi/lib/Strapi.js b/packages/strapi/lib/Strapi.js index 8b9acb5e95..180a9612ba 100644 --- a/packages/strapi/lib/Strapi.js +++ b/packages/strapi/lib/Strapi.js @@ -297,6 +297,7 @@ class Strapi { await this.runBootstrapFunctions(); await this.freeze(); + return this; } async startWebhooks() { diff --git a/packages/strapi/lib/commands/configurationDump.js b/packages/strapi/lib/commands/configurationDump.js index cd144d0d4f..b88a803f7e 100644 --- a/packages/strapi/lib/commands/configurationDump.js +++ b/packages/strapi/lib/commands/configurationDump.js @@ -12,9 +12,7 @@ const CHUNK_SIZE = 100; module.exports = async function({ file }) { const output = file ? fs.createWriteStream(file) : process.stdout; - const app = strapi(); - - await app.load(); + const app = await strapi().load(); const count = await app.query('core_store').count(); diff --git a/packages/strapi/lib/commands/configurationRestore.js b/packages/strapi/lib/commands/configurationRestore.js index 1e94717fb0..967bf6ac28 100644 --- a/packages/strapi/lib/commands/configurationRestore.js +++ b/packages/strapi/lib/commands/configurationRestore.js @@ -12,8 +12,7 @@ const strapi = require('../index'); module.exports = async function({ file, strategy = 'replace' }) { const input = file ? fs.readFileSync(file) : await readStdin(process.stdin); - const app = strapi(); - await app.load(); + const app = await strapi().load(); let dataToImport; try { From 768b699eb63b817a868697c44257cf7842194fb9 Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Tue, 12 May 2020 10:55:10 +0200 Subject: [PATCH 4/8] Add aliases Signed-off-by: Alexandre Bodin --- packages/strapi/bin/strapi.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/strapi/bin/strapi.js b/packages/strapi/bin/strapi.js index 524183ee42..47acca5c6b 100755 --- a/packages/strapi/bin/strapi.js +++ b/packages/strapi/bin/strapi.js @@ -216,11 +216,13 @@ program program .command('configuration:dump') + .alias('config:dump') .option('-f, --file ', 'Output file, default output is stdout') .action(getLocalScript('configurationDump')); program .command('configuration:restore') + .alias('config:restore') .option('-f, --file ', 'Input file, default input is stdin') .option('-s, --strategy ', 'Strategy name, one of: "replace", "merge", "keep"') .action(getLocalScript('configurationRestore')); From d1d13fab2393267211a6cb186f18d490a69b640f Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Tue, 12 May 2020 12:47:23 +0200 Subject: [PATCH 5/8] Cleanup code Signed-off-by: Alexandre Bodin --- packages/strapi/lib/Strapi.js | 5 +- .../strapi/lib/commands/configurationDump.js | 10 ++-- .../lib/commands/configurationRestore.js | 46 ++++++++++++------- 3 files changed, 38 insertions(+), 23 deletions(-) diff --git a/packages/strapi/lib/Strapi.js b/packages/strapi/lib/Strapi.js index 180a9612ba..9d1d9973a1 100644 --- a/packages/strapi/lib/Strapi.js +++ b/packages/strapi/lib/Strapi.js @@ -234,8 +234,6 @@ class Strapi { } async load() { - this.isLoaded = true; - this.app.use(async (ctx, next) => { if (ctx.request.url === '/_health' && ctx.request.method === 'HEAD') { ctx.set('strapi', 'You are so French!'); @@ -297,6 +295,9 @@ class Strapi { await this.runBootstrapFunctions(); await this.freeze(); + + this.isLoaded = true; + return this; } diff --git a/packages/strapi/lib/commands/configurationDump.js b/packages/strapi/lib/commands/configurationDump.js index b88a803f7e..58c08ee799 100644 --- a/packages/strapi/lib/commands/configurationDump.js +++ b/packages/strapi/lib/commands/configurationDump.js @@ -9,8 +9,8 @@ const CHUNK_SIZE = 100; * Will dump configurations to a file or stdout * @param {string} file filepath to use as output */ -module.exports = async function({ file }) { - const output = file ? fs.createWriteStream(file) : process.stdout; +module.exports = async function({ file: filePath }) { + const output = filePath ? fs.createWriteStream(filePath) : process.stdout; const app = await strapi().load(); @@ -18,12 +18,12 @@ module.exports = async function({ file }) { const exportData = []; - const pageCount = Math.ceil(count / 100); + const pageCount = Math.ceil(count / CHUNK_SIZE); for (let page = 0; page < pageCount; page++) { const results = await app .query('core_store') - .find({ _limit: CHUNK_SIZE, _start: page * CHUNK_SIZE }); + .find({ _limit: CHUNK_SIZE, _start: page * CHUNK_SIZE, _sort: 'key' }); results .filter(result => result.key.startsWith('plugin_')) @@ -43,7 +43,7 @@ module.exports = async function({ file }) { output.end(); // log success only when writting to file - if (file) { + if (filePath) { console.log(`Successfully exported ${exportData.length} configuration entries`); } process.exit(0); diff --git a/packages/strapi/lib/commands/configurationRestore.js b/packages/strapi/lib/commands/configurationRestore.js index 967bf6ac28..c7778732a4 100644 --- a/packages/strapi/lib/commands/configurationRestore.js +++ b/packages/strapi/lib/commands/configurationRestore.js @@ -9,29 +9,31 @@ const strapi = require('../index'); * @param {string} file filepath to use as input * @param {string} strategy import strategy. one of (replace, merge, keep, default: replace) */ -module.exports = async function({ file, strategy = 'replace' }) { - const input = file ? fs.readFileSync(file) : await readStdin(process.stdin); +module.exports = async function({ file: filePath, strategy = 'replace' }) { + const input = filePath ? fs.readFileSync(filePath) : await readStdin(process.stdin); const app = await strapi().load(); let dataToImport; try { dataToImport = JSON.parse(input); - - if (!Array.isArray(dataToImport)) { - throw new Error(`Invalid input data. Expected a valid JSON array.`); - } } catch (error) { throw new Error(`Invalid input data: ${error.message}. Expected a valid JSON array.`); } + if (!Array.isArray(dataToImport)) { + throw new Error(`Invalid input data. Expected a valid JSON array.`); + } + const importer = createImporter(app.db, strategy); for (const config of dataToImport) { await importer.import(config); } - console.log(`Successfully imported ${dataToImport.length} configuration entries`); + console.log( + `Successfully imported ${dataToImport.length} configuration entries using the "${strategy}" strategy.` + ); process.exit(0); }; @@ -61,17 +63,21 @@ const readStdin = () => { const createImporter = (db, strategy) => { switch (strategy) { case 'replace': - return new ReplaceImporter(db); + return createReplaceImporter(db); case 'merge': - return new MergeImporter(db); + return createMergeImporter(db); case 'keep': - return new KeepImporter(db); + return createKeepImporter(db); default: throw new Error(`No importer available for strategy "${strategy}"`); } }; -function ReplaceImporter(db) { +/** + * Replace importer. Will replace the keys that already exist and create the new ones + * @param {Object} db - DatabaseManager instance + */ +const createReplaceImporter = db => { return { async import(conf) { const matching = await db.query('core_store').count({ key: conf.key }); @@ -82,9 +88,13 @@ function ReplaceImporter(db) { } }, }; -} +}; -function MergeImporter(db) { +/** + * Merge importer. Will merge the keys that already exist with their new value and create the new ones + * @param {Object} db - DatabaseManager instance + */ +const createMergeImporter = db => { return { async import(conf) { const existingConf = await db.query('core_store').find({ key: conf.key }); @@ -95,9 +105,13 @@ function MergeImporter(db) { } }, }; -} +}; -function KeepImporter(db) { +/** + * Merge importer. Will keep the keys that already exist without changing them and create the new ones + * @param {Object} db - DatabaseManager instance + */ +const createKeepImporter = db => { return { async import(conf) { const matching = await db.query('core_store').count({ key: conf.key }); @@ -109,4 +123,4 @@ function KeepImporter(db) { await db.query('core_store').create(conf); }, }; -} +}; From 35f4b1748e0cf72e36ba33e8b00fa0d3dbd6e4da Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Tue, 12 May 2020 18:02:11 +0200 Subject: [PATCH 6/8] Add import stats Signed-off-by: Alexandre Bodin --- .../lib/commands/configurationRestore.js | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/packages/strapi/lib/commands/configurationRestore.js b/packages/strapi/lib/commands/configurationRestore.js index c7778732a4..706164bcac 100644 --- a/packages/strapi/lib/commands/configurationRestore.js +++ b/packages/strapi/lib/commands/configurationRestore.js @@ -32,8 +32,9 @@ module.exports = async function({ file: filePath, strategy = 'replace' }) { } console.log( - `Successfully imported ${dataToImport.length} configuration entries using the "${strategy}" strategy.` + `Successfully imported configuration with ${strategy} strategy. Statistics: ${importer.printStatistics()}.` ); + process.exit(0); }; @@ -78,12 +79,23 @@ const createImporter = (db, strategy) => { * @param {Object} db - DatabaseManager instance */ const createReplaceImporter = db => { + const stats = { + created: 0, + replaced: 0, + }; + return { + printStatistics() { + return `${stats.created} created, ${stats.replaced} replaced`; + }, + async import(conf) { const matching = await db.query('core_store').count({ key: conf.key }); if (matching > 0) { + stats.replaced += 1; await db.query('core_store').update({ key: conf.key }, conf); } else { + stats.created += 1; await db.query('core_store').create(conf); } }, @@ -95,12 +107,23 @@ const createReplaceImporter = db => { * @param {Object} db - DatabaseManager instance */ const createMergeImporter = db => { + const stats = { + created: 0, + merged: 0, + }; + return { + printStatistics() { + return `${stats.created} created, ${stats.merged} merged`; + }, + async import(conf) { const existingConf = await db.query('core_store').find({ key: conf.key }); if (existingConf) { + stats.merged += 1; await db.query('core_store').update({ key: conf.key }, _.merge(existingConf, conf)); } else { + stats.created += 1; await db.query('core_store').create(conf); } }, @@ -112,14 +135,25 @@ const createMergeImporter = db => { * @param {Object} db - DatabaseManager instance */ const createKeepImporter = db => { + const stats = { + created: 0, + untouched: 0, + }; + return { + printStatistics() { + return `${stats.created} created, ${stats.untouched} untouched`; + }, + async import(conf) { const matching = await db.query('core_store').count({ key: conf.key }); if (matching > 0) { + stats.untouched += 1; // if configuration already exists do not overwrite it return; } + stats.created += 1; await db.query('core_store').create(conf); }, }; From 79e9f6b1aca5951573980ddfac136bde0e4e0985 Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Wed, 13 May 2020 09:52:39 +0200 Subject: [PATCH 7/8] Add CLI documentation Signed-off-by: Alexandre Bodin --- docs/3.0.0-beta.x/cli/CLI.md | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/3.0.0-beta.x/cli/CLI.md b/docs/3.0.0-beta.x/cli/CLI.md index dd5d5512ef..2d33aa2617 100644 --- a/docs/3.0.0-beta.x/cli/CLI.md +++ b/docs/3.0.0-beta.x/cli/CLI.md @@ -78,6 +78,63 @@ options: [--no-optimization] - **strapi build --no-optimization**
Builds the administration panel without minimizing the assets. The build duration is faster. +## strapi configuration:dump|config:dump + +Dumps configurations to a file or stdout to help you migrate to production. + +The dump format will be a JSON array. + +``` +strapi configuration:dump + +Options: + -f, --file Output file, default output is stdout +``` + +**Examples** + +- `strapi configuration:dump -f dump.json` +- `strapi config:dump --file dump.json` +- `strapi config:dump > dump.json` + +All these examples are equivalent. + +::: warning +When configuring your application you often enter credentials for thrid party services (e.g authentication providers), Be aware that those credentials will also be dumped into the output of this command. +When you aren't sure of what you are doing, you should avoid commiting this into a versioning system and rather copy the dump to the environment you want to restore it into when needed. +::: + +## strapi configuration:restore|config:restore + +Restore a configuration dump into your application. + +The input format must be a JSON array. + +``` +strapi configuration:restore + +Options: + -f, --file Input file, default input is stdin + -s, --strategy Strategy name, one of: "replace", "merge", "keep". Defaults to: "replace" +``` + +**Examples** + +- `strapi configuration:restore -f dump.json` +- `strapi config:restore --file dump.json -s replace` +- `cat dump.json | strapi config:restore` +- `strapi config:restore < dump.json` + +All these examples are equivalent. + +**Strategies** + +When running the restore command, you can choose from three different strategies: + +- **replace**: Will create missing keys and replace existing ones. +- **merge**: Will create missing keys and merge existing keys whith there new value. +- **keep**: Will create missing keys and keep existing keys as is. + ## strapi generate:api Scaffold a complete API with its configurations, controller, model and service. From 2e31e79b60adc9a940ef6be9f5a57ed5d0f4b302 Mon Sep 17 00:00:00 2001 From: Alexandre Bodin Date: Wed, 13 May 2020 12:05:17 +0200 Subject: [PATCH 8/8] Udpate doc Signed-off-by: Alexandre Bodin --- docs/3.0.0-beta.x/cli/CLI.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/3.0.0-beta.x/cli/CLI.md b/docs/3.0.0-beta.x/cli/CLI.md index 2d33aa2617..9218e3fc83 100644 --- a/docs/3.0.0-beta.x/cli/CLI.md +++ b/docs/3.0.0-beta.x/cli/CLI.md @@ -100,13 +100,18 @@ Options: All these examples are equivalent. ::: warning -When configuring your application you often enter credentials for thrid party services (e.g authentication providers), Be aware that those credentials will also be dumped into the output of this command. -When you aren't sure of what you are doing, you should avoid commiting this into a versioning system and rather copy the dump to the environment you want to restore it into when needed. +When configuring your application you often enter credentials for thrid party services (e.g authentication providers). Be aware that those credentials will also be dumped into the output of this command. +In case of doubt, you should avoid committing the dump file into a versioning system. Here are some methods you can explore: + +- Copy the file directly to the environment you want and run the restore command there. +- Put the file in a secure location and download it at deploy time with the right credentials. +- Encrypt the file before committing and decrypt it when running the restore command. + ::: ## strapi configuration:restore|config:restore -Restore a configuration dump into your application. +Restores a configuration dump into your application. The input format must be a JSON array.