From 1a8b6e78d83baf961028dae80bf0c433b77c7ed1 Mon Sep 17 00:00:00 2001 From: Jamie Howard Date: Thu, 13 Jul 2023 15:03:23 +0100 Subject: [PATCH 1/4] feature(ee): WIP weekly cron job --- packages/core/admin/ee/server/bootstrap.js | 2 + .../core/admin/ee/server/services/index.js | 1 + .../{metrics.js => metrics/index.js} | 11 ++++ .../metrics/weekly-metrics.js | 56 +++++++++++++++++++ .../server/services/metrics/weekly-metrics.js | 4 +- packages/core/upload/server/utils/cron.js | 8 --- .../src/__tests__/cron.test.ts} | 4 +- packages/core/utils/src/cron.ts | 6 ++ packages/core/utils/src/index.ts | 1 + 9 files changed, 79 insertions(+), 14 deletions(-) rename packages/core/admin/ee/server/services/review-workflows/{metrics.js => metrics/index.js} (71%) create mode 100644 packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js delete mode 100644 packages/core/upload/server/utils/cron.js rename packages/core/{upload/server/utils/__tests__/cron.test.js => utils/src/__tests__/cron.test.ts} (81%) create mode 100644 packages/core/utils/src/cron.ts diff --git a/packages/core/admin/ee/server/bootstrap.js b/packages/core/admin/ee/server/bootstrap.js index 678ce883e9..5467f8bc74 100644 --- a/packages/core/admin/ee/server/bootstrap.js +++ b/packages/core/admin/ee/server/bootstrap.js @@ -31,6 +31,8 @@ module.exports = async () => { // Decorate the entity service with review workflow logic const { decorator } = getService('review-workflows-decorator'); strapi.entityService.decorate(decorator); + + await getService('review-workflows-weekly-metrics').registerCron(); } await getService('seat-enforcement').seatEnforcementWorkflow(); diff --git a/packages/core/admin/ee/server/services/index.js b/packages/core/admin/ee/server/services/index.js index 11064d381a..d9f71e1b1b 100644 --- a/packages/core/admin/ee/server/services/index.js +++ b/packages/core/admin/ee/server/services/index.js @@ -12,4 +12,5 @@ module.exports = { 'review-workflows-validation': require('./review-workflows/validation'), 'review-workflows-decorator': require('./review-workflows/entity-service-decorator'), 'review-workflows-metrics': require('./review-workflows/metrics'), + 'review-workflows-weekly-metrics': require('./review-workflows/metrics/weekly-metrics'), }; diff --git a/packages/core/admin/ee/server/services/review-workflows/metrics.js b/packages/core/admin/ee/server/services/review-workflows/metrics/index.js similarity index 71% rename from packages/core/admin/ee/server/services/review-workflows/metrics.js rename to packages/core/admin/ee/server/services/review-workflows/metrics/index.js index 6e487588b2..7f64f18b3b 100644 --- a/packages/core/admin/ee/server/services/review-workflows/metrics.js +++ b/packages/core/admin/ee/server/services/review-workflows/metrics/index.js @@ -24,6 +24,16 @@ const sendDidEditWorkflow = async () => { strapi.telemetry.send('didEditWorkflow', {}); }; +const sendDidSendReviewWorkflowPropertiesOnceAWeek = async ( + numberOfActiveWorkflows, + activatedContentTypes +) => { + strapi.telemetry.send('didSendReviewWorkflowPropertiesOnceAWeek', { + numberOfActiveWorkflows, + activatedContentTypes, + }); +}; + module.exports = { sendDidCreateStage, sendDidEditStage, @@ -31,4 +41,5 @@ module.exports = { sendDidChangeEntryStage, sendDidCreateWorkflow, sendDidEditWorkflow, + sendDidSendReviewWorkflowPropertiesOnceAWeek, }; diff --git a/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js b/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js new file mode 100644 index 0000000000..ae9d782e6c --- /dev/null +++ b/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js @@ -0,0 +1,56 @@ +'use strict'; + +const { defaultTo } = require('lodash/fp'); +const { add } = require('date-fns'); + +const { ONE_WEEK, getWeeklyCronScheduleAt } = require('@strapi/utils').cron; +const { getService } = require('../../../../../server/utils'); + +const getMetricsStoreValue = async () => { + const value = await strapi.store.get({ type: 'plugin', name: 'ee', key: 'metrics' }); + return defaultTo({}, value); +}; +const setMetricsStoreValue = (value) => + strapi.store.set({ type: 'plugin', name: 'ee', key: 'metrics', value }); + +module.exports = ({ strapi }) => { + const metrics = getService('review-workflows-metrics', { strapi }); + + return { + async computeMetrics() { + /* + TODO: compute metrics + numberOfActiveWorkflows, + activatedContentTypes + */ + }, + + async sendMetrics() { + metrics.sendDidSendReviewWorkflowPropertiesOnceAWeek(); + + const metricsInfoStored = await getMetricsStoreValue(); + await setMetricsStoreValue({ ...metricsInfoStored, lastWeeklyUpdate: new Date().getTime() }); + }, + + async ensureWeeklyStoredCronSchedule() { + const metricsInfoStored = await getMetricsStoreValue(); + const { weeklySchedule: currentSchedule, lastWeeklyUpdate } = metricsInfoStored; + + const now = new Date(); + let weeklySchedule = currentSchedule; + + if (!currentSchedule || !lastWeeklyUpdate || lastWeeklyUpdate + ONE_WEEK < now.getTime()) { + weeklySchedule = getWeeklyCronScheduleAt(add(now, { minutes: 5 })); + await setMetricsStoreValue({ ...metricsInfoStored, weeklySchedule }); + } + + return weeklySchedule; + }, + + async registerCron() { + const weeklySchedule = await this.ensureWeeklyStoredCronSchedule(); + + strapi.cron.add({ [weeklySchedule]: this.sendMetrics.bind(this) }); + }, + }; +}; diff --git a/packages/core/upload/server/services/metrics/weekly-metrics.js b/packages/core/upload/server/services/metrics/weekly-metrics.js index a2074d3cc9..8c6afbeed7 100644 --- a/packages/core/upload/server/services/metrics/weekly-metrics.js +++ b/packages/core/upload/server/services/metrics/weekly-metrics.js @@ -2,10 +2,8 @@ const { defaultTo } = require('lodash/fp'); const { add } = require('date-fns'); +const { ONE_WEEK, getWeeklyCronScheduleAt } = require('@strapi/utils').cron; const { FOLDER_MODEL_UID, FILE_MODEL_UID } = require('../../constants'); -const { getWeeklyCronScheduleAt } = require('../../utils/cron'); - -const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; const getMetricsStoreValue = async () => { const value = await strapi.store.get({ type: 'plugin', name: 'upload', key: 'metrics' }); diff --git a/packages/core/upload/server/utils/cron.js b/packages/core/upload/server/utils/cron.js deleted file mode 100644 index a9397f8759..0000000000 --- a/packages/core/upload/server/utils/cron.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict'; - -const getWeeklyCronScheduleAt = (date) => - `${date.getSeconds()} ${date.getMinutes()} ${date.getHours()} * * ${date.getDay()}`; - -module.exports = { - getWeeklyCronScheduleAt, -}; diff --git a/packages/core/upload/server/utils/__tests__/cron.test.js b/packages/core/utils/src/__tests__/cron.test.ts similarity index 81% rename from packages/core/upload/server/utils/__tests__/cron.test.js rename to packages/core/utils/src/__tests__/cron.test.ts index dbbd281238..09af6c029d 100644 --- a/packages/core/upload/server/utils/__tests__/cron.test.js +++ b/packages/core/utils/src/__tests__/cron.test.ts @@ -1,6 +1,4 @@ -'use strict'; - -const { getWeeklyCronScheduleAt } = require('../cron'); +import { getWeeklyCronScheduleAt } from '../cron'; describe('cron', () => { describe('getWeeklyCronScheduleAt', () => { diff --git a/packages/core/utils/src/cron.ts b/packages/core/utils/src/cron.ts new file mode 100644 index 0000000000..e843b4098d --- /dev/null +++ b/packages/core/utils/src/cron.ts @@ -0,0 +1,6 @@ +const ONE_WEEK: number = 7 * 24 * 60 * 60 * 1000; +const getWeeklyCronScheduleAt = (date: Date): string => `0 * * * * *`; +// TODO revert to this +// `${date.getSeconds()} ${date.getMinutes()} ${date.getHours()} * * ${date.getDay()}`; + +export { ONE_WEEK, getWeeklyCronScheduleAt }; diff --git a/packages/core/utils/src/index.ts b/packages/core/utils/src/index.ts index 350395a96a..fb5334466b 100644 --- a/packages/core/utils/src/index.ts +++ b/packages/core/utils/src/index.ts @@ -41,3 +41,4 @@ export * as file from './file'; export * as traverse from './traverse'; export { default as webhook } from './webhook'; export { isOperator, isOperatorOfType } from './operators'; +export * as cron from './cron'; From e024f590abf18349a57710e8b76f63f8398643fd Mon Sep 17 00:00:00 2001 From: Marc-Roig Date: Tue, 18 Jul 2023 15:50:48 +0200 Subject: [PATCH 2/4] feat: compute review workflow weekly metrics --- .../review-workflows/metrics/index.js | 10 +++++-- .../metrics/weekly-metrics.js | 30 ++++++++++++++----- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/packages/core/admin/ee/server/services/review-workflows/metrics/index.js b/packages/core/admin/ee/server/services/review-workflows/metrics/index.js index 7f64f18b3b..062b089a1a 100644 --- a/packages/core/admin/ee/server/services/review-workflows/metrics/index.js +++ b/packages/core/admin/ee/server/services/review-workflows/metrics/index.js @@ -26,11 +26,17 @@ const sendDidEditWorkflow = async () => { const sendDidSendReviewWorkflowPropertiesOnceAWeek = async ( numberOfActiveWorkflows, + avgStagesCount, + maxStagesCount, activatedContentTypes ) => { strapi.telemetry.send('didSendReviewWorkflowPropertiesOnceAWeek', { - numberOfActiveWorkflows, - activatedContentTypes, + groupProperties: { + numberOfActiveWorkflows, + avgStagesCount, + maxStagesCount, + activatedContentTypes, + }, }); }; diff --git a/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js b/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js index ae9d782e6c..b77f9673ff 100644 --- a/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js +++ b/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js @@ -1,6 +1,6 @@ 'use strict'; -const { defaultTo } = require('lodash/fp'); +const { flow, map, sum, size, mean, max, defaultTo } = require('lodash/fp'); const { add } = require('date-fns'); const { ONE_WEEK, getWeeklyCronScheduleAt } = require('@strapi/utils').cron; @@ -15,18 +15,34 @@ const setMetricsStoreValue = (value) => module.exports = ({ strapi }) => { const metrics = getService('review-workflows-metrics', { strapi }); + const workflowsService = getService('workflows', { strapi }); return { async computeMetrics() { - /* - TODO: compute metrics - numberOfActiveWorkflows, - activatedContentTypes - */ + // There will never be more than 200 workflow, so we can safely fetch them all + const workflows = await workflowsService.findMany({ populate: 'stages' }); + + const stagesCount = flow( + map('stages'), // Number of stages per workflow + map(size) + )(workflows); + + const contentTypesCount = flow( + map('contentTypes'), // Number of content types per workflow + map(size) + )(workflows); + + return { + numberOfActiveWorkflows: size(workflows), + avgStagesCount: mean(stagesCount), + maxStagesCount: max(stagesCount), + activatedContentTypes: sum(contentTypesCount), + }; }, async sendMetrics() { - metrics.sendDidSendReviewWorkflowPropertiesOnceAWeek(); + const computedMetrics = this.computeMetrics(); + metrics.sendDidSendReviewWorkflowPropertiesOnceAWeek(computedMetrics); const metricsInfoStored = await getMetricsStoreValue(); await setMetricsStoreValue({ ...metricsInfoStored, lastWeeklyUpdate: new Date().getTime() }); From dc09aa2f8fcd7cbbed2bec00a603b36f45ed5a42 Mon Sep 17 00:00:00 2001 From: Marc-Roig Date: Tue, 18 Jul 2023 16:22:20 +0200 Subject: [PATCH 3/4] fix: move cron utils to upload --- .../review-workflows/metrics/weekly-metrics.js | 14 +++++++++----- .../server/services/metrics/weekly-metrics.js | 4 +++- .../server/utils/__tests__/cron.test.js} | 5 +++-- packages/core/upload/server/utils/cron.js | 8 ++++++++ packages/core/utils/src/cron.ts | 6 ------ packages/core/utils/src/index.ts | 1 - 6 files changed, 23 insertions(+), 15 deletions(-) rename packages/core/{utils/src/__tests__/cron.test.ts => upload/server/utils/__tests__/cron.test.js} (82%) create mode 100644 packages/core/upload/server/utils/cron.js delete mode 100644 packages/core/utils/src/cron.ts diff --git a/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js b/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js index b77f9673ff..fc61df3834 100644 --- a/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js +++ b/packages/core/admin/ee/server/services/review-workflows/metrics/weekly-metrics.js @@ -2,14 +2,18 @@ const { flow, map, sum, size, mean, max, defaultTo } = require('lodash/fp'); const { add } = require('date-fns'); - -const { ONE_WEEK, getWeeklyCronScheduleAt } = require('@strapi/utils').cron; const { getService } = require('../../../../../server/utils'); +const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; + +const getWeeklyCronScheduleAt = (date) => + `${date.getSeconds()} ${date.getMinutes()} ${date.getHours()} * * ${date.getDay()}`; + const getMetricsStoreValue = async () => { const value = await strapi.store.get({ type: 'plugin', name: 'ee', key: 'metrics' }); return defaultTo({}, value); }; + const setMetricsStoreValue = (value) => strapi.store.set({ type: 'plugin', name: 'ee', key: 'metrics', value }); @@ -20,7 +24,7 @@ module.exports = ({ strapi }) => { return { async computeMetrics() { // There will never be more than 200 workflow, so we can safely fetch them all - const workflows = await workflowsService.findMany({ populate: 'stages' }); + const workflows = await workflowsService.find({ populate: 'stages' }); const stagesCount = flow( map('stages'), // Number of stages per workflow @@ -41,7 +45,7 @@ module.exports = ({ strapi }) => { }, async sendMetrics() { - const computedMetrics = this.computeMetrics(); + const computedMetrics = await this.computeMetrics(); metrics.sendDidSendReviewWorkflowPropertiesOnceAWeek(computedMetrics); const metricsInfoStored = await getMetricsStoreValue(); @@ -56,7 +60,7 @@ module.exports = ({ strapi }) => { let weeklySchedule = currentSchedule; if (!currentSchedule || !lastWeeklyUpdate || lastWeeklyUpdate + ONE_WEEK < now.getTime()) { - weeklySchedule = getWeeklyCronScheduleAt(add(now, { minutes: 5 })); + weeklySchedule = getWeeklyCronScheduleAt(add(now, { seconds: 10 })); await setMetricsStoreValue({ ...metricsInfoStored, weeklySchedule }); } diff --git a/packages/core/upload/server/services/metrics/weekly-metrics.js b/packages/core/upload/server/services/metrics/weekly-metrics.js index 8c6afbeed7..e8ba5fb864 100644 --- a/packages/core/upload/server/services/metrics/weekly-metrics.js +++ b/packages/core/upload/server/services/metrics/weekly-metrics.js @@ -2,9 +2,11 @@ const { defaultTo } = require('lodash/fp'); const { add } = require('date-fns'); -const { ONE_WEEK, getWeeklyCronScheduleAt } = require('@strapi/utils').cron; +const { getWeeklyCronScheduleAt } = require('../../utils/cron'); const { FOLDER_MODEL_UID, FILE_MODEL_UID } = require('../../constants'); +const ONE_WEEK = 7 * 24 * 60 * 60 * 1000; + const getMetricsStoreValue = async () => { const value = await strapi.store.get({ type: 'plugin', name: 'upload', key: 'metrics' }); return defaultTo({}, value); diff --git a/packages/core/utils/src/__tests__/cron.test.ts b/packages/core/upload/server/utils/__tests__/cron.test.js similarity index 82% rename from packages/core/utils/src/__tests__/cron.test.ts rename to packages/core/upload/server/utils/__tests__/cron.test.js index 09af6c029d..9ef896bcd7 100644 --- a/packages/core/utils/src/__tests__/cron.test.ts +++ b/packages/core/upload/server/utils/__tests__/cron.test.js @@ -1,10 +1,11 @@ -import { getWeeklyCronScheduleAt } from '../cron'; +'use strict'; + +const { getWeeklyCronScheduleAt } = require('../cron'); describe('cron', () => { describe('getWeeklyCronScheduleAt', () => { test('2022-07-22T15:43:40.036 => 40 43 15 * * 5', () => { const date = new Date('2022-07-22T15:43:40.036'); // it's a friday - const result = getWeeklyCronScheduleAt(date); expect(result).toBe('40 43 15 * * 5'); }); diff --git a/packages/core/upload/server/utils/cron.js b/packages/core/upload/server/utils/cron.js new file mode 100644 index 0000000000..9fc47e7e72 --- /dev/null +++ b/packages/core/upload/server/utils/cron.js @@ -0,0 +1,8 @@ +('use strict'); + +const getWeeklyCronScheduleAt = (date) => + `${date.getSeconds()} ${date.getMinutes()} ${date.getHours()} * * ${date.getDay()}`; + +module.exports = { + getWeeklyCronScheduleAt, +}; diff --git a/packages/core/utils/src/cron.ts b/packages/core/utils/src/cron.ts deleted file mode 100644 index e843b4098d..0000000000 --- a/packages/core/utils/src/cron.ts +++ /dev/null @@ -1,6 +0,0 @@ -const ONE_WEEK: number = 7 * 24 * 60 * 60 * 1000; -const getWeeklyCronScheduleAt = (date: Date): string => `0 * * * * *`; -// TODO revert to this -// `${date.getSeconds()} ${date.getMinutes()} ${date.getHours()} * * ${date.getDay()}`; - -export { ONE_WEEK, getWeeklyCronScheduleAt }; diff --git a/packages/core/utils/src/index.ts b/packages/core/utils/src/index.ts index fb5334466b..350395a96a 100644 --- a/packages/core/utils/src/index.ts +++ b/packages/core/utils/src/index.ts @@ -41,4 +41,3 @@ export * as file from './file'; export * as traverse from './traverse'; export { default as webhook } from './webhook'; export { isOperator, isOperatorOfType } from './operators'; -export * as cron from './cron'; From 007c28987797ead51f3e0e51aa65782cd0aceedd Mon Sep 17 00:00:00 2001 From: Marc-Roig Date: Tue, 18 Jul 2023 16:48:50 +0200 Subject: [PATCH 4/4] fix: typo --- packages/core/upload/server/utils/cron.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/upload/server/utils/cron.js b/packages/core/upload/server/utils/cron.js index 9fc47e7e72..a9397f8759 100644 --- a/packages/core/upload/server/utils/cron.js +++ b/packages/core/upload/server/utils/cron.js @@ -1,4 +1,4 @@ -('use strict'); +'use strict'; const getWeeklyCronScheduleAt = (date) => `${date.getSeconds()} ${date.getMinutes()} ${date.getHours()} * * ${date.getDay()}`;