add backend telemetry events in ML

packages/core/strapi/package.json

@@ -120,7 +120,7 @@
     "mime-types": "2.1.35",
     "node-fetch": "2.6.7",
     "node-machine-id": "1.1.12",
-    "node-schedule": "2.0.0",
+    "node-schedule": "2.1.0",
     "open": "8.4.0",
     "ora": "5.4.1",
     "package-json": "7.0.0",

packages/core/upload/package.json

@@ -34,6 +34,7 @@
     "koa-static": "5.0.0",
     "lodash": "4.17.21",
     "mime-types": "2.1.35",
+    "node-schedule": "2.1.0",
     "react": "^17.0.2",
     "react-copy-to-clipboard": "^5.1.0",
     "react-dom": "^17.0.2",

packages/core/upload/server/bootstrap.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const { getService } = require('./utils');
+
 module.exports = async ({ strapi }) => {
   // set plugin store
   const configurator = strapi.store({ type: 'plugin', name: 'upload', key: 'settings' });
@@ -18,6 +20,8 @@ module.exports = async ({ strapi }) => {
   }
 
   await registerPermissionActions();
+
+  getService('metrics').startRegularMetricsUpdate();
 };
 
 const registerPermissionActions = async () => {

packages/core/upload/server/controllers/admin-folder-file.js

@@ -32,7 +32,20 @@ module.exports = {
     const folderService = getService('folder');
 
     const deletedFiles = await fileService.deleteByIds(body.fileIds);
-    const deletedFolders = await folderService.deleteByIds(body.folderIds);
+    const {
+      folders: deletedFolders,
+      totalFolderNumber,
+      totalFileNumber,
+    } = await folderService.deleteByIds(body.folderIds);
+
+    if (deletedFiles.length + deletedFolders.length > 1) {
+      strapi.telemetry.send('didBulkDeleteMediaLibraryElements', {
+        rootFolderNumber: deletedFolders.length,
+        rootAssetNumber: deletedFiles.length,
+        totalFolderNumber,
+        totalAssetNumber: totalFileNumber + deletedFiles.length,
+      });
+    }
 
     ctx.body = {
       data: {
@@ -61,6 +74,9 @@ module.exports = {
     await validateMoveManyFoldersFiles(body);
     const { folderIds = [], fileIds = [], destinationFolderId } = body;
 
+    let totalFolderNumber = 0;
+    let totalFileNumber = 0;
+
     const trx = await strapi.db.transaction();
     try {
       // fetch folders
@@ -137,7 +153,7 @@ module.exports = {
           }
 
           // update path for folders themselves & folders below
-          await strapi.db
+          totalFolderNumber = await strapi.db
             .connection(folderTable)
             .transacting(trx)
             .where(pathColName, existingFolder.path)
@@ -152,7 +168,7 @@ module.exports = {
             );
 
           // update path of files below
-          await strapi.db
+          totalFileNumber = await strapi.db
             .connection(fileTable)
             .transacting(trx)
             .where(folderPathColName, existingFolder.path)
@@ -212,6 +228,13 @@ module.exports = {
       filters: { id: { $in: fileIds } },
     });
 
+    strapi.telemetry.send('didBulkMoveMediaLibraryElements', {
+      rootFolderNumber: updatedFolders.length,
+      rootAssetNumber: updatedFiles.length,
+      totalFolderNumber,
+      totalAssetNumber: totalFileNumber + updatedFiles.length,
+    });
+
     ctx.body = {
       data: {
         files: await pmFile.sanitizeOutput(updatedFiles),

packages/core/upload/server/destroy.js

@@ -0,0 +1,7 @@
+'use strict';
+
+const { getService } = require('./utils');
+
+module.exports = () => {
+  getService('metrics').destroy();
+};

packages/core/upload/server/index.js

@@ -2,6 +2,7 @@
 
 const register = require('./register');
 const bootstrap = require('./bootstrap');
+const destroy = require('./destroy');
 const contentTypes = require('./content-types');
 const services = require('./services');
 const routes = require('./routes');
@@ -12,6 +13,7 @@ module.exports = () => {
   return {
     register,
     bootstrap,
+    destroy,
     config,
     routes,
     controllers,

packages/core/upload/server/services/folder.js

@@ -61,13 +61,17 @@ const deleteByIds = async (ids = []) => {
   await Promise.all(filesToDelete.map(file => getService('upload').remove(file)));
 
   // delete folders
-  await strapi.db.query(FOLDER_MODEL_UID).deleteMany({
+  const { count: totalFolderNumber } = await strapi.db.query(FOLDER_MODEL_UID).deleteMany({
     where: {
       $or: pathsToDelete.map(path => ({ path: { $startsWith: path } })),
     },
   });
 
-  return folders;
+  return {
+    folders,
+    totalFolderNumber,
+    totalFileNumber: filesToDelete.length,
+  };
 };
 
 /**

packages/core/upload/server/services/index.js

@@ -5,6 +5,7 @@ const upload = require('./upload');
 const imageManipulation = require('./image-manipulation');
 const folder = require('./folder');
 const file = require('./file');
+const metrics = require('./metrics');
 const apiUploadFolder = require('./api-upload-folder');
 
 module.exports = {
@@ -12,6 +13,7 @@ module.exports = {
   upload,
   folder,
   file,
+  metrics,
   'image-manipulation': imageManipulation,
   'api-upload-folder': apiUploadFolder,
 };

packages/core/upload/server/services/metrics.js

@@ -0,0 +1,86 @@
+'use strict';
+
+const { scheduleJob } = require('node-schedule');
+const { FOLDER_MODEL_UID, FILE_MODEL_UID } = require('../constants');
+
+const rand = max => Math.floor(Math.random() * max);
+const getCronRandomHour = () => `${rand(60)} ${rand(60)} ${rand(24)} ${rand(7)} * *`;
+
+module.exports = ({ strapi }) => {
+  const crons = [];
+  let started = false;
+
+  return {
+    async computeMetrics() {
+      // Folder metrics
+      const pathColName = strapi.db.metadata.get(FOLDER_MODEL_UID).attributes.path.columnName;
+      const folderTable = strapi.getModel(FOLDER_MODEL_UID).collectionName;
+
+      let keepOnlySlashesSQLString = '??';
+      let queryParams = [pathColName];
+      for (let i = 0; i < 10; i += 1) {
+        keepOnlySlashesSQLString = `REPLACE(${keepOnlySlashesSQLString}, ?, ?)`;
+        queryParams.push(String(i), '');
+      }
+
+      const knex = strapi.db.connection;
+      const folderLevelsArray = (
+        await knex(folderTable)
+          .select(
+            knex.raw(
+              `LENGTH(${keepOnlySlashesSQLString}) as depth, count(*) as occurence`,
+              queryParams
+            )
+          )
+          .groupBy('depth')
+      ).map(map => ({ depth: Number(map.depth), occurence: Number(map.occurence) })); // values can be strings depending on the database
+
+      let product = 0;
+      let folderNumber = 0;
+      let maxDepth = 0;
+      for (const folderLevel of folderLevelsArray) {
+        product += folderLevel.depth * folderLevel.occurence;
+        folderNumber += folderLevel.occurence;
+        if (folderLevel.depth > maxDepth) {
+          maxDepth = folderLevel.depth;
+        }
+      }
+      const averageDepth = product / folderNumber;
+
+      let sumOfDeviation = 0;
+      for (const folderLevel of folderLevelsArray) {
+        sumOfDeviation += Math.abs(folderLevel.depth * folderLevel.occurence - averageDepth);
+      }
+      const averageDeviationDepth = sumOfDeviation / folderNumber;
+
+      // File metrics
+      const assetNumber = await strapi.entityService.count(FILE_MODEL_UID);
+
+      return {
+        assetNumber,
+        folderNumber,
+        averageDepth,
+        maxDepth,
+        averageDeviationDepth,
+      };
+    },
+
+    async startRegularMetricsUpdate() {
+      if (started) {
+        throw new Error('Upload metrics already started');
+      }
+      started = true;
+
+      const pingCron = scheduleJob(getCronRandomHour(), async () => {
+        const metrics = await this.computeMetrics();
+        strapi.telemetry.send('mediaLibraryMetrics', metrics);
+      });
+
+      crons.push(pingCron);
+    },
+
+    destroy() {
+      crons.forEach(cron => cron.cancel());
+    },
+  };
+};

yarn.lock

@@ -10459,7 +10459,7 @@ create-hmac@^1.1.0, create-hmac@^1.1.4, create-hmac@^1.1.7:
     safe-buffer "^5.0.1"
     sha.js "^2.4.8"
 
-cron-parser@^3.1.0:
+cron-parser@^3.5.0:
   version "3.5.0"
   resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-3.5.0.tgz#b1a9da9514c0310aa7ef99c2f3f1d0f8c235257c"
   integrity sha512-wyVZtbRs6qDfFd8ap457w3XVntdvqcwBGxBoTvJQH9KGVKL/fB+h2k3C8AqiVxvUQKN1Ps/Ns46CNViOpVDhfQ==
@@ -18384,12 +18384,12 @@ node-releases@^2.0.5:
   resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.5.tgz#280ed5bc3eba0d96ce44897d8aee478bfb3d9666"
   integrity sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q==
 
-node-schedule@2.0.0:
-  version "2.0.0"
-  resolved "https://registry.yarnpkg.com/node-schedule/-/node-schedule-2.0.0.tgz#73ab4957d056c63708409cc1fab676e0e149c191"
-  integrity sha512-cHc9KEcfiuXxYDU+HjsBVo2FkWL1jRAUoczFoMIzRBpOA4p/NRHuuLs85AWOLgKsHtSPjN8csvwIxc2SqMv+CQ==
+node-schedule@2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/node-schedule/-/node-schedule-2.1.0.tgz#068ae38d7351c330616f7fe7cdb05036f977cbaf"
+  integrity sha512-nl4JTiZ7ZQDc97MmpTq9BQjYhq7gOtoh7SiPH069gBFBj0PzD8HI7zyFs6rzqL8Y5tTiEEYLxgtbx034YPrbyQ==
   dependencies:
-    cron-parser "^3.1.0"
+    cron-parser "^3.5.0"
     long-timeout "0.1.1"
     sorted-array-functions "^1.3.0"