Use mongoose.find on simple queries for a perf boost

README.md

@@ -60,7 +60,7 @@ Node:
 **Please note that right now Node 11 is not supported, and the current Node LTS (v10) should be used.**
 
 Database:
- * MongoDB >= 3.x
+ * MongoDB >= 3.6
  * MySQL >= 5.6
  * MariaDB >= 10.1
  * PostgreSQL >= 10

packages/strapi-generate-api/templates/mongoose/service.template

@@ -64,8 +64,7 @@ module.exports = {
       model,
       filters: { where: filters.where },
     })
-      .count('count')
-      .then(results => _.get(results, [0, 'count'], 0));
+      .count()
   },
 
   /**

packages/strapi-hook-mongoose/lib/buildQuery.js

@@ -7,8 +7,52 @@ const utils = require('./utils')();
  * @param {Object} options.model - The model you are querying
  * @param {Object} options.filers - An object with the possible filters (start, limit, sort, where)
  * @param {Object} options.populate - An array of paths to populate
+ * @param {boolean} options.aggregate - Force aggregate function to use group by feature
  */
-const buildQuery = ({ model, filters, populate = [] } = {}) => {
+const buildQuery = ({ model, filters = {}, populate = [], aggregate = false } = {}) => {
+  const deepFilters = (filters.where || []).filter(({ field }) => field.split('.').length > 1);
+
+  if (deepFilters.length === 0 && aggregate === false) {
+    return buildSimpleQuery({ model, filters, populate });
+  }
+
+  return buildDeepQuery({ model, filters, populate });
+};
+
+/**
+ * Builds a simple find query when there are no deep filters
+ * @param {Object} options - Query options
+ * @param {Object} options.model - The model you are querying
+ * @param {Object} options.filers - An object with the possible filters (start, limit, sort, where)
+ * @param {Object} options.populate - An array of paths to populate
+ */
+const buildSimpleQuery = ({ model, filters, populate }) => {
+  const { where = [] } = filters;
+
+  const wheres = where.reduce(
+    (acc, whereClause) => _.assign(acc, buildWhereClause(whereClause)),
+    {}
+  );
+
+  let query = model.find(wheres).populate(populate);
+  query = applyQueryParams({ query, filters });
+
+  return Object.assign(query, {
+    // override count to use countDocuments on simple find query
+    count(...args) {
+      return query.countDocuments(...args);
+    },
+  });
+};
+
+/**
+ * Builds a deep aggregate query when there are deep filters
+ * @param {Object} options - Query options
+ * @param {Object} options.model - The model you are querying
+ * @param {Object} options.filers - An object with the possible filters (start, limit, sort, where)
+ * @param {Object} options.populate - An array of paths to populate
+ */
+const buildDeepQuery = ({ model, filters, populate }) => {
   // build a tree of paths to populate based on the filtering and the populate option
   const { populatePaths, wherePaths } = computePopulatedPaths({
     model,
@@ -21,6 +65,65 @@ const buildQuery = ({ model, filters, populate = [] } = {}) => {
     .aggregate(buildQueryAggregate(model, { paths: _.merge({}, populatePaths, wherePaths) }))
     .append(buildQueryMatches(model, filters));
 
+  query = applyQueryParams({ query, filters });
+
+  return {
+    /**
+     * Overrides the promise to rehydrate mongoose docs after the aggregation query
+     */
+    then(...args) {
+      // hydrate function
+      const hydrateFn = hydrateModel({
+        model,
+        populatedModels: populatePaths,
+      });
+
+      return query
+        .then(async result => {
+          const hydratedResults = await Promise.all(result.map(hydrateFn));
+
+          // TODO: run this only when necessary
+          const populatedResults = await model.populate(hydratedResults, [
+            {
+              path: 'related.ref',
+            },
+          ]);
+
+          return populatedResults;
+        })
+        .then(...args);
+    },
+    catch(...args) {
+      return this.then(r => r).catch(...args);
+    },
+    /**
+     * Maps to query.count
+     */
+    count() {
+      return query.count('count').then(results => _.get(results, ['0', 'count'], 0));
+    },
+
+    group(...args) {
+      return query.group(...args);
+    },
+    /**
+     * Returns an array of plain JS object instead of mongoose documents
+     */
+    lean() {
+      // return plain js objects without the transformations we normally do on find
+      return this.then(results => {
+        return results.map(r => r.toObject({ transform: false }));
+      });
+    },
+  };
+};
+
+/**
+ * @param {Object} options - Options
+ * @param {Object} options.query - Mongoose query
+ * @param {Object} options.filters - Filters object
+ */
+const applyQueryParams = ({ query, filters }) => {
   // apply sort
   if (_.has(filters, 'sort')) {
     const sortFilter = filters.sort.reduce((acc, sort) => {
@@ -42,60 +145,7 @@ const buildQuery = ({ model, filters, populate = [] } = {}) => {
     query = query.limit(filters.limit);
   }
 
-  return {
-    /**
-     * Overrides the promise to rehydrate mongoose docs after the aggregation query
-     */
-    then(onSuccess, onError) {
-      // hydrate function
-      const hydrateFn = hydrateModel({
-        model,
-        populatedModels: populatePaths,
-      });
-
-      return query
-        .then(async result => {
-          const hydratedResults = await Promise.all(result.map(hydrateFn));
-
-          // TODO: run this only when necessary
-          const populatedResults = await model.populate(hydratedResults, [
-            {
-              path: 'related.ref',
-            },
-          ]);
-
-          return populatedResults;
-        })
-        .then(onSuccess, onError);
-    },
-    /**
-     * Pass through query.catch
-     */
-    catch(...args) {
-      return query.catch(...args);
-    },
-    /**
-     * Maps to query.count
-     */
-    count(...args) {
-      return query.count(...args);
-    },
-    /**
-     * Maps to query.group
-     */
-    group(...args) {
-      return query.group(...args);
-    },
-    /**
-     * Returns an array of plain JS object instead of mongoose documents
-     */
-    lean() {
-      // return plain js objects without the transformations we normally do on find
-      return this.then(results => {
-        return results.map(r => r.toObject({ transform: false }));
-      });
-    },
-  };
+  return query;
 };
 
 /**
@@ -280,6 +330,11 @@ const buildLookupMatch = ({ assoc }) => {
   }
 };
 
+/**
+ * Match query for lookups
+ * @param {Object} model - Mongoose model
+ * @param {Object} filters - Filters object
+ */
 const buildQueryMatches = (model, filters) => {
   if (_.has(filters, 'where') && Array.isArray(filters.where)) {
     return filters.where.map(whereClause => {
@@ -292,6 +347,10 @@ const buildQueryMatches = (model, filters) => {
   return [];
 };
 
+/**
+ * Cast values
+ * @param {*} value - Value to cast
+ */
 const formatValue = value => {
   if (Array.isArray(value)) {
     return value.map(formatValue);
@@ -303,6 +362,13 @@ const formatValue = value => {
   return utils.valueToId(value);
 };
 
+/**
+ * Builds a where clause
+ * @param {Object} options - Options
+ * @param {string} options.field - Where clause field
+ * @param {string} options.operator - Where clause operator
+ * @param {*} options.value - Where clause alue
+ */
 const buildWhereClause = ({ field, operator, value }) => {
   const val = formatValue(value);
 
@@ -369,6 +435,14 @@ const buildWhereClause = ({ field, operator, value }) => {
   }
 };
 
+/**
+ * Add primaryKey on relation where clause for lookups match
+ * @param {Object} model - Mongoose model
+ * @param {Object} whereClause - Where clause
+ * @param {string} whereClause.field - Where clause field
+ * @param {string} whereClause.operator - Where clause operator
+ * @param {*} whereClause.value - Where clause alue
+ */
 const formatWhereClause = (model, { field, operator, value }) => {
   const { assoc, model: assocModel } = getAssociationFromFieldKey(model, field);
 
@@ -385,25 +459,36 @@ const formatWhereClause = (model, { field, operator, value }) => {
   };
 };
 
-const getAssociationFromFieldKey = (strapiModel, fieldKey) => {
-  let model = strapiModel;
+/**
+ * Returns an association from a path starting from model
+ * @param {Object} model - Mongoose model
+ * @param {string} fieldKey - Relation path
+ */
+const getAssociationFromFieldKey = (model, fieldKey) => {
+  let tmpModel = model;
   let assoc;
 
   const parts = fieldKey.split('.');
 
   for (let key of parts) {
-    assoc = model.associations.find(ast => ast.alias === key);
+    assoc = tmpModel.associations.find(ast => ast.alias === key);
     if (assoc) {
-      model = findModelByAssoc({ assoc });
+      tmpModel = findModelByAssoc({ assoc });
     }
   }
 
   return {
     assoc,
-    model,
+    model: tmpModel,
   };
 };
 
+/**
+ * Re hydrate mongoose model from lookup data
+ * @param {Object} options - Options
+ * @param {Object} options.model - Mongoose model
+ * @param {Object} options.populatedModels - Population models
+ */
 const hydrateModel = ({ model: rootModel, populatedModels }) => async obj => {
   const toSet = Object.keys(populatedModels).reduce((acc, key) => {
     const val = _.get(obj, key);
@@ -434,6 +519,12 @@ const hydrateModel = ({ model: rootModel, populatedModels }) => async obj => {
   return doc;
 };
 
+/**
+ * Returns a model from a realtion path and a root model
+ * @param {Object} options - Options
+ * @param {Object} options.rootModel - Mongoose model
+ * @param {string} options.path - Relation path
+ */
 const findModelByPath = ({ rootModel, path }) => {
   const parts = path.split('.');
 
@@ -448,6 +539,12 @@ const findModelByPath = ({ rootModel, path }) => {
   return tmpModel;
 };
 
+/**
+ * Returns a model path from an attribute path and a root model
+ * @param {Object} options - Options
+ * @param {Object} options.rootModel - Mongoose model
+ * @param {string} options.path - Attribute path
+ */
 const findModelPath = ({ rootModel, path }) => {
   const parts = path.split('.');
 

packages/strapi-plugin-content-manager/config/queries/mongoose.js

@@ -23,9 +23,7 @@ module.exports = {
     return buildQuery({
       model,
       filters: { where: filters.where },
-    })
-      .count('count')
-      .then(results => _.get(results, [0, 'count'], 0));
+    }).count();
   },
 
   search: async function(params, populate) {

packages/strapi-plugin-email/config/queries/mongoose.js

@@ -21,9 +21,7 @@ module.exports = {
     return buildQuery({
       model,
       filters: { where: filters.where },
-    })
-      .count('count')
-      .then(results => _.get(results, [0, 'count'], 0));
+    }).count();
   },
 
   findOne: async function(params, populate) {

packages/strapi-plugin-graphql/services/Aggregator.js

@@ -1,5 +1,3 @@
-'use strict';
-
 /**
  * Aggregator.js service
  *
@@ -176,11 +174,12 @@ const createAggregationFieldsResolver = function(model, fields, operation, typeC
     fields,
     async (filters, options, context, fieldResolver, fieldKey) => {
       // eslint-disable-line no-unused-vars
-      return buildQuery({ model, filters })
+      return buildQuery({ model, filters, aggregate: true })
         .group({
           _id: null,
           [fieldKey]: { [`$${operation}`]: `$${fieldKey}` },
         })
+        .exec()
         .then(result => _.get(result, [0, fieldKey]));
     },
     typeCheck
@@ -229,6 +228,7 @@ const createGroupByFieldsResolver = function(model, fields, name) {
     const result = await buildQuery({
       model,
       filters: convertRestQueryParams(params),
+      aggregate: true,
     }).group({
       _id: `$${fieldKey}`,
     });
@@ -325,9 +325,7 @@ const formatConnectionAggregator = function(fields, model) {
             limit: obj.limit,
             where: obj.where,
           },
-        })
-          .count('count')
-          .then(results => _.get(results, [0, 'count'], 0));
+        }).count();
       },
       totalCount: async (obj, options, context) => {
         return buildQuery({
@@ -335,9 +333,7 @@ const formatConnectionAggregator = function(fields, model) {
           filters: {
             where: obj.where,
           },
-        })
-          .count('count')
-          .then(results => _.get(results, [0, 'count'], 0));
+        }).count();
       },
     },
   };

packages/strapi-plugin-upload/config/queries/mongoose.js

@@ -21,9 +21,7 @@ module.exports = {
     return buildQuery({
       model,
       filters: { where: filters.where },
-    })
-      .count('count')
-      .then(results => _.get(results, [0, 'count'], 0));
+    }).count();
   },
 
   findOne: async function(params, populate) {

packages/strapi-plugin-users-permissions/config/queries/mongoose.js

@@ -21,9 +21,7 @@ module.exports = {
     return buildQuery({
       model,
       filters: { where: filters.where },
-    })
-      .count('count')
-      .then(results => _.get(results, [0, 'count'], 0));
+    }).count();
   },
 
   findOne: async function(params, populate) {

packages/strapi-utils/lib/buildQuery.js

@@ -31,10 +31,25 @@ const createFilterValidator = model => ({ field }) => {
   return isValid;
 };
 
+/**
+ *
+ * @param {Object} options - Options
+ * @param {Object} options.model - The model for which the query will be built
+ * @param {Object} options.filters - The filters for the query (start, sort, limit, and where clauses)
+ * @param {Object} options.rest - In case the database layer requires any other params pass them
+ */
 const buildQuery = ({ model, filters, ...rest }) => {
   const validator = createFilterValidator(model);
 
+  // Validate query clauses
   if (filters.where && Array.isArray(filters.where)) {
+    const deepFilters = filters.where.filter(({ field }) => field.split('.').length > 1);
+    if (deepFilters.length > 0) {
+      strapi.log.warn(
+        'Deep filtering queries should be used carefully (e.g Can cause performance issues).\nWhen possible build custom routes which will in most case be more optimised.'
+      );
+    }
+
     filters.where.forEach(whereClause => {
       if (!validator(whereClause)) {
         const err = new Error(
@@ -49,9 +64,10 @@ const buildQuery = ({ model, filters, ...rest }) => {
     });
   }
 
-  const hook = strapi.hook[model.orm];
+  const orm = strapi.hook[model.orm];
 
-  return hook.load().buildQuery({ model, filters, ...rest });
+  // call the orm's buildQuery implementation
+  return orm.load().buildQuery({ model, filters, ...rest });
 };
 
 module.exports = buildQuery;