Implement support for custom seed sources (#4842)

Co-authored-by: maximelkin <maxelkin@list.ru>

lib/migrations/common/MigrationsLoader.js

@@ -0,0 +1,36 @@
+const path = require('path');
+const DEFAULT_LOAD_EXTENSIONS = Object.freeze([
+  '.co',
+  '.coffee',
+  '.eg',
+  '.iced',
+  '.js',
+  '.cjs',
+  '.litcoffee',
+  '.ls',
+  '.ts',
+]);
+
+class AbstractMigrationsLoader {
+  constructor(migrationDirectories, sortDirsSeparately, loadExtensions) {
+    this.sortDirsSeparately = sortDirsSeparately;
+
+    if (!Array.isArray(migrationDirectories)) {
+      migrationDirectories = [migrationDirectories];
+    }
+    this.migrationsPaths = migrationDirectories;
+    this.loadExtensions = loadExtensions || DEFAULT_LOAD_EXTENSIONS;
+  }
+
+  getFile(migrationsInfo) {
+    const absoluteDir = path.resolve(process.cwd(), migrationsInfo.directory);
+    const _path = path.join(absoluteDir, migrationsInfo.file);
+    const importFile = require('../util/import-file'); // late import
+    return importFile(_path);
+  }
+}
+
+module.exports = {
+  DEFAULT_LOAD_EXTENSIONS,
+  AbstractMigrationsLoader,
+};

lib/migrations/migrate/MigrationGenerator.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const { writeJsFileUsingTemplate } = require('../util/template');
-const { getMergedConfig } = require('./configuration-merger');
+const { getMergedConfig } = require('./migrator-configuration-merger');
 const { ensureDirectoryExists } = require('../util/fs');
 const { yyyymmddhhmmss } = require('../util/timestamp');
 

lib/migrations/migrate/Migrator.js

@@ -4,7 +4,6 @@ const differenceWith = require('lodash/differenceWith');
 const get = require('lodash/get');
 const isEmpty = require('lodash/isEmpty');
 const max = require('lodash/max');
-const { inherits } = require('util');
 const {
   getLockTableName,
   getTable,
@@ -13,16 +12,16 @@ const {
 const { getSchemaBuilder } = require('./table-creator');
 const migrationListResolver = require('./migration-list-resolver');
 const MigrationGenerator = require('./MigrationGenerator');
-const { getMergedConfig } = require('./configuration-merger');
+const { getMergedConfig } = require('./migrator-configuration-merger');
 const { isBoolean, isFunction } = require('../../util/is');
 
-function LockError(msg) {
-  this.name = 'MigrationLocked';
-  this.message = msg;
+class LockError extends Error {
+  constructor(msg) {
+    super(msg);
+    this.name = 'MigrationLocked';
+  }
 }
 
-inherits(LockError, Error);
-
 // The new migration we're performing, typically called from the `knex.migrate`
 // interface on the main `knex` object. Passes the `knex` instance performing
 // the migration.

lib/migrations/migrate/migrator-configuration-merger.js

@@ -1,8 +1,6 @@
-const {
-  FsMigrations,
-  DEFAULT_LOAD_EXTENSIONS,
-} = require('./sources/fs-migrations');
+const { FsMigrations } = require('./sources/fs-migrations');
 const Logger = require('../../logger');
+const { DEFAULT_LOAD_EXTENSIONS } = require('../common/MigrationsLoader');
 const defaultLogger = new Logger();
 
 const CONFIG_DEFAULT = Object.freeze({

lib/migrations/migrate/sources/fs-migrations.js

@@ -2,30 +2,9 @@ const path = require('path');
 const sortBy = require('lodash/sortBy');
 
 const { readdir } = require('../../util/fs');
+const { AbstractMigrationsLoader } = require('../../common/MigrationsLoader');
 
-const DEFAULT_LOAD_EXTENSIONS = Object.freeze([
-  '.co',
-  '.coffee',
-  '.eg',
-  '.iced',
-  '.js',
-  '.cjs',
-  '.litcoffee',
-  '.ls',
-  '.ts',
-]);
-
-class FsMigrations {
-  constructor(migrationDirectories, sortDirsSeparately, loadExtensions) {
-    this.sortDirsSeparately = sortDirsSeparately;
-
-    if (!Array.isArray(migrationDirectories)) {
-      migrationDirectories = [migrationDirectories];
-    }
-    this.migrationsPaths = migrationDirectories;
-    this.loadExtensions = loadExtensions || DEFAULT_LOAD_EXTENSIONS;
-  }
-
+class FsMigrations extends AbstractMigrationsLoader {
   /**
    * Gets the migration names
    * @returns Promise<string[]>
@@ -77,11 +56,8 @@ class FsMigrations {
     return migration.file;
   }
 
-  getMigration(migration) {
-    const absoluteDir = path.resolve(process.cwd(), migration.directory);
-    const _path = path.join(absoluteDir, migration.file);
-    const importFile = require('../../util/import-file'); // late import
-    return importFile(_path);
+  getMigration(migrationInfo) {
+    return this.getFile(migrationInfo);
   }
 }
 
@@ -94,6 +70,5 @@ function filterMigrations(migrationSource, migrations, loadExtensions) {
 }
 
 module.exports = {
-  DEFAULT_LOAD_EXTENSIONS,
   FsMigrations,
 };

lib/migrations/seed/Seeder.js

@@ -2,17 +2,10 @@
 // -------
 
 const path = require('path');
-const flatten = require('lodash/flatten');
-const extend = require('lodash/extend');
-const includes = require('lodash/includes');
-const { ensureDirectoryExists, getFilepathsInFolder } = require('../util/fs');
+const { ensureDirectoryExists } = require('../util/fs');
 const { writeJsFileUsingTemplate } = require('../util/template');
 const { yyyymmddhhmmss } = require('../util/timestamp');
-
-const filterByLoadExtensions = (extensions) => (value) => {
-  const extension = path.extname(value);
-  return includes(extensions, extension);
-};
+const { getMergedConfig } = require('./seeder-configuration-merger');
 
 // The new seeds we're performing, typically called from the `knex.seed`
 // interface on the main `knex` object. Passes the `knex` instance performing
@@ -20,27 +13,19 @@ const filterByLoadExtensions = (extensions) => (value) => {
 class Seeder {
   constructor(knex) {
     this.knex = knex;
-    this.config = this.setConfig(knex.client.config.seeds);
+    this.config = this.resolveConfig(knex.client.config.seeds);
   }
 
   // Runs seed files for the given environment.
   async run(config) {
-    this.config = this.setConfig(config);
-    let files = await this._listAll();
-    if (config && config.specific) {
-      files = files.filter((file) => path.basename(file) === config.specific);
-      if (files.length === 0) {
-        throw new Error(
-          `Invalid argument provided: the specific seed "${config.specific}" does not exist.`
-        );
-      }
-    }
+    this.config = this.resolveConfig(config);
+    const files = await this.config.seedSource.getSeeds(this.config);
     return this._runSeeds(files);
   }
 
   // Creates a new seed file, with a given name.
   async make(name, config) {
-    this.config = this.setConfig(config);
+    this.config = this.resolveConfig(config);
     if (!name)
       throw new Error('A name must be specified for the generated seed');
     await this._ensureFolder(config);
@@ -48,45 +33,26 @@ class Seeder {
     return seedPath;
   }
 
-  // Lists all available seed files as a sorted array.
-  async _listAll(config) {
-    this.config = this.setConfig(config);
-    const { loadExtensions, recursive } = this.config;
-    const seeds = flatten(
-      await Promise.all(
-        this._absoluteConfigDirs().map((d) =>
-          getFilepathsInFolder(d, recursive)
-        )
-      )
-    );
-    // if true, each dir are already sorted
-    // (getFilepathsInFolderRecursively does this)
-    // if false, we need to sort all the seeds
-    if (this.config.sortDirsSeparately) {
-      return seeds.filter(filterByLoadExtensions(loadExtensions));
-    } else {
-      return seeds.filter(filterByLoadExtensions(loadExtensions)).sort();
-    }
-  }
-
   // Ensures a folder for the seeds exist, dependent on the
   // seed config settings.
   _ensureFolder() {
-    const dirs = this._absoluteConfigDirs();
+    const dirs = this.config.seedSource._getConfigDirectories(
+      this.config.logger
+    );
     const promises = dirs.map(ensureDirectoryExists);
     return Promise.all(promises);
   }
 
   // Run seed files, in sequence.
-  _runSeeds(seeds) {
-    seeds.forEach((seed) => this._validateSeedStructure(seed));
+  async _runSeeds(seeds) {
+    for (const seed of seeds) {
+      await this._validateSeedStructure(seed);
+    }
     return this._waterfallBatch(seeds);
   }
 
-  // Validates seed files by requiring and checking for a `seed` function.
   async _validateSeedStructure(filepath) {
-    const importFile = require('../util/import-file'); // late import
-    const seed = await importFile(filepath);
+    const seed = await this.config.seedSource.getSeed(filepath);
     if (typeof seed.seed !== 'function') {
       throw new Error(
         `Invalid seed file: ${filepath} must have a seed function`
@@ -114,7 +80,9 @@ class Seeder {
 
   _getNewStubFilePath(name) {
     const fileName = this._getNewStubFileName(name);
-    const dirs = this._absoluteConfigDirs();
+    const dirs = this.config.seedSource._getConfigDirectories(
+      this.config.logger
+    );
     const dir = dirs.slice(-1)[0]; // Get last specified directory
     return path.join(dir, fileName);
   }
@@ -132,13 +100,17 @@ class Seeder {
     return seedPath;
   }
 
+  async _listAll(config) {
+    this.config = this.resolveConfig(config);
+    return this.config.seedSource.getSeeds(this.config);
+  }
+
   // Runs a batch of seed files.
   async _waterfallBatch(seeds) {
     const { knex } = this;
     const log = [];
     for (const seedPath of seeds) {
-      const importFile = require('../util/import-file'); // late import
-      const seed = await importFile(seedPath);
+      const seed = await this.config.seedSource.getSeed(seedPath);
       try {
         await seed.seed(knex);
         log.push(seedPath);
@@ -157,47 +129,8 @@ class Seeder {
     return [log];
   }
 
-  /**
-   * Return all the config directories
-   * @returns {string[]}
-   */
-  _absoluteConfigDirs() {
-    const directories = Array.isArray(this.config.directory)
-      ? this.config.directory
-      : [this.config.directory];
-    return directories.map((directory) => {
-      if (!directory) {
-        console.warn(
-          'Failed to resolve config file, knex cannot determine where to run or make seeds'
-        );
-      }
-      return path.resolve(process.cwd(), directory);
-    });
-  }
-
-  setConfig(config) {
-    return extend(
-      {
-        extension: 'js',
-        directory: './seeds',
-        loadExtensions: [
-          '.co',
-          '.coffee',
-          '.eg',
-          '.iced',
-          '.js',
-          '.litcoffee',
-          '.ls',
-          '.ts',
-          '.cjs',
-        ],
-        timestampFilenamePrefix: false,
-        sortDirsSeparately: false,
-        recursive: false,
-      },
-      this.config || {},
-      config
-    );
+  resolveConfig(config) {
+    return getMergedConfig(config, this.config, this.knex.client.logger);
   }
 }
 

lib/migrations/seed/seeder-configuration-merger.js

@@ -0,0 +1,60 @@
+const { FsSeeds } = require('./sources/fs-seeds');
+const Logger = require('../../logger');
+const { DEFAULT_LOAD_EXTENSIONS } = require('../common/MigrationsLoader');
+const defaultLogger = new Logger();
+
+const CONFIG_DEFAULT = Object.freeze({
+  extension: 'js',
+  directory: './seeds',
+  loadExtensions: DEFAULT_LOAD_EXTENSIONS,
+  specific: null,
+  timestampFilenamePrefix: false,
+  recursive: false,
+  sortDirsSeparately: false,
+});
+
+function getMergedConfig(config, currentConfig, logger = defaultLogger) {
+  // config is the user specified config, mergedConfig has defaults and current config
+  // applied to it.
+  const mergedConfig = Object.assign(
+    {},
+    CONFIG_DEFAULT,
+    currentConfig || {},
+    config,
+    {
+      logger,
+    }
+  );
+
+  if (
+    config &&
+    // If user specifies any FS related config,
+    // clear specified migrationSource to avoid ambiguity
+    (config.directory ||
+      config.sortDirsSeparately !== undefined ||
+      config.loadExtensions)
+  ) {
+    if (config.seedSource) {
+      logger.warn(
+        'FS-related option specified for seed configuration. This resets seedSource to default FsMigrations'
+      );
+    }
+    mergedConfig.seedSource = null;
+  }
+
+  // If the user has not specified any configs, we need to
+  // default to fs migrations to maintain compatibility
+  if (!mergedConfig.seedSource) {
+    mergedConfig.seedSource = new FsSeeds(
+      mergedConfig.directory,
+      mergedConfig.sortDirsSeparately,
+      mergedConfig.loadExtensions
+    );
+  }
+
+  return mergedConfig;
+}
+
+module.exports = {
+  getMergedConfig,
+};

lib/migrations/seed/sources/fs-seeds.js

@@ -0,0 +1,65 @@
+const path = require('path');
+const flatten = require('lodash/flatten');
+const includes = require('lodash/includes');
+const { AbstractMigrationsLoader } = require('../../common/MigrationsLoader');
+const { getFilepathsInFolder } = require('../../util/fs');
+
+const filterByLoadExtensions = (extensions) => (value) => {
+  const extension = path.extname(value);
+  return includes(extensions, extension);
+};
+
+class FsSeeds extends AbstractMigrationsLoader {
+  _getConfigDirectories(logger) {
+    const directories = this.migrationsPaths;
+    return directories.map((directory) => {
+      if (!directory) {
+        logger.warn(
+          'Empty value passed as a directory for Seeder, this is not supported.'
+        );
+      }
+      return path.resolve(process.cwd(), directory);
+    });
+  }
+
+  async getSeeds(config) {
+    const { loadExtensions, recursive, specific } = config;
+
+    const seeds = flatten(
+      await Promise.all(
+        this._getConfigDirectories(config.logger).map((d) =>
+          getFilepathsInFolder(d, recursive)
+        )
+      )
+    );
+
+    // if true, each dir are already sorted
+    // (getFilepathsInFolderRecursively does this)
+    // if false, we need to sort all the seeds
+    let files = seeds.filter(filterByLoadExtensions(loadExtensions));
+    if (!this.sortDirsSeparately) {
+      files.sort();
+    }
+
+    if (specific) {
+      files = files.filter((file) => path.basename(file) === specific);
+      if (files.length === 0) {
+        throw new Error(
+          `Invalid argument provided: the specific seed "${specific}" does not exist.`
+        );
+      }
+    }
+
+    return files;
+  }
+
+  async getSeed(filepath) {
+    const importFile = require('../../util/import-file'); // late import
+    const seed = await importFile(filepath);
+    return seed;
+  }
+}
+
+module.exports = {
+  FsSeeds,
+};

lib/migrations/util/import-file.js

@@ -4,7 +4,6 @@ const isModuleType = require('./is-module-type');
  * imports 'mjs', else requires.
  * NOTE: require me late!
  * @param {string} filepath
- * @todo WARN on version 10 and '--experimental-modules' and '--esm'
  */
 module.exports = async function importFile(filepath) {
   return (await isModuleType(filepath))

test/tape/migrate.js

@@ -3,7 +3,7 @@
 const tape = require('tape');
 const { Migrator } = require('../../lib/migrations/migrate/Migrator');
 const mergeConfig =
-  require('../../lib/migrations/migrate/configuration-merger').getMergedConfig;
+  require('../../lib/migrations/migrate/migrator-configuration-merger').getMergedConfig;
 
 tape('migrate: constructor uses config.migrations', function (t) {
   t.plan(1);

types/index.d.ts

@@ -2605,6 +2605,15 @@ export declare namespace Knex {
     forceFreeMigrationsLock(config?: MigratorConfig): Promise<any>;
   }
 
+  interface Seed {
+    seed: (knex: Knex) => PromiseLike<void>;
+  }
+
+  interface SeedSource<TSeedSpec> {
+    getSeeds(config: SeederConfig): Promise<TSeedSpec[]>;
+    getSeed(seed: TSeedSpec): Promise<Seed>;
+  }
+
   interface SeederConfig<V extends {} = any> {
     extension?: string;
     directory?: string | readonly string[];
@@ -2615,6 +2624,7 @@ export declare namespace Knex {
     sortDirsSeparately?: boolean;
     stub?: string;
     variables?: V;
+    seedSource?: SeedSource<unknown>;
   }
 
   class Seeder {